@anvia/core 1.5.1 → 1.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/README.md +32 -1073
  2. package/dist/agent/agent-tool.js +4 -4
  3. package/dist/agent/agent.js +17 -17
  4. package/dist/agent/index.js +20 -20
  5. package/dist/agent/resolve-options.js +6 -6
  6. package/dist/agent/resolved-agent.js +18 -18
  7. package/dist/agent/team/agent-team.js +20 -20
  8. package/dist/agent/team/index.js +20 -20
  9. package/dist/agent/tool-catalog.js +6 -6
  10. package/dist/{chunk-QHGYCYV6.js → chunk-3FIX7GLM.js} +2 -2
  11. package/dist/{chunk-SIXS4CCU.js → chunk-4IOC4SP6.js} +4 -4
  12. package/dist/chunk-4IOC4SP6.js.map +1 -0
  13. package/dist/{chunk-JMMZICU7.js → chunk-4OSA3MMA.js} +39 -5
  14. package/dist/chunk-4OSA3MMA.js.map +1 -0
  15. package/dist/{chunk-YOLOFTAY.js → chunk-57AUTEQ6.js} +57 -5
  16. package/dist/chunk-57AUTEQ6.js.map +1 -0
  17. package/dist/{chunk-4VDDYI6P.js → chunk-5CQRCTIN.js} +2 -2
  18. package/dist/{chunk-F4BOW76L.js → chunk-5TGVDMGM.js} +4 -4
  19. package/dist/{chunk-TK2VXQVZ.js → chunk-7G3YQQVE.js} +3 -3
  20. package/dist/{chunk-NDAGW32C.js → chunk-7OHHMYRO.js} +2 -2
  21. package/dist/{chunk-3SUEXGRA.js → chunk-7V3DKQTI.js} +4 -4
  22. package/dist/{chunk-DXW37JLZ.js → chunk-AMPQJMDG.js} +5 -5
  23. package/dist/{chunk-ZQC6M22O.js → chunk-AQ47TF5K.js} +2 -2
  24. package/dist/{chunk-PZ7ATNR6.js → chunk-BI3T62GL.js} +9 -9
  25. package/dist/chunk-BI3T62GL.js.map +1 -0
  26. package/dist/{chunk-HAAGSIZH.js → chunk-CB2L2NIQ.js} +2 -2
  27. package/dist/{chunk-KX2TRKO2.js → chunk-GNBPXJC6.js} +2 -2
  28. package/dist/{chunk-YMTBHHLQ.js → chunk-HG6ASDNI.js} +3 -3
  29. package/dist/{chunk-JMNIWUFV.js → chunk-HILMAFZK.js} +2 -2
  30. package/dist/{chunk-PWWKX2HA.js → chunk-LGMH6PCB.js} +7 -7
  31. package/dist/{chunk-PPMK7CG4.js → chunk-LW6F3IGP.js} +2 -2
  32. package/dist/{chunk-UYODZ7OE.js → chunk-LWF6AHDG.js} +2 -2
  33. package/dist/{chunk-2JZV4LUS.js → chunk-MKY7KTZY.js} +3 -13
  34. package/dist/chunk-MKY7KTZY.js.map +1 -0
  35. package/dist/{chunk-RFFOL6ON.js → chunk-MWG2PMTE.js} +18 -5
  36. package/dist/chunk-MWG2PMTE.js.map +1 -0
  37. package/dist/{chunk-AJC6YRVB.js → chunk-O7NN7PF4.js} +3 -3
  38. package/dist/{chunk-GDY2Z2SZ.js → chunk-PFJN5OIQ.js} +2 -2
  39. package/dist/{chunk-XAVSWLX5.js → chunk-PV6KJWYO.js} +2 -2
  40. package/dist/chunk-PZC5UHIN.js +190 -0
  41. package/dist/chunk-PZC5UHIN.js.map +1 -0
  42. package/dist/{chunk-V5DWNXVH.js → chunk-W2XPREWL.js} +135 -26
  43. package/dist/chunk-W2XPREWL.js.map +1 -0
  44. package/dist/chunk-W6V63YPY.js +96 -0
  45. package/dist/chunk-W6V63YPY.js.map +1 -0
  46. package/dist/{chunk-LJGQ6VQB.js → chunk-XJ5X3NI4.js} +89 -30
  47. package/dist/chunk-XJ5X3NI4.js.map +1 -0
  48. package/dist/{chunk-HHMLE3TZ.js → chunk-YCTXTOOQ.js} +7 -16
  49. package/dist/chunk-YCTXTOOQ.js.map +1 -0
  50. package/dist/{chunk-7TUAAGVR.js → chunk-YKWUFYXT.js} +2 -2
  51. package/dist/{chunk-5SJBSCG2.js → chunk-YSR6KLCS.js} +8 -2
  52. package/dist/chunk-YSR6KLCS.js.map +1 -0
  53. package/dist/completion/generate-completion.js +3 -3
  54. package/dist/completion/index.js +3 -3
  55. package/dist/completion/stream-accumulator.js +1 -1
  56. package/dist/embeddings/embed.js +1 -1
  57. package/dist/embeddings/index.js +1 -1
  58. package/dist/evals/advanced-metrics.js +6 -6
  59. package/dist/evals/agent-target.js +2 -1
  60. package/dist/evals/cli.js +6 -6
  61. package/dist/evals/execution.js +3 -1
  62. package/dist/evals/index.js +12 -12
  63. package/dist/evals/judge.js +5 -5
  64. package/dist/evals/metrics.js +6 -6
  65. package/dist/evals/runner.js +5 -5
  66. package/dist/extractor/extractor.js +4 -4
  67. package/dist/extractor/index.js +4 -4
  68. package/dist/index.js +23 -23
  69. package/dist/internal/agent-runtime/agent-run.js +13 -13
  70. package/dist/internal/agent-runtime/continuation-state.js +3 -3
  71. package/dist/internal/agent-runtime/interaction-suspension.js +3 -3
  72. package/dist/internal/agent-runtime/memory.js +5 -5
  73. package/dist/internal/agent-runtime/retrieval.js +5 -5
  74. package/dist/internal/agent-runtime/tool-execution.js +8 -8
  75. package/dist/internal/agent.js +18 -18
  76. package/dist/internal/team-runtime/member.js +3 -3
  77. package/dist/internal/team-runtime/run.js +19 -19
  78. package/dist/memory/compaction.js +4 -4
  79. package/dist/memory/index.js +4 -4
  80. package/dist/observability/group.js +3 -3
  81. package/dist/observability/index.js +3 -3
  82. package/dist/pipeline/index.js +6 -6
  83. package/dist/pipeline/pipeline.js +6 -6
  84. package/dist/schema/standard-json-schema.js +1 -1
  85. package/dist/skills/index.js +7 -7
  86. package/dist/skills/load.js +7 -7
  87. package/dist/skills/tools.js +6 -6
  88. package/dist/streaming/index.d.ts +2 -1
  89. package/dist/streaming/index.js +2 -1
  90. package/dist/streaming/readable-stream.js +2 -1
  91. package/dist/tool/dynamic-tools.js +5 -5
  92. package/dist/tool/index.js +5 -5
  93. package/dist/vector-store/index.js +4 -4
  94. package/dist/vector-store/ingest.js +2 -2
  95. package/dist/vector-store/retrieve.js +2 -2
  96. package/package.json +1 -1
  97. package/dist/chunk-2JZV4LUS.js.map +0 -1
  98. package/dist/chunk-54ZZJRP2.js +0 -57
  99. package/dist/chunk-54ZZJRP2.js.map +0 -1
  100. package/dist/chunk-5SJBSCG2.js.map +0 -1
  101. package/dist/chunk-HHMLE3TZ.js.map +0 -1
  102. package/dist/chunk-JMMZICU7.js.map +0 -1
  103. package/dist/chunk-LJGQ6VQB.js.map +0 -1
  104. package/dist/chunk-NNEKBZSM.js +0 -66
  105. package/dist/chunk-NNEKBZSM.js.map +0 -1
  106. package/dist/chunk-PZ7ATNR6.js.map +0 -1
  107. package/dist/chunk-RFFOL6ON.js.map +0 -1
  108. package/dist/chunk-SIXS4CCU.js.map +0 -1
  109. package/dist/chunk-V5DWNXVH.js.map +0 -1
  110. package/dist/chunk-YOLOFTAY.js.map +0 -1
  111. /package/dist/{chunk-QHGYCYV6.js.map → chunk-3FIX7GLM.js.map} +0 -0
  112. /package/dist/{chunk-4VDDYI6P.js.map → chunk-5CQRCTIN.js.map} +0 -0
  113. /package/dist/{chunk-F4BOW76L.js.map → chunk-5TGVDMGM.js.map} +0 -0
  114. /package/dist/{chunk-TK2VXQVZ.js.map → chunk-7G3YQQVE.js.map} +0 -0
  115. /package/dist/{chunk-NDAGW32C.js.map → chunk-7OHHMYRO.js.map} +0 -0
  116. /package/dist/{chunk-3SUEXGRA.js.map → chunk-7V3DKQTI.js.map} +0 -0
  117. /package/dist/{chunk-DXW37JLZ.js.map → chunk-AMPQJMDG.js.map} +0 -0
  118. /package/dist/{chunk-ZQC6M22O.js.map → chunk-AQ47TF5K.js.map} +0 -0
  119. /package/dist/{chunk-HAAGSIZH.js.map → chunk-CB2L2NIQ.js.map} +0 -0
  120. /package/dist/{chunk-KX2TRKO2.js.map → chunk-GNBPXJC6.js.map} +0 -0
  121. /package/dist/{chunk-YMTBHHLQ.js.map → chunk-HG6ASDNI.js.map} +0 -0
  122. /package/dist/{chunk-JMNIWUFV.js.map → chunk-HILMAFZK.js.map} +0 -0
  123. /package/dist/{chunk-PWWKX2HA.js.map → chunk-LGMH6PCB.js.map} +0 -0
  124. /package/dist/{chunk-PPMK7CG4.js.map → chunk-LW6F3IGP.js.map} +0 -0
  125. /package/dist/{chunk-UYODZ7OE.js.map → chunk-LWF6AHDG.js.map} +0 -0
  126. /package/dist/{chunk-AJC6YRVB.js.map → chunk-O7NN7PF4.js.map} +0 -0
  127. /package/dist/{chunk-GDY2Z2SZ.js.map → chunk-PFJN5OIQ.js.map} +0 -0
  128. /package/dist/{chunk-XAVSWLX5.js.map → chunk-PV6KJWYO.js.map} +0 -0
  129. /package/dist/{chunk-7TUAAGVR.js.map → chunk-YKWUFYXT.js.map} +0 -0
@@ -2,7 +2,7 @@ import {
2
2
  addUsage,
3
3
  evaluationMetadata,
4
4
  runJudge
5
- } from "./chunk-5SJBSCG2.js";
5
+ } from "./chunk-YSR6KLCS.js";
6
6
  import {
7
7
  resolveActualText,
8
8
  resolveExpected
@@ -23,6 +23,9 @@ import {
23
23
  import {
24
24
  isJsonValue
25
25
  } from "./chunk-4C7AXYOM.js";
26
+ import {
27
+ throwIfAborted
28
+ } from "./chunk-USRKPEQN.js";
26
29
 
27
30
  // src/evals/advanced-metrics.ts
28
31
  import { z } from "zod";
@@ -68,6 +71,7 @@ function answerRelevancy(options) {
68
71
  instructions: "Break the answer into concise, independently assessable statements. Return every substantive statement using the schema.",
69
72
  prompt: `Answer:
70
73
  ${actual}`,
74
+ abortSignal: args.signal,
71
75
  retries: config.retries
72
76
  });
73
77
  const statements = statementResult.data.statements;
@@ -79,6 +83,7 @@ ${actual}`,
79
83
  schema: verdictsSchema,
80
84
  instructions: "Classify each answer statement for relevance to the user input. Use yes for relevant, no for irrelevant, and idk only when relevance is genuinely indeterminate. Preserve order and return one verdict per statement.",
81
85
  prompt: jsonPrompt({ input, statements }),
86
+ abortSignal: args.signal,
82
87
  retries: config.retries
83
88
  });
84
89
  verdicts = verdictResult.data.verdicts;
@@ -92,6 +97,7 @@ ${actual}`,
92
97
  includeReason: config.includeReason,
93
98
  retries: config.retries,
94
99
  metric: "answer relevancy",
100
+ abortSignal: args.signal,
95
101
  score,
96
102
  evidence: { input, verdicts: serializedVerdicts }
97
103
  });
@@ -123,6 +129,7 @@ function promptAlignment(options) {
123
129
  schema: binaryVerdictsSchema,
124
130
  instructions: "Determine whether the answer follows each prompt instruction. Preserve order and return exactly one yes or no verdict per instruction.",
125
131
  prompt: jsonPrompt({ input, actual, instructions: options.promptInstructions }),
132
+ abortSignal: args.signal,
126
133
  retries: config.retries
127
134
  });
128
135
  const verdicts = verdictResult.data.verdicts;
@@ -133,6 +140,7 @@ function promptAlignment(options) {
133
140
  includeReason: config.includeReason,
134
141
  retries: config.retries,
135
142
  metric: "prompt alignment",
143
+ abortSignal: args.signal,
136
144
  score,
137
145
  evidence: { verdicts }
138
146
  });
@@ -193,6 +201,7 @@ function jsonCorrectness(options) {
193
201
  schema: reasonSchema,
194
202
  instructions: "Briefly explain why the generated JSON does not match the expected schema. Focus on actionable syntax, field, and type problems.",
195
203
  prompt: jsonPrompt({ actual, validationError }),
204
+ abortSignal: args.signal,
196
205
  retries
197
206
  });
198
207
  comment = reasonResult.data.reason;
@@ -224,6 +233,7 @@ function hallucination(options) {
224
233
  schema: binaryVerdictsSchema,
225
234
  instructions: "Compare the answer with each trusted context. Use yes when the answer is factually aligned with that context and no when it contradicts it. Preserve order and return one verdict per context.",
226
235
  prompt: jsonPrompt({ actual, context }),
236
+ abortSignal: args.signal,
227
237
  retries: config.retries
228
238
  });
229
239
  const verdicts = verdictResult.data.verdicts;
@@ -234,6 +244,7 @@ function hallucination(options) {
234
244
  includeReason: config.includeReason,
235
245
  retries: config.retries,
236
246
  metric: "hallucination",
247
+ abortSignal: args.signal,
237
248
  score,
238
249
  evidence: { verdicts }
239
250
  });
@@ -272,6 +283,7 @@ function faithfulness(options) {
272
283
  schema: factsSchema,
273
284
  instructions: truthsExtractionInstructions(truthsExtractionLimit),
274
285
  prompt: jsonPrompt({ retrievalContext }),
286
+ abortSignal: args.signal,
275
287
  retries: config.retries
276
288
  }),
277
289
  runJudge({
@@ -280,6 +292,7 @@ function faithfulness(options) {
280
292
  instructions: "Extract every concise factual claim made by the answer. Return claims in the facts array and omit opinions or purely stylistic text.",
281
293
  prompt: `Answer:
282
294
  ${actual}`,
295
+ abortSignal: args.signal,
283
296
  retries: config.retries
284
297
  })
285
298
  ]);
@@ -293,6 +306,7 @@ ${actual}`,
293
306
  schema: verdictsSchema,
294
307
  instructions: "Determine whether each answer claim is supported by the supplied truths. Use yes for supported, no for contradicted or unsupported, and idk for genuinely ambiguous support. Preserve order and return one verdict per claim.",
295
308
  prompt: jsonPrompt({ truths, claims }),
309
+ abortSignal: args.signal,
296
310
  retries: config.retries
297
311
  });
298
312
  verdicts = verdictResult.data.verdicts;
@@ -310,6 +324,7 @@ ${actual}`,
310
324
  includeReason: config.includeReason,
311
325
  retries: config.retries,
312
326
  metric: "faithfulness",
327
+ abortSignal: args.signal,
313
328
  score,
314
329
  evidence: { verdicts: serializedVerdicts, penalizeAmbiguousClaims }
315
330
  });
@@ -335,6 +350,7 @@ function abstention(options) {
335
350
  dataType: "CATEGORICAL",
336
351
  async evaluate(args) {
337
352
  try {
353
+ throwIfAborted(args.signal);
338
354
  const actual = await resolveActualText(options.actual, args);
339
355
  const shouldAbstain = await resolveExpected(options.shouldAbstain, args);
340
356
  if (typeof shouldAbstain !== "boolean") {
@@ -347,12 +363,14 @@ function abstention(options) {
347
363
  );
348
364
  }
349
365
  const judgment = await runJudge({
366
+ abortSignal: args.signal,
350
367
  model: options.model,
351
368
  schema: abstentionJudgmentSchema,
352
369
  instructions: "Classify whether the answer abstains or gives a confident answer. For a confident answer, grounded is true only when every substantive factual claim is supported by the supplied context. For an abstention, set grounded to false. Return a concise evidence-based reason.",
353
370
  prompt: jsonPrompt({ actual, context }),
354
371
  retries
355
372
  });
373
+ throwIfAborted(args.signal);
356
374
  const category = abstentionCategory(
357
375
  shouldAbstain,
358
376
  judgment.data.behavior,
@@ -409,6 +427,7 @@ function summarization(options) {
409
427
  instructions: `Generate exactly ${questionCount} important yes-or-no assessment questions whose answers capture the source text's essential information.`,
410
428
  prompt: `Source text:
411
429
  ${input}`,
430
+ abortSignal: args.signal,
412
431
  retries: config.retries
413
432
  }) : Promise.resolve({
414
433
  data: { questions: suppliedQuestions },
@@ -421,6 +440,7 @@ ${input}`,
421
440
  instructions: truthsExtractionInstructions(truthsExtractionLimit),
422
441
  prompt: `Source text:
423
442
  ${input}`,
443
+ abortSignal: args.signal,
424
444
  retries: config.retries
425
445
  }),
426
446
  runJudge({
@@ -429,6 +449,7 @@ ${input}`,
429
449
  instructions: "Extract every concise factual claim made by the summary. Return claims in the facts array.",
430
450
  prompt: `Summary:
431
451
  ${actual}`,
452
+ abortSignal: args.signal,
432
453
  retries: config.retries
433
454
  }),
434
455
  questionPromise
@@ -444,6 +465,7 @@ ${actual}`,
444
465
  schema: answersSchema,
445
466
  instructions: "Answer each assessment question using only the supplied text. Return one yes or no answer per question in the same order.",
446
467
  prompt: jsonPrompt({ questions, text: input }),
468
+ abortSignal: args.signal,
447
469
  retries: config.retries
448
470
  });
449
471
  const summaryAnswerPromise = runJudge({
@@ -451,6 +473,7 @@ ${actual}`,
451
473
  schema: answersSchema,
452
474
  instructions: "Answer each assessment question using only the supplied text. Return one yes or no answer per question in the same order.",
453
475
  prompt: jsonPrompt({ questions, text: actual }),
476
+ abortSignal: args.signal,
454
477
  retries: config.retries
455
478
  });
456
479
  const alignmentPromise = claims.length === 0 ? Promise.resolve({ data: { verdicts: [] }, usage: Usage.empty() }) : runJudge({
@@ -458,6 +481,7 @@ ${actual}`,
458
481
  schema: verdictsSchema,
459
482
  instructions: "Determine whether each summary claim is supported by the source truths. Use yes for supported, no for contradicted, and idk for unsupported filler or ambiguity. Preserve order.",
460
483
  prompt: jsonPrompt({ truths, claims }),
484
+ abortSignal: args.signal,
461
485
  retries: config.retries
462
486
  });
463
487
  const [sourceAnswerResult, summaryAnswerResult, alignmentResult] = await Promise.all([
@@ -489,6 +513,7 @@ ${actual}`,
489
513
  includeReason: config.includeReason,
490
514
  retries: config.retries,
491
515
  metric: "summarization",
516
+ abortSignal: args.signal,
492
517
  score,
493
518
  evidence: {
494
519
  alignmentVerdicts: serializedAlignmentVerdicts,
@@ -543,35 +568,98 @@ function gEval(options) {
543
568
  const config = metricConfig(options, options.name);
544
569
  const rubric = validateRubric(options.rubric);
545
570
  const scoreRange = rubric.length === 0 ? [0, 10] : [rubric[0]?.scoreRange[0] ?? 0, rubric.at(-1)?.scoreRange[1] ?? 10];
546
- let generatedStepsPromise;
547
- let generatedUsageClaimed = false;
548
- async function resolveSteps() {
571
+ let pendingPreparation;
572
+ let preparedSteps;
573
+ function acceptSteps(result, signal) {
574
+ throwIfAborted(signal);
575
+ return { steps: [...result.steps], preparation: result };
576
+ }
577
+ async function resolveSteps(signal) {
578
+ throwIfAborted(signal);
549
579
  if (options.evaluationSteps !== void 0) {
550
- return { steps: options.evaluationSteps, usage: Usage.empty() };
580
+ return { steps: [...options.evaluationSteps] };
551
581
  }
552
- if (generatedStepsPromise === void 0) {
553
- generatedStepsPromise = runJudge({
554
- model: options.model,
555
- schema: z.object({ steps: z.array(z.string()) }),
556
- instructions: "Generate three or four concise evaluation steps from the criteria. Explain how the selected parameters should be judged in relation to one another.",
557
- prompt: jsonPrompt({ criteria: options.criteria, parameters: options.evaluationParams }),
558
- retries: config.retries
559
- }).catch((error) => {
560
- generatedStepsPromise = void 0;
561
- throw error;
562
- });
582
+ if (preparedSteps !== void 0) return acceptSteps(preparedSteps, signal);
583
+ if (pendingPreparation === void 0) {
584
+ const controller = new AbortController();
585
+ const epoch2 = {
586
+ controller,
587
+ waiters: /* @__PURE__ */ new Set(),
588
+ promise: Promise.resolve().then(
589
+ () => runJudge({
590
+ model: options.model,
591
+ schema: z.object({ steps: z.array(z.string()) }),
592
+ instructions: "Generate three or four concise evaluation steps from the criteria. Explain how the selected parameters should be judged in relation to one another.",
593
+ prompt: jsonPrompt({
594
+ criteria: options.criteria,
595
+ parameters: options.evaluationParams
596
+ }),
597
+ retries: config.retries,
598
+ abortSignal: controller.signal
599
+ })
600
+ ).then((result) => {
601
+ if (result.data.steps.length === 0) {
602
+ throw new Error("G-Eval generated no evaluation steps.");
603
+ }
604
+ const prepared = {
605
+ steps: [...result.data.steps],
606
+ usage: result.usage,
607
+ usageClaimed: false
608
+ };
609
+ if (pendingPreparation === epoch2) {
610
+ preparedSteps = prepared;
611
+ pendingPreparation = void 0;
612
+ }
613
+ return prepared;
614
+ }).catch((error) => {
615
+ if (pendingPreparation === epoch2) pendingPreparation = void 0;
616
+ throw error;
617
+ })
618
+ };
619
+ pendingPreparation = epoch2;
563
620
  }
564
- const result = await generatedStepsPromise;
565
- if (result.data.steps.length === 0) throw new Error("G-Eval generated no evaluation steps.");
566
- const usage = generatedUsageClaimed ? Usage.empty() : result.usage;
567
- generatedUsageClaimed = true;
568
- return { steps: result.data.steps, usage };
621
+ const epoch = pendingPreparation;
622
+ const token = {};
623
+ epoch.waiters.add(token);
624
+ return new Promise((resolve, reject) => {
625
+ const release = () => {
626
+ if (!epoch.waiters.delete(token)) return;
627
+ signal.removeEventListener("abort", onAbort);
628
+ if (pendingPreparation === epoch && epoch.waiters.size === 0) {
629
+ pendingPreparation = void 0;
630
+ epoch.controller.abort(signal.reason);
631
+ }
632
+ };
633
+ const onAbort = () => {
634
+ release();
635
+ reject(signal.reason);
636
+ };
637
+ signal.addEventListener("abort", onAbort, { once: true });
638
+ epoch.promise.then(
639
+ (result) => {
640
+ if (!epoch.waiters.has(token)) return;
641
+ try {
642
+ const accepted = acceptSteps(result, signal);
643
+ release();
644
+ resolve(accepted);
645
+ } catch (error) {
646
+ release();
647
+ reject(error);
648
+ }
649
+ },
650
+ (error) => {
651
+ release();
652
+ reject(error);
653
+ }
654
+ );
655
+ });
569
656
  }
570
657
  return numericMetric(config.name, config, "higher_is_better", async (args) => {
571
658
  try {
572
659
  const parameters = await resolveGEvalParameters(options, args);
573
- const stepsResult = await resolveSteps();
660
+ const stepsResult = await resolveSteps(args.signal);
574
661
  const scoreResult = await runJudge({
662
+ abortSignal: args.signal,
575
663
  model: options.model,
576
664
  schema: z.object({ score: z.number(), reason: z.string() }),
577
665
  instructions: config.strictMode ? "Apply the evaluation steps and return score 1 only for complete compliance, otherwise 0. Give a concise evidence-based reason." : `Apply the evaluation steps and return an integer score from ${scoreRange[0]} through ${scoreRange[1]}, plus a concise evidence-based reason.`,
@@ -587,8 +675,11 @@ function gEval(options) {
587
675
  throw new RangeError(`G-Eval score ${rawScore} is outside the requested range.`);
588
676
  }
589
677
  const score = config.strictMode ? rawScore : (rawScore - scoreRange[0]) / (scoreRange[1] - scoreRange[0]);
590
- const usage = addUsage(stepsResult.usage, scoreResult.usage);
591
- return higherOutcome({
678
+ throwIfAborted(args.signal);
679
+ const preparation = stepsResult.preparation;
680
+ const setupUsage = preparation === void 0 || preparation.usageClaimed ? Usage.empty() : preparation.usage;
681
+ const usage = addUsage(setupUsage, scoreResult.usage);
682
+ const outcome = higherOutcome({
592
683
  score,
593
684
  threshold: config.threshold,
594
685
  strictMode: config.strictMode,
@@ -605,6 +696,8 @@ function gEval(options) {
605
696
  },
606
697
  usage
607
698
  });
699
+ if (preparation !== void 0) preparation.usageClaimed = true;
700
+ return outcome;
608
701
  } catch (error) {
609
702
  return EvalOutcome.fromError(error);
610
703
  }
@@ -629,6 +722,7 @@ function turnRelevancy(options) {
629
722
  schema: binaryVerdictSchema,
630
723
  instructions: "Judge whether the final assistant reply is relevant to the preceding conversation. Return yes for relevant and no for irrelevant, with a concise reason.",
631
724
  prompt: jsonPrompt({ turns: window }),
725
+ abortSignal: args.signal,
632
726
  retries: config.retries
633
727
  })
634
728
  );
@@ -642,6 +736,7 @@ function turnRelevancy(options) {
642
736
  includeReason: config.includeReason,
643
737
  retries: config.retries,
644
738
  metric: "turn relevancy",
739
+ abortSignal: args.signal,
645
740
  score,
646
741
  evidence: { verdicts }
647
742
  });
@@ -677,6 +772,7 @@ function knowledgeRetention(options) {
677
772
  previousTurns: turns.slice(0, entry.index),
678
773
  userMessage: entry.turn.content
679
774
  }),
775
+ abortSignal: args.signal,
680
776
  retries: config.retries
681
777
  })
682
778
  );
@@ -696,6 +792,7 @@ function knowledgeRetention(options) {
696
792
  schema: z.object({ attrition: z.boolean(), reason: z.string() }),
697
793
  instructions: "Determine whether the assistant reply forgets, contradicts, or unnecessarily asks again for information already supplied by the user. Set attrition true only when knowledge was lost.",
698
794
  prompt: jsonPrompt({ knownFacts: entry.facts, assistantReply: entry.turn.content }),
795
+ abortSignal: args.signal,
699
796
  retries: config.retries
700
797
  })
701
798
  );
@@ -710,6 +807,7 @@ function knowledgeRetention(options) {
710
807
  includeReason: config.includeReason,
711
808
  retries: config.retries,
712
809
  metric: "knowledge retention",
810
+ abortSignal: args.signal,
713
811
  score,
714
812
  evidence: { verdicts }
715
813
  });
@@ -738,7 +836,16 @@ function numericMetric(name, config, direction, evaluate) {
738
836
  direction,
739
837
  threshold: config.strictMode === true ? direction === "higher_is_better" ? 1 : 0 : config.threshold,
740
838
  dataType: "NUMERIC",
741
- evaluate
839
+ async evaluate(args) {
840
+ try {
841
+ throwIfAborted(args.signal);
842
+ const outcome = await evaluate(args);
843
+ throwIfAborted(args.signal);
844
+ return outcome;
845
+ } catch (error) {
846
+ return EvalOutcome.fromError(error);
847
+ }
848
+ }
742
849
  };
743
850
  }
744
851
  function metricConfig(options, defaultName) {
@@ -788,12 +895,14 @@ function lowerOutcome(args) {
788
895
  return score <= threshold ? EvalOutcome.pass(score, options) : EvalOutcome.fail(score, options);
789
896
  }
790
897
  async function maybeReason(args) {
898
+ throwIfAborted(args.abortSignal);
791
899
  if (!args.includeReason) return { usage: Usage.empty() };
792
900
  const result = await runJudge({
793
901
  model: args.model,
794
902
  schema: reasonSchema,
795
903
  instructions: `Write a concise final explanation for the ${args.metric} score. Ground it only in the supplied evidence and do not repeat the numeric score.`,
796
904
  prompt: jsonPrompt({ score: args.score, evidence: args.evidence }),
905
+ abortSignal: args.abortSignal,
797
906
  retries: args.retries
798
907
  });
799
908
  return { reason: result.data.reason, usage: result.usage };
@@ -1004,4 +1113,4 @@ export {
1004
1113
  turnRelevancy,
1005
1114
  knowledgeRetention
1006
1115
  };
1007
- //# sourceMappingURL=chunk-V5DWNXVH.js.map
1116
+ //# sourceMappingURL=chunk-W2XPREWL.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/evals/advanced-metrics.ts"],"sourcesContent":["import { z } from \"zod\";\nimport type { CompletionModel, JsonObject, JsonValue, Message, Usage } from \"../completion\";\nimport { isJsonValue, Usage as UsageValue } from \"../completion\";\nimport { throwIfAborted } from \"../internal/abort\";\nimport { mapWithConcurrency } from \"../internal/concurrency\";\nimport type { ZodSchema } from \"../schema\";\nimport { errorMessage, formatValue } from \"./format\";\nimport { addUsage, evaluationMetadata, type JudgeResult, runJudge } from \"./judge\";\nimport { EvalOutcome, type EvalOutcome as EvalOutcomeType } from \"./outcome\";\nimport { resolveActualText, resolveExpected } from \"./selectors\";\nimport type {\n EvalCaseRequirements,\n EvalMetric,\n EvalMetricArgs,\n EvalTurn,\n SelectorOrValue,\n ValueSelector,\n} from \"./types\";\n\ntype Verdict = {\n verdict: \"yes\" | \"no\" | \"idk\";\n reason?: string | undefined;\n};\n\nconst statementsSchema = z.object({ statements: z.array(z.string()) });\nconst factsSchema = z.object({ facts: z.array(z.string()) });\nconst questionsSchema = z.object({ questions: z.array(z.string()) });\nconst answersSchema = z.object({ answers: z.array(z.enum([\"yes\", \"no\"])) });\nconst verdictsSchema = z.object({\n verdicts: z.array(\n z.object({\n verdict: z.enum([\"yes\", \"no\", \"idk\"]),\n reason: z.string().optional(),\n }),\n ),\n});\nconst binaryVerdictsSchema = z.object({\n verdicts: z.array(\n z.object({\n verdict: z.enum([\"yes\", \"no\"]),\n reason: z.string(),\n }),\n ),\n});\nconst binaryVerdictSchema = z.object({\n verdict: z.enum([\"yes\", \"no\"]),\n reason: z.string(),\n});\nconst reasonSchema = z.object({ reason: z.string() });\nconst abstentionJudgmentSchema = z.object({\n behavior: z.enum([\"abstention\", \"confident_answer\"]),\n grounded: z.boolean(),\n reason: z.string(),\n});\n\ntype LlmEvalOptions<Input, Output, Expected = unknown> = {\n name?: string | undefined;\n required?: boolean | undefined;\n model: CompletionModel;\n threshold?: number | undefined;\n strictMode?: boolean | undefined;\n includeReason?: boolean | undefined;\n retries?: number | undefined;\n input?: ValueSelector<Input, Output, Expected, string> | undefined;\n actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n};\n\nexport type AnswerRelevancyOptions<Input, Output, Expected = unknown> = LlmEvalOptions<\n Input,\n Output,\n Expected\n>;\n\nexport function answerRelevancy<\n Input,\n Output,\n Expected = unknown,\n const Name extends string = string,\n>(\n options: AnswerRelevancyOptions<Input, Output, Expected> & { name?: Name | undefined },\n): EvalMetric<Input, Output, number, Expected, Name> {\n const config = metricConfig<Name>(options, \"answer_relevancy\");\n return numericMetric(config.name, config, \"higher_is_better\", async (args) => {\n try {\n const input = await resolveInput(options.input, args);\n const actual = await resolveActualText(options.actual, args);\n const statementResult = await runJudge({\n model: options.model,\n schema: statementsSchema,\n instructions:\n \"Break the answer into concise, independently assessable statements. Return every substantive statement using the schema.\",\n prompt: `Answer:\\n${actual}`,\n abortSignal: args.signal,\n retries: config.retries,\n });\n const statements = statementResult.data.statements;\n let verdicts: Verdict[] = [];\n let usage = statementResult.usage;\n if (statements.length > 0) {\n const verdictResult = await runJudge({\n model: options.model,\n schema: verdictsSchema,\n instructions:\n \"Classify each answer statement for relevance to the user input. Use yes for relevant, no for irrelevant, and idk only when relevance is genuinely indeterminate. Preserve order and return one verdict per statement.\",\n prompt: jsonPrompt({ input, statements }),\n abortSignal: args.signal,\n retries: config.retries,\n });\n verdicts = verdictResult.data.verdicts;\n assertSameLength(\"answer relevancy verdicts\", statements, verdicts);\n usage = addUsage(usage, verdictResult.usage);\n }\n const score =\n verdicts.length === 0\n ? 1\n : verdicts.filter((verdict) => verdict.verdict !== \"no\").length / verdicts.length;\n const serializedVerdicts = serializeVerdicts(verdicts);\n const reasonResult = await maybeReason({\n model: options.model,\n includeReason: config.includeReason,\n retries: config.retries,\n metric: \"answer relevancy\",\n abortSignal: args.signal,\n score,\n evidence: { input, verdicts: serializedVerdicts },\n });\n usage = addUsage(usage, reasonResult.usage);\n return higherOutcome({\n score,\n threshold: config.threshold,\n strictMode: config.strictMode,\n comment: reasonResult.reason,\n details: { statements, verdicts: serializedVerdicts },\n usage,\n });\n } catch (error) {\n return EvalOutcome.fromError(error);\n }\n });\n}\n\nexport type PromptAlignmentOptions<Input, Output, Expected = unknown> = LlmEvalOptions<\n Input,\n Output,\n Expected\n> & {\n promptInstructions: string[];\n};\n\nexport function promptAlignment<\n Input,\n Output,\n Expected = unknown,\n const Name extends string = string,\n>(\n options: PromptAlignmentOptions<Input, Output, Expected> & { name?: Name | undefined },\n): EvalMetric<Input, Output, number, Expected, Name> {\n if (options.promptInstructions.length === 0) {\n throw new TypeError(\"promptAlignment requires at least one prompt instruction.\");\n }\n const config = metricConfig<Name>(options, \"prompt_alignment\");\n return numericMetric(config.name, config, \"higher_is_better\", async (args) => {\n try {\n const input = await resolveInput(options.input, args);\n const actual = await resolveActualText(options.actual, args);\n const verdictResult = await runJudge({\n model: options.model,\n schema: binaryVerdictsSchema,\n instructions:\n \"Determine whether the answer follows each prompt instruction. Preserve order and return exactly one yes or no verdict per instruction.\",\n prompt: jsonPrompt({ input, actual, instructions: options.promptInstructions }),\n abortSignal: args.signal,\n retries: config.retries,\n });\n const verdicts = verdictResult.data.verdicts;\n assertSameLength(\"prompt alignment verdicts\", options.promptInstructions, verdicts);\n const score =\n verdicts.filter((verdict) => verdict.verdict === \"yes\").length / verdicts.length;\n const reasonResult = await maybeReason({\n model: options.model,\n includeReason: config.includeReason,\n retries: config.retries,\n metric: \"prompt alignment\",\n abortSignal: args.signal,\n score,\n evidence: { verdicts },\n });\n const usage = addUsage(verdictResult.usage, reasonResult.usage);\n return higherOutcome({\n score,\n threshold: config.threshold,\n strictMode: config.strictMode,\n comment: reasonResult.reason,\n details: { promptInstructions: options.promptInstructions, verdicts },\n usage,\n });\n } catch (error) {\n return EvalOutcome.fromError(error);\n }\n });\n}\n\nexport type JsonCorrectnessOptions<Input, Output, SchemaOutput, Expected = unknown> = {\n name?: string | undefined;\n required?: boolean | undefined;\n schema: ZodSchema<SchemaOutput>;\n model?: CompletionModel | undefined;\n threshold?: number | undefined;\n strictMode?: boolean | undefined;\n includeReason?: boolean | undefined;\n retries?: number | undefined;\n actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n};\n\nexport function jsonCorrectness<\n Input,\n Output,\n SchemaOutput,\n Expected = unknown,\n const Name extends string = string,\n>(\n options: JsonCorrectnessOptions<Input, Output, SchemaOutput, Expected> & {\n name?: Name | undefined;\n },\n): EvalMetric<Input, Output, number, Expected, Name> {\n const threshold = validateThreshold(options.threshold ?? 0.5);\n const retries = validateRetries(options.retries ?? 0);\n const includeReason = options.includeReason ?? true;\n const strictMode = options.strictMode ?? true;\n return numericMetric(\n (options.name ?? \"json_correctness\") as Name,\n {\n threshold: strictMode ? 1 : threshold,\n required: options.required ?? true,\n },\n \"higher_is_better\",\n async (args) => {\n try {\n const actual = await resolveActualText(options.actual, args);\n let parsed: unknown;\n let validationError: string | undefined;\n try {\n parsed = JSON.parse(actual);\n if (!isJsonValue(parsed)) {\n throw new TypeError(\"Generated output is not a JSON value.\");\n }\n const result = options.schema.safeParse(parsed);\n if (!result.success) {\n validationError = z.prettifyError(result.error);\n }\n } catch (error) {\n validationError = errorMessage(error);\n }\n const score = validationError === undefined ? 1 : 0;\n let comment: string | undefined;\n let usage = UsageValue.empty();\n if (includeReason) {\n if (score === 1) {\n comment = \"The generated JSON is syntactically valid and matches the expected schema.\";\n } else if (options.model === undefined) {\n comment = validationError;\n } else {\n const reasonResult = await runJudge({\n model: options.model,\n schema: reasonSchema,\n instructions:\n \"Briefly explain why the generated JSON does not match the expected schema. Focus on actionable syntax, field, and type problems.\",\n prompt: jsonPrompt({ actual, validationError }),\n abortSignal: args.signal,\n retries,\n });\n comment = reasonResult.data.reason;\n usage = reasonResult.usage;\n }\n }\n return higherOutcome({\n score,\n threshold,\n strictMode,\n comment,\n details: validationError === undefined ? {} : { validationError },\n usage,\n });\n } catch (error) {\n return EvalOutcome.fromError(error);\n }\n },\n );\n}\n\nexport type HallucinationOptions<Input, Output, Expected = unknown> = LlmEvalOptions<\n Input,\n Output,\n Expected\n> & {\n context?: SelectorOrValue<Input, Output, Expected, string[]> | undefined;\n};\n\nexport function hallucination<\n Input,\n Output,\n Expected = unknown,\n const Name extends string = string,\n>(\n options: HallucinationOptions<Input, Output, Expected> & {\n context: Exclude<HallucinationOptions<Input, Output, Expected>[\"context\"], undefined>;\n name?: Name | undefined;\n },\n): EvalMetric<Input, Output, number, Expected, Name>;\nexport function hallucination<\n Input,\n Output,\n Expected = unknown,\n const Name extends string = string,\n>(\n options: Omit<HallucinationOptions<Input, Output, Expected>, \"context\"> & {\n context?: undefined;\n name?: Name | undefined;\n },\n): EvalMetric<Input, Output, number, Expected, Name, { context: string[] }>;\nexport function hallucination<\n Input,\n Output,\n Expected = unknown,\n const Name extends string = string,\n>(\n options: HallucinationOptions<Input, Output, Expected> & { name?: Name | undefined },\n): EvalMetric<Input, Output, number, Expected, Name, EvalCaseRequirements> {\n const config = metricConfig<Name>(options, \"hallucination\");\n return numericMetric(config.name, config, \"lower_is_better\", async (args) => {\n try {\n const actual = await resolveActualText(options.actual, args);\n const context = await resolveStringList(options.context, args.case.context, args, \"context\");\n const verdictResult = await runJudge({\n model: options.model,\n schema: binaryVerdictsSchema,\n instructions:\n \"Compare the answer with each trusted context. Use yes when the answer is factually aligned with that context and no when it contradicts it. Preserve order and return one verdict per context.\",\n prompt: jsonPrompt({ actual, context }),\n abortSignal: args.signal,\n retries: config.retries,\n });\n const verdicts = verdictResult.data.verdicts;\n assertSameLength(\"hallucination verdicts\", context, verdicts);\n const score = verdicts.filter((verdict) => verdict.verdict === \"no\").length / verdicts.length;\n const reasonResult = await maybeReason({\n model: options.model,\n includeReason: config.includeReason,\n retries: config.retries,\n metric: \"hallucination\",\n abortSignal: args.signal,\n score,\n evidence: { verdicts },\n });\n const usage = addUsage(verdictResult.usage, reasonResult.usage);\n return lowerOutcome({\n score,\n threshold: config.threshold,\n strictMode: config.strictMode,\n comment: reasonResult.reason,\n details: { verdicts },\n usage,\n });\n } catch (error) {\n return EvalOutcome.fromError(error);\n }\n });\n}\n\nexport type FaithfulnessOptions<Input, Output, Expected = unknown> = LlmEvalOptions<\n Input,\n Output,\n Expected\n> & {\n retrievalContext?: SelectorOrValue<Input, Output, Expected, string[]> | undefined;\n truthsExtractionLimit?: number | undefined;\n penalizeAmbiguousClaims?: boolean | undefined;\n};\n\nexport function faithfulness<Input, Output, Expected = unknown, const Name extends string = string>(\n options: FaithfulnessOptions<Input, Output, Expected> & {\n name?: Name | undefined;\n retrievalContext: Exclude<\n FaithfulnessOptions<Input, Output, Expected>[\"retrievalContext\"],\n undefined\n >;\n },\n): EvalMetric<Input, Output, number, Expected, Name>;\nexport function faithfulness<Input, Output, Expected = unknown, const Name extends string = string>(\n options: Omit<FaithfulnessOptions<Input, Output, Expected>, \"retrievalContext\"> & {\n name?: Name | undefined;\n retrievalContext?: undefined;\n },\n): EvalMetric<Input, Output, number, Expected, Name, { retrievalContext: string[] }>;\nexport function faithfulness<Input, Output, Expected = unknown, const Name extends string = string>(\n options: FaithfulnessOptions<Input, Output, Expected> & { name?: Name | undefined },\n): EvalMetric<Input, Output, number, Expected, Name, EvalCaseRequirements> {\n const config = metricConfig<Name>(options, \"faithfulness\");\n const truthsExtractionLimit = validateOptionalNonNegativeInteger(\n options.truthsExtractionLimit,\n \"truthsExtractionLimit\",\n );\n return numericMetric(config.name, config, \"higher_is_better\", async (args) => {\n try {\n const actual = await resolveActualText(options.actual, args);\n const retrievalContext = await resolveStringList(\n options.retrievalContext,\n args.case.retrievalContext,\n args,\n \"retrievalContext\",\n );\n const [truthResult, claimResult] = await Promise.all([\n runJudge({\n model: options.model,\n schema: factsSchema,\n instructions: truthsExtractionInstructions(truthsExtractionLimit),\n prompt: jsonPrompt({ retrievalContext }),\n abortSignal: args.signal,\n retries: config.retries,\n }),\n runJudge({\n model: options.model,\n schema: factsSchema,\n instructions:\n \"Extract every concise factual claim made by the answer. Return claims in the facts array and omit opinions or purely stylistic text.\",\n prompt: `Answer:\\n${actual}`,\n abortSignal: args.signal,\n retries: config.retries,\n }),\n ]);\n const truths = limitValues(truthResult.data.facts, truthsExtractionLimit);\n const claims = claimResult.data.facts;\n let verdicts: Verdict[] = [];\n let usage = addUsage(truthResult.usage, claimResult.usage);\n if (claims.length > 0) {\n const verdictResult = await runJudge({\n model: options.model,\n schema: verdictsSchema,\n instructions:\n \"Determine whether each answer claim is supported by the supplied truths. Use yes for supported, no for contradicted or unsupported, and idk for genuinely ambiguous support. Preserve order and return one verdict per claim.\",\n prompt: jsonPrompt({ truths, claims }),\n abortSignal: args.signal,\n retries: config.retries,\n });\n verdicts = verdictResult.data.verdicts;\n assertSameLength(\"faithfulness verdicts\", claims, verdicts);\n usage = addUsage(usage, verdictResult.usage);\n }\n const penalizeAmbiguousClaims = options.penalizeAmbiguousClaims ?? false;\n const supported = verdicts.filter(\n (verdict) =>\n verdict.verdict === \"yes\" || (verdict.verdict === \"idk\" && !penalizeAmbiguousClaims),\n ).length;\n const score = verdicts.length === 0 ? 1 : supported / verdicts.length;\n const serializedVerdicts = serializeVerdicts(verdicts);\n const reasonResult = await maybeReason({\n model: options.model,\n includeReason: config.includeReason,\n retries: config.retries,\n metric: \"faithfulness\",\n abortSignal: args.signal,\n score,\n evidence: { verdicts: serializedVerdicts, penalizeAmbiguousClaims },\n });\n usage = addUsage(usage, reasonResult.usage);\n return higherOutcome({\n score,\n threshold: config.threshold,\n strictMode: config.strictMode,\n comment: reasonResult.reason,\n details: { truths, claims, verdicts: serializedVerdicts, penalizeAmbiguousClaims },\n usage,\n });\n } catch (error) {\n return EvalOutcome.fromError(error);\n }\n });\n}\n\nexport type AbstentionCategory =\n | \"correct_abstention\"\n | \"unnecessary_abstention\"\n | \"unsupported_confident_answer\"\n | \"correct_grounded_answer\";\n\nexport type AbstentionOptions<Input, Output, Expected = unknown> = {\n name?: string | undefined;\n required?: boolean | undefined;\n model: CompletionModel;\n shouldAbstain: SelectorOrValue<Input, Output, Expected, boolean>;\n context?: SelectorOrValue<Input, Output, Expected, string[]> | undefined;\n actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n includeReason?: boolean | undefined;\n retries?: number | undefined;\n};\n\nexport function abstention<Input, Output, Expected = unknown, const Name extends string = string>(\n options: AbstentionOptions<Input, Output, Expected> & { name?: Name | undefined },\n): EvalMetric<Input, Output, AbstentionCategory, Expected, Name> {\n const retries = validateRetries(options.retries ?? 0);\n return {\n name: (options.name ?? \"abstention\") as Name,\n required: options.required ?? true,\n dataType: \"CATEGORICAL\",\n async evaluate(args) {\n try {\n throwIfAborted(args.signal);\n const actual = await resolveActualText(options.actual, args);\n const shouldAbstain = await resolveExpected(options.shouldAbstain, args);\n if (typeof shouldAbstain !== \"boolean\") {\n return EvalOutcome.invalid(\"abstention shouldAbstain must resolve to a boolean.\");\n }\n const context = await resolveAbstentionContext(options.context, args);\n if (!shouldAbstain && context.length === 0) {\n return EvalOutcome.invalid(\n \"abstention context must be non-empty when shouldAbstain is false.\",\n );\n }\n const judgment = await runJudge({\n abortSignal: args.signal,\n model: options.model,\n schema: abstentionJudgmentSchema,\n instructions:\n \"Classify whether the answer abstains or gives a confident answer. For a confident answer, grounded is true only when every substantive factual claim is supported by the supplied context. For an abstention, set grounded to false. Return a concise evidence-based reason.\",\n prompt: jsonPrompt({ actual, context }),\n retries,\n });\n throwIfAborted(args.signal);\n const category = abstentionCategory(\n shouldAbstain,\n judgment.data.behavior,\n judgment.data.grounded,\n );\n const outcomeOptions = {\n comment: options.includeReason === false ? undefined : judgment.data.reason,\n metadata: evaluationMetadata(\n {\n behavior: judgment.data.behavior,\n grounded: judgment.data.grounded,\n shouldAbstain,\n },\n judgment.usage,\n ),\n usage: judgment.usage,\n };\n return category === \"correct_abstention\" || category === \"correct_grounded_answer\"\n ? EvalOutcome.pass(category, outcomeOptions)\n : EvalOutcome.fail(category, outcomeOptions);\n } catch (error) {\n return EvalOutcome.fromError(error);\n }\n },\n };\n}\n\nasync function resolveAbstentionContext<Input, Output, Expected>(\n selectorOrValue: SelectorOrValue<Input, Output, Expected, string[]> | undefined,\n args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<string[]> {\n const context =\n selectorOrValue === undefined\n ? (args.case.retrievalContext ?? [])\n : typeof selectorOrValue === \"function\"\n ? await (selectorOrValue as ValueSelector<Input, Output, Expected, string[]>)(args)\n : selectorOrValue;\n if (!Array.isArray(context) || context.some((value) => typeof value !== \"string\")) {\n throw new TypeError(\"abstention context must be an array of strings.\");\n }\n return context;\n}\n\nfunction abstentionCategory(\n shouldAbstain: boolean,\n behavior: \"abstention\" | \"confident_answer\",\n grounded: boolean,\n): AbstentionCategory {\n if (behavior === \"abstention\") {\n return shouldAbstain ? \"correct_abstention\" : \"unnecessary_abstention\";\n }\n if (shouldAbstain || !grounded) return \"unsupported_confident_answer\";\n return \"correct_grounded_answer\";\n}\n\nexport type SummarizationOptions<Input, Output, Expected = unknown> = LlmEvalOptions<\n Input,\n Output,\n Expected\n> & {\n assessmentQuestions?: string[] | undefined;\n questionCount?: number | undefined;\n truthsExtractionLimit?: number | undefined;\n};\n\nexport function summarization<\n Input,\n Output,\n Expected = unknown,\n const Name extends string = string,\n>(\n options: SummarizationOptions<Input, Output, Expected> & { name?: Name | undefined },\n): EvalMetric<Input, Output, number, Expected, Name> {\n const config = metricConfig<Name>(options, \"summarization\");\n const questionCount = validatePositiveInteger(options.questionCount ?? 5, \"questionCount\");\n const truthsExtractionLimit = validateOptionalNonNegativeInteger(\n options.truthsExtractionLimit,\n \"truthsExtractionLimit\",\n );\n const suppliedQuestions =\n options.assessmentQuestions !== undefined && options.assessmentQuestions.length > 0\n ? [...options.assessmentQuestions]\n : undefined;\n return numericMetric(config.name, config, \"higher_is_better\", async (args) => {\n try {\n const input = await resolveInput(options.input, args);\n const actual = await resolveActualText(options.actual, args);\n const questionPromise: Promise<JudgeResult<{ questions: string[] }>> =\n suppliedQuestions === undefined\n ? runJudge({\n model: options.model,\n schema: questionsSchema,\n instructions: `Generate exactly ${questionCount} important yes-or-no assessment questions whose answers capture the source text's essential information.`,\n prompt: `Source text:\\n${input}`,\n abortSignal: args.signal,\n retries: config.retries,\n })\n : Promise.resolve({\n data: { questions: suppliedQuestions },\n usage: UsageValue.empty(),\n });\n const [truthResult, claimResult, questionResult] = await Promise.all([\n runJudge({\n model: options.model,\n schema: factsSchema,\n instructions: truthsExtractionInstructions(truthsExtractionLimit),\n prompt: `Source text:\\n${input}`,\n abortSignal: args.signal,\n retries: config.retries,\n }),\n runJudge({\n model: options.model,\n schema: factsSchema,\n instructions:\n \"Extract every concise factual claim made by the summary. Return claims in the facts array.\",\n prompt: `Summary:\\n${actual}`,\n abortSignal: args.signal,\n retries: config.retries,\n }),\n questionPromise,\n ]);\n const truths = limitValues(truthResult.data.facts, truthsExtractionLimit);\n const claims = claimResult.data.facts;\n const questions = questionResult.data.questions;\n if (questions.length === 0) {\n throw new Error(\"Summarization assessment questions must not be empty.\");\n }\n const sourceAnswerPromise = runJudge({\n model: options.model,\n schema: answersSchema,\n instructions:\n \"Answer each assessment question using only the supplied text. Return one yes or no answer per question in the same order.\",\n prompt: jsonPrompt({ questions, text: input }),\n abortSignal: args.signal,\n retries: config.retries,\n });\n const summaryAnswerPromise = runJudge({\n model: options.model,\n schema: answersSchema,\n instructions:\n \"Answer each assessment question using only the supplied text. Return one yes or no answer per question in the same order.\",\n prompt: jsonPrompt({ questions, text: actual }),\n abortSignal: args.signal,\n retries: config.retries,\n });\n const alignmentPromise: Promise<JudgeResult<{ verdicts: Verdict[] }>> =\n claims.length === 0\n ? Promise.resolve({ data: { verdicts: [] }, usage: UsageValue.empty() })\n : runJudge({\n model: options.model,\n schema: verdictsSchema,\n instructions:\n \"Determine whether each summary claim is supported by the source truths. Use yes for supported, no for contradicted, and idk for unsupported filler or ambiguity. Preserve order.\",\n prompt: jsonPrompt({ truths, claims }),\n abortSignal: args.signal,\n retries: config.retries,\n });\n const [sourceAnswerResult, summaryAnswerResult, alignmentResult] = await Promise.all([\n sourceAnswerPromise,\n summaryAnswerPromise,\n alignmentPromise,\n ]);\n assertSameLength(\"source assessment answers\", questions, sourceAnswerResult.data.answers);\n assertSameLength(\"summary assessment answers\", questions, summaryAnswerResult.data.answers);\n assertSameLength(\"summarization alignment verdicts\", claims, alignmentResult.data.verdicts);\n const alignmentVerdicts = alignmentResult.data.verdicts;\n const alignmentScore =\n alignmentVerdicts.length === 0\n ? 0\n : alignmentVerdicts.filter((verdict) => verdict.verdict === \"yes\").length /\n alignmentVerdicts.length;\n let coverageTotal = 0;\n let coverageMatched = 0;\n const coverageVerdicts = questions.map((question, index) => {\n const originalVerdict = sourceAnswerResult.data.answers[index] as \"yes\" | \"no\";\n const summaryVerdict = summaryAnswerResult.data.answers[index] as \"yes\" | \"no\";\n if (originalVerdict === \"yes\") {\n coverageTotal += 1;\n if (summaryVerdict === \"yes\") coverageMatched += 1;\n }\n return { question, originalVerdict, summaryVerdict };\n });\n const coverageScore = coverageTotal === 0 ? 0 : coverageMatched / coverageTotal;\n const score = Math.min(alignmentScore, coverageScore);\n const serializedAlignmentVerdicts = serializeVerdicts(alignmentVerdicts);\n const reasonResult = await maybeReason({\n model: options.model,\n includeReason: config.includeReason,\n retries: config.retries,\n metric: \"summarization\",\n abortSignal: args.signal,\n score,\n evidence: {\n alignmentVerdicts: serializedAlignmentVerdicts,\n coverageVerdicts,\n alignmentScore,\n coverageScore,\n },\n });\n const usage = addUsage(\n truthResult.usage,\n claimResult.usage,\n questionResult.usage,\n sourceAnswerResult.usage,\n summaryAnswerResult.usage,\n alignmentResult.usage,\n reasonResult.usage,\n );\n return higherOutcome({\n score,\n threshold: config.threshold,\n strictMode: config.strictMode,\n comment: reasonResult.reason,\n details: {\n truths,\n claims,\n assessmentQuestions: questions,\n alignmentVerdicts: serializedAlignmentVerdicts,\n coverageVerdicts,\n scoreBreakdown: { alignment: alignmentScore, coverage: coverageScore },\n },\n usage,\n });\n } catch (error) {\n return EvalOutcome.fromError(error);\n }\n });\n}\n\nexport type GEvalParameter =\n | \"input\"\n | \"actualOutput\"\n | \"expectedOutput\"\n | \"context\"\n | \"retrievalContext\"\n | \"metadata\";\n\nexport type GEvalRubric = {\n scoreRange: readonly [number, number];\n expectedOutcome: string;\n};\n\nexport type GEvalOptions<Input, Output, Expected = unknown> = Omit<\n LlmEvalOptions<Input, Output, Expected>,\n \"name\"\n> & {\n name: string;\n evaluationParams: GEvalParameter[];\n criteria?: string | undefined;\n evaluationSteps?: string[] | undefined;\n rubric?: GEvalRubric[] | undefined;\n expected?: ValueSelector<Input, Output, Expected, unknown> | undefined;\n context?: SelectorOrValue<Input, Output, Expected, string[]> | undefined;\n retrievalContext?: SelectorOrValue<Input, Output, Expected, string[]> | undefined;\n};\n\ntype PreparedSteps = {\n steps: string[];\n usage: Usage;\n usageClaimed: boolean;\n};\n\ntype PreparationEpoch = {\n controller: AbortController;\n promise: Promise<PreparedSteps>;\n waiters: Set<object>;\n};\n\ntype GEvalCaseRequirements<\n Params extends readonly GEvalParameter[],\n ExpectedSelector,\n ContextSelector,\n RetrievalContextSelector,\n> = (\"expectedOutput\" extends Params[number]\n ? [ExpectedSelector] extends [undefined]\n ? { expected: unknown }\n : Record<never, never>\n : Record<never, never>) &\n (\"context\" extends Params[number]\n ? [ContextSelector] extends [undefined]\n ? { context: string[] }\n : Record<never, never>\n : Record<never, never>) &\n (\"retrievalContext\" extends Params[number]\n ? [RetrievalContextSelector] extends [undefined]\n ? { retrievalContext: string[] }\n : Record<never, never>\n : Record<never, never>);\n\nexport function gEval<\n Input,\n Output,\n Expected = unknown,\n const Name extends string = string,\n const Params extends readonly GEvalParameter[] = readonly GEvalParameter[],\n ExpectedSelector extends ValueSelector<Input, Output, Expected, unknown> | undefined = undefined,\n ContextSelector extends SelectorOrValue<Input, Output, Expected, string[]> | undefined =\n undefined,\n RetrievalContextSelector extends SelectorOrValue<Input, Output, Expected, string[]> | undefined =\n undefined,\n>(\n options: Omit<\n GEvalOptions<Input, Output, Expected>,\n \"evaluationParams\" | \"expected\" | \"context\" | \"retrievalContext\"\n > & {\n name: Name;\n evaluationParams: Params;\n expected?: ExpectedSelector;\n context?: ContextSelector;\n retrievalContext?: RetrievalContextSelector;\n },\n): EvalMetric<\n Input,\n Output,\n number,\n Expected,\n Name,\n GEvalCaseRequirements<Params, ExpectedSelector, ContextSelector, RetrievalContextSelector>\n>;\nexport function gEval<Input, Output, Expected = unknown, const Name extends string = string>(\n options: GEvalOptions<Input, Output, Expected> & { name: Name },\n): EvalMetric<Input, Output, number, Expected, Name> {\n if (options.name.trim().length === 0) throw new TypeError(\"gEval name must not be empty.\");\n if (options.evaluationParams.length === 0) {\n throw new TypeError(\"gEval requires at least one evaluation parameter.\");\n }\n if ((options.criteria === undefined) === (options.evaluationSteps === undefined)) {\n throw new TypeError(\"gEval requires exactly one of criteria or evaluationSteps.\");\n }\n if (options.criteria !== undefined && options.criteria.trim().length === 0) {\n throw new TypeError(\"gEval criteria must not be empty.\");\n }\n if (options.evaluationSteps !== undefined && options.evaluationSteps.length === 0) {\n throw new TypeError(\"gEval evaluationSteps must not be empty.\");\n }\n const config = metricConfig<Name>(options, options.name);\n const rubric = validateRubric(options.rubric);\n const scoreRange =\n rubric.length === 0\n ? ([0, 10] as const)\n : ([rubric[0]?.scoreRange[0] ?? 0, rubric.at(-1)?.scoreRange[1] ?? 10] as const);\n let pendingPreparation: PreparationEpoch | undefined;\n let preparedSteps: PreparedSteps | undefined;\n\n function acceptSteps(result: PreparedSteps, signal: AbortSignal) {\n throwIfAborted(signal);\n return { steps: [...result.steps], preparation: result };\n }\n\n async function resolveSteps(\n signal: AbortSignal,\n ): Promise<{ steps: string[]; preparation?: PreparedSteps }> {\n throwIfAborted(signal);\n if (options.evaluationSteps !== undefined) {\n return { steps: [...options.evaluationSteps] };\n }\n if (preparedSteps !== undefined) return acceptSteps(preparedSteps, signal);\n if (pendingPreparation === undefined) {\n const controller = new AbortController();\n const epoch: PreparationEpoch = {\n controller,\n waiters: new Set(),\n promise: Promise.resolve()\n .then(() =>\n runJudge({\n model: options.model,\n schema: z.object({ steps: z.array(z.string()) }),\n instructions:\n \"Generate three or four concise evaluation steps from the criteria. Explain how the selected parameters should be judged in relation to one another.\",\n prompt: jsonPrompt({\n criteria: options.criteria,\n parameters: options.evaluationParams,\n }),\n retries: config.retries,\n abortSignal: controller.signal,\n }),\n )\n .then((result) => {\n if (result.data.steps.length === 0) {\n throw new Error(\"G-Eval generated no evaluation steps.\");\n }\n const prepared = {\n steps: [...result.data.steps],\n usage: result.usage,\n usageClaimed: false,\n };\n if (pendingPreparation === epoch) {\n preparedSteps = prepared;\n pendingPreparation = undefined;\n }\n return prepared;\n })\n .catch((error: unknown) => {\n if (pendingPreparation === epoch) pendingPreparation = undefined;\n throw error;\n }),\n };\n pendingPreparation = epoch;\n }\n const epoch = pendingPreparation;\n const token = {};\n epoch.waiters.add(token);\n return new Promise((resolve, reject) => {\n const release = () => {\n if (!epoch.waiters.delete(token)) return;\n signal.removeEventListener(\"abort\", onAbort);\n if (pendingPreparation === epoch && epoch.waiters.size === 0) {\n pendingPreparation = undefined;\n epoch.controller.abort(signal.reason);\n }\n };\n const onAbort = () => {\n release();\n reject(signal.reason);\n };\n signal.addEventListener(\"abort\", onAbort, { once: true });\n epoch.promise.then(\n (result) => {\n if (!epoch.waiters.has(token)) return;\n try {\n const accepted = acceptSteps(result, signal);\n release();\n resolve(accepted);\n } catch (error) {\n release();\n reject(error);\n }\n },\n (error: unknown) => {\n release();\n reject(error);\n },\n );\n });\n }\n\n return numericMetric(config.name, config, \"higher_is_better\", async (args) => {\n try {\n const parameters = await resolveGEvalParameters(options, args);\n const stepsResult = await resolveSteps(args.signal);\n const scoreResult = await runJudge({\n abortSignal: args.signal,\n model: options.model,\n schema: z.object({ score: z.number(), reason: z.string() }),\n instructions: config.strictMode\n ? \"Apply the evaluation steps and return score 1 only for complete compliance, otherwise 0. Give a concise evidence-based reason.\"\n : `Apply the evaluation steps and return an integer score from ${scoreRange[0]} through ${scoreRange[1]}, plus a concise evidence-based reason.`,\n prompt: jsonPrompt({\n evaluationSteps: stepsResult.steps,\n rubric,\n parameters,\n }),\n retries: config.retries,\n });\n const rawScore = scoreResult.data.score;\n if (\n !Number.isFinite(rawScore) ||\n (config.strictMode && rawScore !== 0 && rawScore !== 1) ||\n (!config.strictMode && (rawScore < scoreRange[0] || rawScore > scoreRange[1]))\n ) {\n throw new RangeError(`G-Eval score ${rawScore} is outside the requested range.`);\n }\n const score = config.strictMode\n ? rawScore\n : (rawScore - scoreRange[0]) / (scoreRange[1] - scoreRange[0]);\n throwIfAborted(args.signal);\n const preparation = stepsResult.preparation;\n const setupUsage =\n preparation === undefined || preparation.usageClaimed\n ? UsageValue.empty()\n : preparation.usage;\n const usage = addUsage(setupUsage, scoreResult.usage);\n const outcome = higherOutcome({\n score,\n threshold: config.threshold,\n strictMode: config.strictMode,\n comment: config.includeReason ? scoreResult.data.reason : undefined,\n details: {\n evaluationSteps: stepsResult.steps,\n evaluationParams: options.evaluationParams,\n rawScore,\n scoreRange: [scoreRange[0], scoreRange[1]],\n rubric: rubric.map((entry) => ({\n scoreRange: [entry.scoreRange[0], entry.scoreRange[1]],\n expectedOutcome: entry.expectedOutcome,\n })),\n },\n usage,\n });\n if (preparation !== undefined) preparation.usageClaimed = true;\n return outcome;\n } catch (error) {\n return EvalOutcome.fromError(error);\n }\n });\n}\n\ntype ConversationEvalOptions<Input, Output, Expected = unknown> = {\n name?: string | undefined;\n required?: boolean | undefined;\n model: CompletionModel;\n threshold?: number | undefined;\n strictMode?: boolean | undefined;\n includeReason?: boolean | undefined;\n retries?: number | undefined;\n concurrency?: number | undefined;\n turns?: ValueSelector<Input, Output, Expected, ConversationSource> | undefined;\n};\n\nexport type TurnRelevancyOptions<Input, Output, Expected = unknown> = ConversationEvalOptions<\n Input,\n Output,\n Expected\n> & {\n windowSize?: number | undefined;\n};\n\nexport function turnRelevancy<\n Input,\n Output,\n Expected = unknown,\n const Name extends string = string,\n>(\n options: TurnRelevancyOptions<Input, Output, Expected> & { name?: Name | undefined },\n): EvalMetric<Input, Output, number, Expected, Name> {\n const config = metricConfig<Name>(options, \"turn_relevancy\");\n const windowSize = validatePositiveInteger(options.windowSize ?? 10, \"windowSize\");\n const concurrency = validatePositiveInteger(options.concurrency ?? 4, \"concurrency\");\n return numericMetric(config.name, config, \"higher_is_better\", async (args) => {\n try {\n const turns = await resolveTurns(options.turns, args);\n const interactions = unitInteractions(turns);\n const windows = interactions.map((_, index) =>\n interactions.slice(Math.max(0, index - windowSize + 1), index + 1).flat(),\n );\n const verdictResults = await mapWithConcurrency(windows, concurrency, (window) =>\n runJudge({\n model: options.model,\n schema: binaryVerdictSchema,\n instructions:\n \"Judge whether the final assistant reply is relevant to the preceding conversation. Return yes for relevant and no for irrelevant, with a concise reason.\",\n prompt: jsonPrompt({ turns: window }),\n abortSignal: args.signal,\n retries: config.retries,\n }),\n );\n const verdicts = verdictResults.map((result, index) => ({\n interaction: index + 1,\n ...result.data,\n }));\n const score =\n verdicts.length === 0\n ? 1\n : verdicts.filter((verdict) => verdict.verdict === \"yes\").length / verdicts.length;\n const reasonResult = await maybeReason({\n model: options.model,\n includeReason: config.includeReason,\n retries: config.retries,\n metric: \"turn relevancy\",\n abortSignal: args.signal,\n score,\n evidence: { verdicts },\n });\n const usage = addUsage(...verdictResults.map((result) => result.usage), reasonResult.usage);\n return higherOutcome({\n score,\n threshold: config.threshold,\n strictMode: config.strictMode,\n comment: reasonResult.reason,\n details: { windowSize, concurrency, interactionCount: interactions.length, verdicts },\n usage,\n });\n } catch (error) {\n return EvalOutcome.fromError(error);\n }\n });\n}\n\nexport type KnowledgeRetentionOptions<Input, Output, Expected = unknown> = ConversationEvalOptions<\n Input,\n Output,\n Expected\n>;\n\nexport function knowledgeRetention<\n Input,\n Output,\n Expected = unknown,\n const Name extends string = string,\n>(\n options: KnowledgeRetentionOptions<Input, Output, Expected> & { name?: Name | undefined },\n): EvalMetric<Input, Output, number, Expected, Name> {\n const config = metricConfig<Name>(options, \"knowledge_retention\");\n const concurrency = validatePositiveInteger(options.concurrency ?? 4, \"concurrency\");\n return numericMetric(config.name, config, \"higher_is_better\", async (args) => {\n try {\n const turns = await resolveTurns(options.turns, args);\n const userTurns = turns\n .map((turn, index) => ({ turn, index }))\n .filter((entry) => entry.turn.role === \"user\");\n const knowledgeResults = await mapWithConcurrency(userTurns, concurrency, (entry) =>\n runJudge({\n model: options.model,\n schema: factsSchema,\n instructions:\n \"Extract durable factual information newly supplied by the final user message. Use prior turns only to resolve references. Return concise facts; return an empty array when nothing new was supplied.\",\n prompt: jsonPrompt({\n previousTurns: turns.slice(0, entry.index),\n userMessage: entry.turn.content,\n }),\n abortSignal: args.signal,\n retries: config.retries,\n }),\n );\n const knowledge = userTurns.map((entry, index) => ({\n turnIndex: entry.index,\n facts: knowledgeResults[index]?.data.facts ?? [],\n }));\n const assistantChecks = turns\n .map((turn, index) => ({ turn, index }))\n .filter((entry) => entry.turn.role === \"assistant\")\n .map((entry) => ({\n ...entry,\n facts: knowledge\n .filter((item) => item.turnIndex < entry.index)\n .flatMap((item) => item.facts),\n }))\n .filter((entry) => entry.facts.length > 0);\n const verdictResults = await mapWithConcurrency(assistantChecks, concurrency, (entry) =>\n runJudge({\n model: options.model,\n schema: z.object({ attrition: z.boolean(), reason: z.string() }),\n instructions:\n \"Determine whether the assistant reply forgets, contradicts, or unnecessarily asks again for information already supplied by the user. Set attrition true only when knowledge was lost.\",\n prompt: jsonPrompt({ knownFacts: entry.facts, assistantReply: entry.turn.content }),\n abortSignal: args.signal,\n retries: config.retries,\n }),\n );\n const verdicts = assistantChecks.map((entry, index) => ({\n turnIndex: entry.index,\n attrition: verdictResults[index]?.data.attrition ?? true,\n reason: verdictResults[index]?.data.reason ?? \"Missing knowledge-retention verdict.\",\n }));\n const score =\n verdicts.length === 0\n ? 1\n : verdicts.filter((verdict) => !verdict.attrition).length / verdicts.length;\n const reasonResult = await maybeReason({\n model: options.model,\n includeReason: config.includeReason,\n retries: config.retries,\n metric: \"knowledge retention\",\n abortSignal: args.signal,\n score,\n evidence: { verdicts },\n });\n const usage = addUsage(\n ...knowledgeResults.map((result) => result.usage),\n ...verdictResults.map((result) => result.usage),\n reasonResult.usage,\n );\n return higherOutcome({\n score,\n threshold: config.threshold,\n strictMode: config.strictMode,\n comment: reasonResult.reason,\n details: { concurrency, knowledge, verdicts },\n usage,\n });\n } catch (error) {\n return EvalOutcome.fromError(error);\n }\n });\n}\n\nfunction numericMetric<Input, Output, Expected, const Name extends string>(\n name: Name,\n config: { threshold: number; required: boolean; strictMode?: boolean | undefined },\n direction: \"higher_is_better\" | \"lower_is_better\",\n evaluate: (args: EvalMetricArgs<Input, Output, Expected>) => Promise<EvalOutcomeType<number>>,\n): EvalMetric<Input, Output, number, Expected, Name> {\n return {\n name,\n required: config.required,\n direction,\n threshold:\n config.strictMode === true ? (direction === \"higher_is_better\" ? 1 : 0) : config.threshold,\n dataType: \"NUMERIC\",\n async evaluate(args) {\n try {\n throwIfAborted(args.signal);\n const outcome = await evaluate(args);\n throwIfAborted(args.signal);\n return outcome;\n } catch (error) {\n return EvalOutcome.fromError(error);\n }\n },\n };\n}\n\ntype MetricConfig<Name extends string = string> = {\n name: Name;\n threshold: number;\n strictMode: boolean;\n includeReason: boolean;\n retries: number;\n required: boolean;\n};\n\nfunction metricConfig<const Name extends string>(\n options: {\n name?: string | undefined;\n threshold?: number | undefined;\n strictMode?: boolean | undefined;\n includeReason?: boolean | undefined;\n retries?: number | undefined;\n required?: boolean | undefined;\n },\n defaultName: string,\n): MetricConfig<Name> {\n return {\n name: (options.name ?? defaultName) as Name,\n threshold: validateThreshold(options.threshold ?? 0.5),\n strictMode: options.strictMode ?? false,\n includeReason: options.includeReason ?? true,\n retries: validateRetries(options.retries ?? 0),\n required: options.required ?? true,\n };\n}\n\nfunction higherOutcome(args: {\n score: number;\n threshold: number;\n strictMode: boolean;\n comment?: string | undefined;\n details: JsonObject;\n usage: Usage;\n}) {\n const score = args.strictMode ? (args.score === 1 ? 1 : 0) : args.score;\n const threshold = args.strictMode ? 1 : args.threshold;\n const options = {\n comment: args.comment,\n usage: args.usage,\n metadata: evaluationMetadata(\n {\n ...args.details,\n scoreDirection: \"higher_is_better\",\n threshold,\n strictMode: args.strictMode,\n },\n args.usage,\n ),\n };\n return score >= threshold ? EvalOutcome.pass(score, options) : EvalOutcome.fail(score, options);\n}\n\nfunction lowerOutcome(args: {\n score: number;\n threshold: number;\n strictMode: boolean;\n comment?: string | undefined;\n details: JsonObject;\n usage: Usage;\n}) {\n const score = args.strictMode ? (args.score === 0 ? 0 : 1) : args.score;\n const threshold = args.strictMode ? 0 : args.threshold;\n const options = {\n comment: args.comment,\n usage: args.usage,\n metadata: evaluationMetadata(\n {\n ...args.details,\n scoreDirection: \"lower_is_better\",\n threshold,\n strictMode: args.strictMode,\n },\n args.usage,\n ),\n };\n return score <= threshold ? EvalOutcome.pass(score, options) : EvalOutcome.fail(score, options);\n}\n\nasync function maybeReason(args: {\n model: CompletionModel;\n includeReason: boolean;\n retries: number;\n metric: string;\n score: number;\n evidence: JsonObject;\n abortSignal: AbortSignal;\n}): Promise<{ reason?: string | undefined; usage: Usage }> {\n throwIfAborted(args.abortSignal);\n if (!args.includeReason) return { usage: UsageValue.empty() };\n const result = await runJudge({\n model: args.model,\n schema: reasonSchema,\n instructions: `Write a concise final explanation for the ${args.metric} score. Ground it only in the supplied evidence and do not repeat the numeric score.`,\n prompt: jsonPrompt({ score: args.score, evidence: args.evidence }),\n abortSignal: args.abortSignal,\n retries: args.retries,\n });\n return { reason: result.data.reason, usage: result.usage };\n}\n\nasync function resolveInput<Input, Output, Expected>(\n selector: ValueSelector<Input, Output, Expected, string> | undefined,\n args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<string> {\n return selector === undefined ? formatValue(args.case.input) : selector(args);\n}\n\nasync function resolveStringList<Input, Output, Expected>(\n selectorOrValue: SelectorOrValue<Input, Output, Expected, string[]> | undefined,\n fallback: string[] | undefined,\n args: EvalMetricArgs<Input, Output, Expected>,\n label: string,\n): Promise<string[]> {\n const value =\n selectorOrValue === undefined\n ? fallback\n : typeof selectorOrValue === \"function\"\n ? await (selectorOrValue as ValueSelector<Input, Output, Expected, string[]>)(args)\n : selectorOrValue;\n if (\n !Array.isArray(value) ||\n value.length === 0 ||\n value.some((item) => typeof item !== \"string\")\n ) {\n throw new TypeError(`${label} must be a non-empty array of strings.`);\n }\n return value;\n}\n\nasync function resolveGEvalParameters<Input, Output, Expected>(\n options: GEvalOptions<Input, Output, Expected>,\n args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<JsonObject> {\n const parameters: JsonObject = {};\n for (const parameter of options.evaluationParams) {\n if (parameter === \"input\") parameters.input = await resolveInput(options.input, args);\n if (parameter === \"actualOutput\") {\n parameters.actualOutput = await resolveActualText(options.actual, args);\n }\n if (parameter === \"expectedOutput\") {\n const expected =\n options.expected === undefined ? args.case.expected : await options.expected(args);\n if (expected === undefined) throw new Error(\"G-Eval expectedOutput is missing.\");\n parameters.expectedOutput = toJsonValue(expected);\n }\n if (parameter === \"context\") {\n parameters.context = await resolveStringList(\n options.context,\n args.case.context,\n args,\n \"context\",\n );\n }\n if (parameter === \"retrievalContext\") {\n parameters.retrievalContext = await resolveStringList(\n options.retrievalContext,\n args.case.retrievalContext,\n args,\n \"retrievalContext\",\n );\n }\n if (parameter === \"metadata\") parameters.metadata = args.case.metadata ?? {};\n }\n return parameters;\n}\n\nfunction validateRubric(rubric: GEvalRubric[] | undefined): GEvalRubric[] {\n if (rubric === undefined || rubric.length === 0) return [];\n const sorted = [...rubric].sort((left, right) => left.scoreRange[0] - right.scoreRange[0]);\n for (const [index, entry] of sorted.entries()) {\n const [start, end] = entry.scoreRange;\n if (\n !Number.isInteger(start) ||\n !Number.isInteger(end) ||\n start < 0 ||\n end > 10 ||\n start > end\n ) {\n throw new RangeError(\n \"G-Eval rubric score ranges must be ordered integers from 0 through 10.\",\n );\n }\n if (entry.expectedOutcome.trim().length === 0) {\n throw new TypeError(\"G-Eval rubric expectedOutcome must not be empty.\");\n }\n const next = sorted[index + 1];\n if (next !== undefined && end >= next.scoreRange[0]) {\n throw new RangeError(\"G-Eval rubric score ranges must not overlap.\");\n }\n }\n const first = sorted[0];\n const last = sorted.at(-1);\n if (first !== undefined && last !== undefined && first.scoreRange[0] === last.scoreRange[1]) {\n throw new RangeError(\"G-Eval rubric score range must span more than one value.\");\n }\n return sorted;\n}\n\nfunction truthsExtractionInstructions(limit: number | undefined): string {\n return limit === undefined\n ? \"Extract concise, factual, undisputed truths from the supplied source material, ordered by importance. Return them in the facts array.\"\n : `Extract at most ${limit} concise, factual, undisputed truths from the supplied source material, ordered by importance. Return them in the facts array.`;\n}\n\nfunction limitValues(values: string[], limit: number | undefined): string[] {\n return limit === undefined ? values : values.slice(0, limit);\n}\n\nfunction validateThreshold(value: number): number {\n if (!Number.isFinite(value) || value < 0 || value > 1) {\n throw new RangeError(\"Eval metric threshold must be between 0 and 1.\");\n }\n return value;\n}\n\nfunction validateRetries(value: number): number {\n if (!Number.isInteger(value) || value < 0) {\n throw new RangeError(\"Eval metric retries must be a non-negative integer.\");\n }\n return value;\n}\n\nfunction validatePositiveInteger(value: number, label: string): number {\n if (!Number.isInteger(value) || value < 1) {\n throw new RangeError(`${label} must be a positive integer.`);\n }\n return value;\n}\n\nfunction validateOptionalNonNegativeInteger(\n value: number | undefined,\n label: string,\n): number | undefined {\n if (value === undefined) return undefined;\n if (!Number.isInteger(value) || value < 0) {\n throw new RangeError(`${label} must be a non-negative integer.`);\n }\n return value;\n}\n\nfunction assertSameLength(label: string, inputs: unknown[], outputs: unknown[]): void {\n if (inputs.length !== outputs.length) {\n throw new Error(`${label} count ${outputs.length} did not match input count ${inputs.length}.`);\n }\n}\n\nfunction serializeVerdicts(verdicts: readonly Verdict[]): JsonObject[] {\n return verdicts.map((verdict) => {\n const serialized: JsonObject = { verdict: verdict.verdict };\n if (verdict.reason !== undefined) serialized.reason = verdict.reason;\n return serialized;\n });\n}\n\nfunction jsonPrompt(value: unknown): string {\n return JSON.stringify(value, null, 2);\n}\n\nfunction toJsonValue(value: unknown): JsonValue {\n if (!isJsonValue(value)) {\n throw new TypeError(\"G-Eval expectedOutput must be a JSON value.\");\n }\n return cloneJsonValue(value);\n}\n\nfunction cloneJsonValue(value: JsonValue): JsonValue {\n if (value === null || typeof value !== \"object\") return value;\n if (Array.isArray(value)) return value.map(cloneJsonValue);\n return Object.fromEntries(\n Object.entries(value).map(([key, item]) => {\n if (item === undefined) {\n throw new TypeError(\"G-Eval expectedOutput must be a JSON value.\");\n }\n return [key, cloneJsonValue(item)];\n }),\n );\n}\n\ntype ConversationSource = EvalTurn[] | Message[];\n\nfunction normalizeEvalTurns(value: unknown): EvalTurn[] | undefined {\n const source = conversationArray(value);\n if (source === undefined) return undefined;\n const turns: EvalTurn[] = [];\n for (const entry of source) {\n if (typeof entry !== \"object\" || entry === null) continue;\n const role = (entry as { role?: unknown }).role;\n if (role !== \"user\" && role !== \"assistant\") continue;\n const content = (entry as { content?: unknown }).content;\n const text = contentText(content);\n if (text.length === 0) continue;\n const metadata = (entry as { metadata?: unknown }).metadata;\n turns.push(\n typeof metadata === \"object\" && metadata !== null && !Array.isArray(metadata)\n ? { role, content: text, metadata: metadata as EvalTurn[\"metadata\"] }\n : { role, content: text },\n );\n }\n return turns.length === 0 ? undefined : turns;\n}\n\nasync function resolveTurns<Input, Output, Expected>(\n selector: ValueSelector<Input, Output, Expected, ConversationSource> | undefined,\n args: EvalMetricArgs<Input, Output, Expected>,\n): Promise<EvalTurn[]> {\n const source = selector === undefined ? args.output : await selector(args);\n const turns = normalizeEvalTurns(source);\n if (turns === undefined) {\n throw new TypeError(\n \"Conversational eval requires non-empty EvalTurn[], Message[], or an output with messages.\",\n );\n }\n return turns;\n}\n\nfunction unitInteractions(turns: EvalTurn[]): EvalTurn[][] {\n const interactions: EvalTurn[][] = [];\n let current: EvalTurn[] = [];\n let hasUser = false;\n for (const turn of turns) {\n if (current.at(-1)?.role === \"assistant\" && turn.role === \"user\" && hasUser) {\n interactions.push(current);\n current = [turn];\n hasUser = true;\n continue;\n }\n current.push(turn);\n if (turn.role === \"user\") hasUser = true;\n }\n if (current.length > 1 && current.at(-1)?.role === \"assistant\" && hasUser) {\n interactions.push(current);\n }\n return interactions;\n}\n\nfunction conversationArray(value: unknown): unknown[] | undefined {\n if (Array.isArray(value)) return value;\n if (typeof value === \"object\" && value !== null && \"messages\" in value) {\n const messages = (value as { messages?: unknown }).messages;\n return Array.isArray(messages) ? messages : undefined;\n }\n return undefined;\n}\n\nfunction contentText(content: unknown): string {\n if (typeof content === \"string\") return content;\n if (!Array.isArray(content)) return \"\";\n return content\n .flatMap((part) =>\n typeof part === \"object\" &&\n part !== null &&\n \"type\" in part &&\n part.type === \"text\" &&\n \"text\" in part &&\n typeof part.text === \"string\"\n ? [part.text]\n : [],\n )\n .join(\"\\n\");\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,SAAS;AAwBlB,IAAM,mBAAmB,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;AACrE,IAAM,cAAc,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;AAC3D,IAAM,kBAAkB,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;AACnE,IAAM,gBAAgB,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,CAAC;AAC1E,IAAM,iBAAiB,EAAE,OAAO;AAAA,EAC9B,UAAU,EAAE;AAAA,IACV,EAAE,OAAO;AAAA,MACP,SAAS,EAAE,KAAK,CAAC,OAAO,MAAM,KAAK,CAAC;AAAA,MACpC,QAAQ,EAAE,OAAO,EAAE,SAAS;AAAA,IAC9B,CAAC;AAAA,EACH;AACF,CAAC;AACD,IAAM,uBAAuB,EAAE,OAAO;AAAA,EACpC,UAAU,EAAE;AAAA,IACV,EAAE,OAAO;AAAA,MACP,SAAS,EAAE,KAAK,CAAC,OAAO,IAAI,CAAC;AAAA,MAC7B,QAAQ,EAAE,OAAO;AAAA,IACnB,CAAC;AAAA,EACH;AACF,CAAC;AACD,IAAM,sBAAsB,EAAE,OAAO;AAAA,EACnC,SAAS,EAAE,KAAK,CAAC,OAAO,IAAI,CAAC;AAAA,EAC7B,QAAQ,EAAE,OAAO;AACnB,CAAC;AACD,IAAM,eAAe,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;AACpD,IAAM,2BAA2B,EAAE,OAAO;AAAA,EACxC,UAAU,EAAE,KAAK,CAAC,cAAc,kBAAkB,CAAC;AAAA,EACnD,UAAU,EAAE,QAAQ;AAAA,EACpB,QAAQ,EAAE,OAAO;AACnB,CAAC;AAoBM,SAAS,gBAMd,SACmD;AACnD,QAAM,SAAS,aAAmB,SAAS,kBAAkB;AAC7D,SAAO,cAAc,OAAO,MAAM,QAAQ,oBAAoB,OAAO,SAAS;AAC5E,QAAI;AACF,YAAM,QAAQ,MAAM,aAAa,QAAQ,OAAO,IAAI;AACpD,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,kBAAkB,MAAM,SAAS;AAAA,QACrC,OAAO,QAAQ;AAAA,QACf,QAAQ;AAAA,QACR,cACE;AAAA,QACF,QAAQ;AAAA,EAAY,MAAM;AAAA,QAC1B,aAAa,KAAK;AAAA,QAClB,SAAS,OAAO;AAAA,MAClB,CAAC;AACD,YAAM,aAAa,gBAAgB,KAAK;AACxC,UAAI,WAAsB,CAAC;AAC3B,UAAI,QAAQ,gBAAgB;AAC5B,UAAI,WAAW,SAAS,GAAG;AACzB,cAAM,gBAAgB,MAAM,SAAS;AAAA,UACnC,OAAO,QAAQ;AAAA,UACf,QAAQ;AAAA,UACR,cACE;AAAA,UACF,QAAQ,WAAW,EAAE,OAAO,WAAW,CAAC;AAAA,UACxC,aAAa,KAAK;AAAA,UAClB,SAAS,OAAO;AAAA,QAClB,CAAC;AACD,mBAAW,cAAc,KAAK;AAC9B,yBAAiB,6BAA6B,YAAY,QAAQ;AAClE,gBAAQ,SAAS,OAAO,cAAc,KAAK;AAAA,MAC7C;AACA,YAAM,QACJ,SAAS,WAAW,IAChB,IACA,SAAS,OAAO,CAAC,YAAY,QAAQ,YAAY,IAAI,EAAE,SAAS,SAAS;AAC/E,YAAM,qBAAqB,kBAAkB,QAAQ;AACrD,YAAM,eAAe,MAAM,YAAY;AAAA,QACrC,OAAO,QAAQ;AAAA,QACf,eAAe,OAAO;AAAA,QACtB,SAAS,OAAO;AAAA,QAChB,QAAQ;AAAA,QACR,aAAa,KAAK;AAAA,QAClB;AAAA,QACA,UAAU,EAAE,OAAO,UAAU,mBAAmB;AAAA,MAClD,CAAC;AACD,cAAQ,SAAS,OAAO,aAAa,KAAK;AAC1C,aAAO,cAAc;AAAA,QACnB;AAAA,QACA,WAAW,OAAO;AAAA,QAClB,YAAY,OAAO;AAAA,QACnB,SAAS,aAAa;AAAA,QACtB,SAAS,EAAE,YAAY,UAAU,mBAAmB;AAAA,QACpD;AAAA,MACF,CAAC;AAAA,IACH,SAAS,OAAO;AACd,aAAO,YAAY,UAAU,KAAK;AAAA,IACpC;AAAA,EACF,CAAC;AACH;AAUO,SAAS,gBAMd,SACmD;AACnD,MAAI,QAAQ,mBAAmB,WAAW,GAAG;AAC3C,UAAM,IAAI,UAAU,2DAA2D;AAAA,EACjF;AACA,QAAM,SAAS,aAAmB,SAAS,kBAAkB;AAC7D,SAAO,cAAc,OAAO,MAAM,QAAQ,oBAAoB,OAAO,SAAS;AAC5E,QAAI;AACF,YAAM,QAAQ,MAAM,aAAa,QAAQ,OAAO,IAAI;AACpD,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,gBAAgB,MAAM,SAAS;AAAA,QACnC,OAAO,QAAQ;AAAA,QACf,QAAQ;AAAA,QACR,cACE;AAAA,QACF,QAAQ,WAAW,EAAE,OAAO,QAAQ,cAAc,QAAQ,mBAAmB,CAAC;AAAA,QAC9E,aAAa,KAAK;AAAA,QAClB,SAAS,OAAO;AAAA,MAClB,CAAC;AACD,YAAM,WAAW,cAAc,KAAK;AACpC,uBAAiB,6BAA6B,QAAQ,oBAAoB,QAAQ;AAClF,YAAM,QACJ,SAAS,OAAO,CAAC,YAAY,QAAQ,YAAY,KAAK,EAAE,SAAS,SAAS;AAC5E,YAAM,eAAe,MAAM,YAAY;AAAA,QACrC,OAAO,QAAQ;AAAA,QACf,eAAe,OAAO;AAAA,QACtB,SAAS,OAAO;AAAA,QAChB,QAAQ;AAAA,QACR,aAAa,KAAK;AAAA,QAClB;AAAA,QACA,UAAU,EAAE,SAAS;AAAA,MACvB,CAAC;AACD,YAAM,QAAQ,SAAS,cAAc,OAAO,aAAa,KAAK;AAC9D,aAAO,cAAc;AAAA,QACnB;AAAA,QACA,WAAW,OAAO;AAAA,QAClB,YAAY,OAAO;AAAA,QACnB,SAAS,aAAa;AAAA,QACtB,SAAS,EAAE,oBAAoB,QAAQ,oBAAoB,SAAS;AAAA,QACpE;AAAA,MACF,CAAC;AAAA,IACH,SAAS,OAAO;AACd,aAAO,YAAY,UAAU,KAAK;AAAA,IACpC;AAAA,EACF,CAAC;AACH;AAcO,SAAS,gBAOd,SAGmD;AACnD,QAAM,YAAY,kBAAkB,QAAQ,aAAa,GAAG;AAC5D,QAAM,UAAU,gBAAgB,QAAQ,WAAW,CAAC;AACpD,QAAM,gBAAgB,QAAQ,iBAAiB;AAC/C,QAAM,aAAa,QAAQ,cAAc;AACzC,SAAO;AAAA,IACJ,QAAQ,QAAQ;AAAA,IACjB;AAAA,MACE,WAAW,aAAa,IAAI;AAAA,MAC5B,UAAU,QAAQ,YAAY;AAAA,IAChC;AAAA,IACA;AAAA,IACA,OAAO,SAAS;AACd,UAAI;AACF,cAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAI;AACJ,YAAI;AACJ,YAAI;AACF,mBAAS,KAAK,MAAM,MAAM;AAC1B,cAAI,CAAC,YAAY,MAAM,GAAG;AACxB,kBAAM,IAAI,UAAU,uCAAuC;AAAA,UAC7D;AACA,gBAAM,SAAS,QAAQ,OAAO,UAAU,MAAM;AAC9C,cAAI,CAAC,OAAO,SAAS;AACnB,8BAAkB,EAAE,cAAc,OAAO,KAAK;AAAA,UAChD;AAAA,QACF,SAAS,OAAO;AACd,4BAAkB,aAAa,KAAK;AAAA,QACtC;AACA,cAAM,QAAQ,oBAAoB,SAAY,IAAI;AAClD,YAAI;AACJ,YAAI,QAAQ,MAAW,MAAM;AAC7B,YAAI,eAAe;AACjB,cAAI,UAAU,GAAG;AACf,sBAAU;AAAA,UACZ,WAAW,QAAQ,UAAU,QAAW;AACtC,sBAAU;AAAA,UACZ,OAAO;AACL,kBAAM,eAAe,MAAM,SAAS;AAAA,cAClC,OAAO,QAAQ;AAAA,cACf,QAAQ;AAAA,cACR,cACE;AAAA,cACF,QAAQ,WAAW,EAAE,QAAQ,gBAAgB,CAAC;AAAA,cAC9C,aAAa,KAAK;AAAA,cAClB;AAAA,YACF,CAAC;AACD,sBAAU,aAAa,KAAK;AAC5B,oBAAQ,aAAa;AAAA,UACvB;AAAA,QACF;AACA,eAAO,cAAc;AAAA,UACnB;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA,SAAS,oBAAoB,SAAY,CAAC,IAAI,EAAE,gBAAgB;AAAA,UAChE;AAAA,QACF,CAAC;AAAA,MACH,SAAS,OAAO;AACd,eAAO,YAAY,UAAU,KAAK;AAAA,MACpC;AAAA,IACF;AAAA,EACF;AACF;AAgCO,SAAS,cAMd,SACyE;AACzE,QAAM,SAAS,aAAmB,SAAS,eAAe;AAC1D,SAAO,cAAc,OAAO,MAAM,QAAQ,mBAAmB,OAAO,SAAS;AAC3E,QAAI;AACF,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,UAAU,MAAM,kBAAkB,QAAQ,SAAS,KAAK,KAAK,SAAS,MAAM,SAAS;AAC3F,YAAM,gBAAgB,MAAM,SAAS;AAAA,QACnC,OAAO,QAAQ;AAAA,QACf,QAAQ;AAAA,QACR,cACE;AAAA,QACF,QAAQ,WAAW,EAAE,QAAQ,QAAQ,CAAC;AAAA,QACtC,aAAa,KAAK;AAAA,QAClB,SAAS,OAAO;AAAA,MAClB,CAAC;AACD,YAAM,WAAW,cAAc,KAAK;AACpC,uBAAiB,0BAA0B,SAAS,QAAQ;AAC5D,YAAM,QAAQ,SAAS,OAAO,CAAC,YAAY,QAAQ,YAAY,IAAI,EAAE,SAAS,SAAS;AACvF,YAAM,eAAe,MAAM,YAAY;AAAA,QACrC,OAAO,QAAQ;AAAA,QACf,eAAe,OAAO;AAAA,QACtB,SAAS,OAAO;AAAA,QAChB,QAAQ;AAAA,QACR,aAAa,KAAK;AAAA,QAClB;AAAA,QACA,UAAU,EAAE,SAAS;AAAA,MACvB,CAAC;AACD,YAAM,QAAQ,SAAS,cAAc,OAAO,aAAa,KAAK;AAC9D,aAAO,aAAa;AAAA,QAClB;AAAA,QACA,WAAW,OAAO;AAAA,QAClB,YAAY,OAAO;AAAA,QACnB,SAAS,aAAa;AAAA,QACtB,SAAS,EAAE,SAAS;AAAA,QACpB;AAAA,MACF,CAAC;AAAA,IACH,SAAS,OAAO;AACd,aAAO,YAAY,UAAU,KAAK;AAAA,IACpC;AAAA,EACF,CAAC;AACH;AA2BO,SAAS,aACd,SACyE;AACzE,QAAM,SAAS,aAAmB,SAAS,cAAc;AACzD,QAAM,wBAAwB;AAAA,IAC5B,QAAQ;AAAA,IACR;AAAA,EACF;AACA,SAAO,cAAc,OAAO,MAAM,QAAQ,oBAAoB,OAAO,SAAS;AAC5E,QAAI;AACF,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,mBAAmB,MAAM;AAAA,QAC7B,QAAQ;AAAA,QACR,KAAK,KAAK;AAAA,QACV;AAAA,QACA;AAAA,MACF;AACA,YAAM,CAAC,aAAa,WAAW,IAAI,MAAM,QAAQ,IAAI;AAAA,QACnD,SAAS;AAAA,UACP,OAAO,QAAQ;AAAA,UACf,QAAQ;AAAA,UACR,cAAc,6BAA6B,qBAAqB;AAAA,UAChE,QAAQ,WAAW,EAAE,iBAAiB,CAAC;AAAA,UACvC,aAAa,KAAK;AAAA,UAClB,SAAS,OAAO;AAAA,QAClB,CAAC;AAAA,QACD,SAAS;AAAA,UACP,OAAO,QAAQ;AAAA,UACf,QAAQ;AAAA,UACR,cACE;AAAA,UACF,QAAQ;AAAA,EAAY,MAAM;AAAA,UAC1B,aAAa,KAAK;AAAA,UAClB,SAAS,OAAO;AAAA,QAClB,CAAC;AAAA,MACH,CAAC;AACD,YAAM,SAAS,YAAY,YAAY,KAAK,OAAO,qBAAqB;AACxE,YAAM,SAAS,YAAY,KAAK;AAChC,UAAI,WAAsB,CAAC;AAC3B,UAAI,QAAQ,SAAS,YAAY,OAAO,YAAY,KAAK;AACzD,UAAI,OAAO,SAAS,GAAG;AACrB,cAAM,gBAAgB,MAAM,SAAS;AAAA,UACnC,OAAO,QAAQ;AAAA,UACf,QAAQ;AAAA,UACR,cACE;AAAA,UACF,QAAQ,WAAW,EAAE,QAAQ,OAAO,CAAC;AAAA,UACrC,aAAa,KAAK;AAAA,UAClB,SAAS,OAAO;AAAA,QAClB,CAAC;AACD,mBAAW,cAAc,KAAK;AAC9B,yBAAiB,yBAAyB,QAAQ,QAAQ;AAC1D,gBAAQ,SAAS,OAAO,cAAc,KAAK;AAAA,MAC7C;AACA,YAAM,0BAA0B,QAAQ,2BAA2B;AACnE,YAAM,YAAY,SAAS;AAAA,QACzB,CAAC,YACC,QAAQ,YAAY,SAAU,QAAQ,YAAY,SAAS,CAAC;AAAA,MAChE,EAAE;AACF,YAAM,QAAQ,SAAS,WAAW,IAAI,IAAI,YAAY,SAAS;AAC/D,YAAM,qBAAqB,kBAAkB,QAAQ;AACrD,YAAM,eAAe,MAAM,YAAY;AAAA,QACrC,OAAO,QAAQ;AAAA,QACf,eAAe,OAAO;AAAA,QACtB,SAAS,OAAO;AAAA,QAChB,QAAQ;AAAA,QACR,aAAa,KAAK;AAAA,QAClB;AAAA,QACA,UAAU,EAAE,UAAU,oBAAoB,wBAAwB;AAAA,MACpE,CAAC;AACD,cAAQ,SAAS,OAAO,aAAa,KAAK;AAC1C,aAAO,cAAc;AAAA,QACnB;AAAA,QACA,WAAW,OAAO;AAAA,QAClB,YAAY,OAAO;AAAA,QACnB,SAAS,aAAa;AAAA,QACtB,SAAS,EAAE,QAAQ,QAAQ,UAAU,oBAAoB,wBAAwB;AAAA,QACjF;AAAA,MACF,CAAC;AAAA,IACH,SAAS,OAAO;AACd,aAAO,YAAY,UAAU,KAAK;AAAA,IACpC;AAAA,EACF,CAAC;AACH;AAmBO,SAAS,WACd,SAC+D;AAC/D,QAAM,UAAU,gBAAgB,QAAQ,WAAW,CAAC;AACpD,SAAO;AAAA,IACL,MAAO,QAAQ,QAAQ;AAAA,IACvB,UAAU,QAAQ,YAAY;AAAA,IAC9B,UAAU;AAAA,IACV,MAAM,SAAS,MAAM;AACnB,UAAI;AACF,uBAAe,KAAK,MAAM;AAC1B,cAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,cAAM,gBAAgB,MAAM,gBAAgB,QAAQ,eAAe,IAAI;AACvE,YAAI,OAAO,kBAAkB,WAAW;AACtC,iBAAO,YAAY,QAAQ,qDAAqD;AAAA,QAClF;AACA,cAAM,UAAU,MAAM,yBAAyB,QAAQ,SAAS,IAAI;AACpE,YAAI,CAAC,iBAAiB,QAAQ,WAAW,GAAG;AAC1C,iBAAO,YAAY;AAAA,YACjB;AAAA,UACF;AAAA,QACF;AACA,cAAM,WAAW,MAAM,SAAS;AAAA,UAC9B,aAAa,KAAK;AAAA,UAClB,OAAO,QAAQ;AAAA,UACf,QAAQ;AAAA,UACR,cACE;AAAA,UACF,QAAQ,WAAW,EAAE,QAAQ,QAAQ,CAAC;AAAA,UACtC;AAAA,QACF,CAAC;AACD,uBAAe,KAAK,MAAM;AAC1B,cAAM,WAAW;AAAA,UACf;AAAA,UACA,SAAS,KAAK;AAAA,UACd,SAAS,KAAK;AAAA,QAChB;AACA,cAAM,iBAAiB;AAAA,UACrB,SAAS,QAAQ,kBAAkB,QAAQ,SAAY,SAAS,KAAK;AAAA,UACrE,UAAU;AAAA,YACR;AAAA,cACE,UAAU,SAAS,KAAK;AAAA,cACxB,UAAU,SAAS,KAAK;AAAA,cACxB;AAAA,YACF;AAAA,YACA,SAAS;AAAA,UACX;AAAA,UACA,OAAO,SAAS;AAAA,QAClB;AACA,eAAO,aAAa,wBAAwB,aAAa,4BACrD,YAAY,KAAK,UAAU,cAAc,IACzC,YAAY,KAAK,UAAU,cAAc;AAAA,MAC/C,SAAS,OAAO;AACd,eAAO,YAAY,UAAU,KAAK;AAAA,MACpC;AAAA,IACF;AAAA,EACF;AACF;AAEA,eAAe,yBACb,iBACA,MACmB;AACnB,QAAM,UACJ,oBAAoB,SACf,KAAK,KAAK,oBAAoB,CAAC,IAChC,OAAO,oBAAoB,aACzB,MAAO,gBAAqE,IAAI,IAChF;AACR,MAAI,CAAC,MAAM,QAAQ,OAAO,KAAK,QAAQ,KAAK,CAAC,UAAU,OAAO,UAAU,QAAQ,GAAG;AACjF,UAAM,IAAI,UAAU,iDAAiD;AAAA,EACvE;AACA,SAAO;AACT;AAEA,SAAS,mBACP,eACA,UACA,UACoB;AACpB,MAAI,aAAa,cAAc;AAC7B,WAAO,gBAAgB,uBAAuB;AAAA,EAChD;AACA,MAAI,iBAAiB,CAAC,SAAU,QAAO;AACvC,SAAO;AACT;AAYO,SAAS,cAMd,SACmD;AACnD,QAAM,SAAS,aAAmB,SAAS,eAAe;AAC1D,QAAM,gBAAgB,wBAAwB,QAAQ,iBAAiB,GAAG,eAAe;AACzF,QAAM,wBAAwB;AAAA,IAC5B,QAAQ;AAAA,IACR;AAAA,EACF;AACA,QAAM,oBACJ,QAAQ,wBAAwB,UAAa,QAAQ,oBAAoB,SAAS,IAC9E,CAAC,GAAG,QAAQ,mBAAmB,IAC/B;AACN,SAAO,cAAc,OAAO,MAAM,QAAQ,oBAAoB,OAAO,SAAS;AAC5E,QAAI;AACF,YAAM,QAAQ,MAAM,aAAa,QAAQ,OAAO,IAAI;AACpD,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,kBACJ,sBAAsB,SAClB,SAAS;AAAA,QACP,OAAO,QAAQ;AAAA,QACf,QAAQ;AAAA,QACR,cAAc,oBAAoB,aAAa;AAAA,QAC/C,QAAQ;AAAA,EAAiB,KAAK;AAAA,QAC9B,aAAa,KAAK;AAAA,QAClB,SAAS,OAAO;AAAA,MAClB,CAAC,IACD,QAAQ,QAAQ;AAAA,QACd,MAAM,EAAE,WAAW,kBAAkB;AAAA,QACrC,OAAO,MAAW,MAAM;AAAA,MAC1B,CAAC;AACP,YAAM,CAAC,aAAa,aAAa,cAAc,IAAI,MAAM,QAAQ,IAAI;AAAA,QACnE,SAAS;AAAA,UACP,OAAO,QAAQ;AAAA,UACf,QAAQ;AAAA,UACR,cAAc,6BAA6B,qBAAqB;AAAA,UAChE,QAAQ;AAAA,EAAiB,KAAK;AAAA,UAC9B,aAAa,KAAK;AAAA,UAClB,SAAS,OAAO;AAAA,QAClB,CAAC;AAAA,QACD,SAAS;AAAA,UACP,OAAO,QAAQ;AAAA,UACf,QAAQ;AAAA,UACR,cACE;AAAA,UACF,QAAQ;AAAA,EAAa,MAAM;AAAA,UAC3B,aAAa,KAAK;AAAA,UAClB,SAAS,OAAO;AAAA,QAClB,CAAC;AAAA,QACD;AAAA,MACF,CAAC;AACD,YAAM,SAAS,YAAY,YAAY,KAAK,OAAO,qBAAqB;AACxE,YAAM,SAAS,YAAY,KAAK;AAChC,YAAM,YAAY,eAAe,KAAK;AACtC,UAAI,UAAU,WAAW,GAAG;AAC1B,cAAM,IAAI,MAAM,uDAAuD;AAAA,MACzE;AACA,YAAM,sBAAsB,SAAS;AAAA,QACnC,OAAO,QAAQ;AAAA,QACf,QAAQ;AAAA,QACR,cACE;AAAA,QACF,QAAQ,WAAW,EAAE,WAAW,MAAM,MAAM,CAAC;AAAA,QAC7C,aAAa,KAAK;AAAA,QAClB,SAAS,OAAO;AAAA,MAClB,CAAC;AACD,YAAM,uBAAuB,SAAS;AAAA,QACpC,OAAO,QAAQ;AAAA,QACf,QAAQ;AAAA,QACR,cACE;AAAA,QACF,QAAQ,WAAW,EAAE,WAAW,MAAM,OAAO,CAAC;AAAA,QAC9C,aAAa,KAAK;AAAA,QAClB,SAAS,OAAO;AAAA,MAClB,CAAC;AACD,YAAM,mBACJ,OAAO,WAAW,IACd,QAAQ,QAAQ,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,GAAG,OAAO,MAAW,MAAM,EAAE,CAAC,IACrE,SAAS;AAAA,QACP,OAAO,QAAQ;AAAA,QACf,QAAQ;AAAA,QACR,cACE;AAAA,QACF,QAAQ,WAAW,EAAE,QAAQ,OAAO,CAAC;AAAA,QACrC,aAAa,KAAK;AAAA,QAClB,SAAS,OAAO;AAAA,MAClB,CAAC;AACP,YAAM,CAAC,oBAAoB,qBAAqB,eAAe,IAAI,MAAM,QAAQ,IAAI;AAAA,QACnF;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AACD,uBAAiB,6BAA6B,WAAW,mBAAmB,KAAK,OAAO;AACxF,uBAAiB,8BAA8B,WAAW,oBAAoB,KAAK,OAAO;AAC1F,uBAAiB,oCAAoC,QAAQ,gBAAgB,KAAK,QAAQ;AAC1F,YAAM,oBAAoB,gBAAgB,KAAK;AAC/C,YAAM,iBACJ,kBAAkB,WAAW,IACzB,IACA,kBAAkB,OAAO,CAAC,YAAY,QAAQ,YAAY,KAAK,EAAE,SACjE,kBAAkB;AACxB,UAAI,gBAAgB;AACpB,UAAI,kBAAkB;AACtB,YAAM,mBAAmB,UAAU,IAAI,CAAC,UAAU,UAAU;AAC1D,cAAM,kBAAkB,mBAAmB,KAAK,QAAQ,KAAK;AAC7D,cAAM,iBAAiB,oBAAoB,KAAK,QAAQ,KAAK;AAC7D,YAAI,oBAAoB,OAAO;AAC7B,2BAAiB;AACjB,cAAI,mBAAmB,MAAO,oBAAmB;AAAA,QACnD;AACA,eAAO,EAAE,UAAU,iBAAiB,eAAe;AAAA,MACrD,CAAC;AACD,YAAM,gBAAgB,kBAAkB,IAAI,IAAI,kBAAkB;AAClE,YAAM,QAAQ,KAAK,IAAI,gBAAgB,aAAa;AACpD,YAAM,8BAA8B,kBAAkB,iBAAiB;AACvE,YAAM,eAAe,MAAM,YAAY;AAAA,QACrC,OAAO,QAAQ;AAAA,QACf,eAAe,OAAO;AAAA,QACtB,SAAS,OAAO;AAAA,QAChB,QAAQ;AAAA,QACR,aAAa,KAAK;AAAA,QAClB;AAAA,QACA,UAAU;AAAA,UACR,mBAAmB;AAAA,UACnB;AAAA,UACA;AAAA,UACA;AAAA,QACF;AAAA,MACF,CAAC;AACD,YAAM,QAAQ;AAAA,QACZ,YAAY;AAAA,QACZ,YAAY;AAAA,QACZ,eAAe;AAAA,QACf,mBAAmB;AAAA,QACnB,oBAAoB;AAAA,QACpB,gBAAgB;AAAA,QAChB,aAAa;AAAA,MACf;AACA,aAAO,cAAc;AAAA,QACnB;AAAA,QACA,WAAW,OAAO;AAAA,QAClB,YAAY,OAAO;AAAA,QACnB,SAAS,aAAa;AAAA,QACtB,SAAS;AAAA,UACP;AAAA,UACA;AAAA,UACA,qBAAqB;AAAA,UACrB,mBAAmB;AAAA,UACnB;AAAA,UACA,gBAAgB,EAAE,WAAW,gBAAgB,UAAU,cAAc;AAAA,QACvE;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH,SAAS,OAAO;AACd,aAAO,YAAY,UAAU,KAAK;AAAA,IACpC;AAAA,EACF,CAAC;AACH;AA4FO,SAAS,MACd,SACmD;AACnD,MAAI,QAAQ,KAAK,KAAK,EAAE,WAAW,EAAG,OAAM,IAAI,UAAU,+BAA+B;AACzF,MAAI,QAAQ,iBAAiB,WAAW,GAAG;AACzC,UAAM,IAAI,UAAU,mDAAmD;AAAA,EACzE;AACA,MAAK,QAAQ,aAAa,YAAgB,QAAQ,oBAAoB,SAAY;AAChF,UAAM,IAAI,UAAU,4DAA4D;AAAA,EAClF;AACA,MAAI,QAAQ,aAAa,UAAa,QAAQ,SAAS,KAAK,EAAE,WAAW,GAAG;AAC1E,UAAM,IAAI,UAAU,mCAAmC;AAAA,EACzD;AACA,MAAI,QAAQ,oBAAoB,UAAa,QAAQ,gBAAgB,WAAW,GAAG;AACjF,UAAM,IAAI,UAAU,0CAA0C;AAAA,EAChE;AACA,QAAM,SAAS,aAAmB,SAAS,QAAQ,IAAI;AACvD,QAAM,SAAS,eAAe,QAAQ,MAAM;AAC5C,QAAM,aACJ,OAAO,WAAW,IACb,CAAC,GAAG,EAAE,IACN,CAAC,OAAO,CAAC,GAAG,WAAW,CAAC,KAAK,GAAG,OAAO,GAAG,EAAE,GAAG,WAAW,CAAC,KAAK,EAAE;AACzE,MAAI;AACJ,MAAI;AAEJ,WAAS,YAAY,QAAuB,QAAqB;AAC/D,mBAAe,MAAM;AACrB,WAAO,EAAE,OAAO,CAAC,GAAG,OAAO,KAAK,GAAG,aAAa,OAAO;AAAA,EACzD;AAEA,iBAAe,aACb,QAC2D;AAC3D,mBAAe,MAAM;AACrB,QAAI,QAAQ,oBAAoB,QAAW;AACzC,aAAO,EAAE,OAAO,CAAC,GAAG,QAAQ,eAAe,EAAE;AAAA,IAC/C;AACA,QAAI,kBAAkB,OAAW,QAAO,YAAY,eAAe,MAAM;AACzE,QAAI,uBAAuB,QAAW;AACpC,YAAM,aAAa,IAAI,gBAAgB;AACvC,YAAMA,SAA0B;AAAA,QAC9B;AAAA,QACA,SAAS,oBAAI,IAAI;AAAA,QACjB,SAAS,QAAQ,QAAQ,EACtB;AAAA,UAAK,MACJ,SAAS;AAAA,YACP,OAAO,QAAQ;AAAA,YACf,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;AAAA,YAC/C,cACE;AAAA,YACF,QAAQ,WAAW;AAAA,cACjB,UAAU,QAAQ;AAAA,cAClB,YAAY,QAAQ;AAAA,YACtB,CAAC;AAAA,YACD,SAAS,OAAO;AAAA,YAChB,aAAa,WAAW;AAAA,UAC1B,CAAC;AAAA,QACH,EACC,KAAK,CAAC,WAAW;AAChB,cAAI,OAAO,KAAK,MAAM,WAAW,GAAG;AAClC,kBAAM,IAAI,MAAM,uCAAuC;AAAA,UACzD;AACA,gBAAM,WAAW;AAAA,YACf,OAAO,CAAC,GAAG,OAAO,KAAK,KAAK;AAAA,YAC5B,OAAO,OAAO;AAAA,YACd,cAAc;AAAA,UAChB;AACA,cAAI,uBAAuBA,QAAO;AAChC,4BAAgB;AAChB,iCAAqB;AAAA,UACvB;AACA,iBAAO;AAAA,QACT,CAAC,EACA,MAAM,CAAC,UAAmB;AACzB,cAAI,uBAAuBA,OAAO,sBAAqB;AACvD,gBAAM;AAAA,QACR,CAAC;AAAA,MACL;AACA,2BAAqBA;AAAA,IACvB;AACA,UAAM,QAAQ;AACd,UAAM,QAAQ,CAAC;AACf,UAAM,QAAQ,IAAI,KAAK;AACvB,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,YAAM,UAAU,MAAM;AACpB,YAAI,CAAC,MAAM,QAAQ,OAAO,KAAK,EAAG;AAClC,eAAO,oBAAoB,SAAS,OAAO;AAC3C,YAAI,uBAAuB,SAAS,MAAM,QAAQ,SAAS,GAAG;AAC5D,+BAAqB;AACrB,gBAAM,WAAW,MAAM,OAAO,MAAM;AAAA,QACtC;AAAA,MACF;AACA,YAAM,UAAU,MAAM;AACpB,gBAAQ;AACR,eAAO,OAAO,MAAM;AAAA,MACtB;AACA,aAAO,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;AACxD,YAAM,QAAQ;AAAA,QACZ,CAAC,WAAW;AACV,cAAI,CAAC,MAAM,QAAQ,IAAI,KAAK,EAAG;AAC/B,cAAI;AACF,kBAAM,WAAW,YAAY,QAAQ,MAAM;AAC3C,oBAAQ;AACR,oBAAQ,QAAQ;AAAA,UAClB,SAAS,OAAO;AACd,oBAAQ;AACR,mBAAO,KAAK;AAAA,UACd;AAAA,QACF;AAAA,QACA,CAAC,UAAmB;AAClB,kBAAQ;AACR,iBAAO,KAAK;AAAA,QACd;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO,cAAc,OAAO,MAAM,QAAQ,oBAAoB,OAAO,SAAS;AAC5E,QAAI;AACF,YAAM,aAAa,MAAM,uBAAuB,SAAS,IAAI;AAC7D,YAAM,cAAc,MAAM,aAAa,KAAK,MAAM;AAClD,YAAM,cAAc,MAAM,SAAS;AAAA,QACjC,aAAa,KAAK;AAAA,QAClB,OAAO,QAAQ;AAAA,QACf,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,GAAG,QAAQ,EAAE,OAAO,EAAE,CAAC;AAAA,QAC1D,cAAc,OAAO,aACjB,mIACA,+DAA+D,WAAW,CAAC,CAAC,YAAY,WAAW,CAAC,CAAC;AAAA,QACzG,QAAQ,WAAW;AAAA,UACjB,iBAAiB,YAAY;AAAA,UAC7B;AAAA,UACA;AAAA,QACF,CAAC;AAAA,QACD,SAAS,OAAO;AAAA,MAClB,CAAC;AACD,YAAM,WAAW,YAAY,KAAK;AAClC,UACE,CAAC,OAAO,SAAS,QAAQ,KACxB,OAAO,cAAc,aAAa,KAAK,aAAa,KACpD,CAAC,OAAO,eAAe,WAAW,WAAW,CAAC,KAAK,WAAW,WAAW,CAAC,IAC3E;AACA,cAAM,IAAI,WAAW,gBAAgB,QAAQ,kCAAkC;AAAA,MACjF;AACA,YAAM,QAAQ,OAAO,aACjB,YACC,WAAW,WAAW,CAAC,MAAM,WAAW,CAAC,IAAI,WAAW,CAAC;AAC9D,qBAAe,KAAK,MAAM;AAC1B,YAAM,cAAc,YAAY;AAChC,YAAM,aACJ,gBAAgB,UAAa,YAAY,eACrC,MAAW,MAAM,IACjB,YAAY;AAClB,YAAM,QAAQ,SAAS,YAAY,YAAY,KAAK;AACpD,YAAM,UAAU,cAAc;AAAA,QAC5B;AAAA,QACA,WAAW,OAAO;AAAA,QAClB,YAAY,OAAO;AAAA,QACnB,SAAS,OAAO,gBAAgB,YAAY,KAAK,SAAS;AAAA,QAC1D,SAAS;AAAA,UACP,iBAAiB,YAAY;AAAA,UAC7B,kBAAkB,QAAQ;AAAA,UAC1B;AAAA,UACA,YAAY,CAAC,WAAW,CAAC,GAAG,WAAW,CAAC,CAAC;AAAA,UACzC,QAAQ,OAAO,IAAI,CAAC,WAAW;AAAA,YAC7B,YAAY,CAAC,MAAM,WAAW,CAAC,GAAG,MAAM,WAAW,CAAC,CAAC;AAAA,YACrD,iBAAiB,MAAM;AAAA,UACzB,EAAE;AAAA,QACJ;AAAA,QACA;AAAA,MACF,CAAC;AACD,UAAI,gBAAgB,OAAW,aAAY,eAAe;AAC1D,aAAO;AAAA,IACT,SAAS,OAAO;AACd,aAAO,YAAY,UAAU,KAAK;AAAA,IACpC;AAAA,EACF,CAAC;AACH;AAsBO,SAAS,cAMd,SACmD;AACnD,QAAM,SAAS,aAAmB,SAAS,gBAAgB;AAC3D,QAAM,aAAa,wBAAwB,QAAQ,cAAc,IAAI,YAAY;AACjF,QAAM,cAAc,wBAAwB,QAAQ,eAAe,GAAG,aAAa;AACnF,SAAO,cAAc,OAAO,MAAM,QAAQ,oBAAoB,OAAO,SAAS;AAC5E,QAAI;AACF,YAAM,QAAQ,MAAM,aAAa,QAAQ,OAAO,IAAI;AACpD,YAAM,eAAe,iBAAiB,KAAK;AAC3C,YAAM,UAAU,aAAa;AAAA,QAAI,CAAC,GAAG,UACnC,aAAa,MAAM,KAAK,IAAI,GAAG,QAAQ,aAAa,CAAC,GAAG,QAAQ,CAAC,EAAE,KAAK;AAAA,MAC1E;AACA,YAAM,iBAAiB,MAAM;AAAA,QAAmB;AAAA,QAAS;AAAA,QAAa,CAAC,WACrE,SAAS;AAAA,UACP,OAAO,QAAQ;AAAA,UACf,QAAQ;AAAA,UACR,cACE;AAAA,UACF,QAAQ,WAAW,EAAE,OAAO,OAAO,CAAC;AAAA,UACpC,aAAa,KAAK;AAAA,UAClB,SAAS,OAAO;AAAA,QAClB,CAAC;AAAA,MACH;AACA,YAAM,WAAW,eAAe,IAAI,CAAC,QAAQ,WAAW;AAAA,QACtD,aAAa,QAAQ;AAAA,QACrB,GAAG,OAAO;AAAA,MACZ,EAAE;AACF,YAAM,QACJ,SAAS,WAAW,IAChB,IACA,SAAS,OAAO,CAAC,YAAY,QAAQ,YAAY,KAAK,EAAE,SAAS,SAAS;AAChF,YAAM,eAAe,MAAM,YAAY;AAAA,QACrC,OAAO,QAAQ;AAAA,QACf,eAAe,OAAO;AAAA,QACtB,SAAS,OAAO;AAAA,QAChB,QAAQ;AAAA,QACR,aAAa,KAAK;AAAA,QAClB;AAAA,QACA,UAAU,EAAE,SAAS;AAAA,MACvB,CAAC;AACD,YAAM,QAAQ,SAAS,GAAG,eAAe,IAAI,CAAC,WAAW,OAAO,KAAK,GAAG,aAAa,KAAK;AAC1F,aAAO,cAAc;AAAA,QACnB;AAAA,QACA,WAAW,OAAO;AAAA,QAClB,YAAY,OAAO;AAAA,QACnB,SAAS,aAAa;AAAA,QACtB,SAAS,EAAE,YAAY,aAAa,kBAAkB,aAAa,QAAQ,SAAS;AAAA,QACpF;AAAA,MACF,CAAC;AAAA,IACH,SAAS,OAAO;AACd,aAAO,YAAY,UAAU,KAAK;AAAA,IACpC;AAAA,EACF,CAAC;AACH;AAQO,SAAS,mBAMd,SACmD;AACnD,QAAM,SAAS,aAAmB,SAAS,qBAAqB;AAChE,QAAM,cAAc,wBAAwB,QAAQ,eAAe,GAAG,aAAa;AACnF,SAAO,cAAc,OAAO,MAAM,QAAQ,oBAAoB,OAAO,SAAS;AAC5E,QAAI;AACF,YAAM,QAAQ,MAAM,aAAa,QAAQ,OAAO,IAAI;AACpD,YAAM,YAAY,MACf,IAAI,CAAC,MAAM,WAAW,EAAE,MAAM,MAAM,EAAE,EACtC,OAAO,CAAC,UAAU,MAAM,KAAK,SAAS,MAAM;AAC/C,YAAM,mBAAmB,MAAM;AAAA,QAAmB;AAAA,QAAW;AAAA,QAAa,CAAC,UACzE,SAAS;AAAA,UACP,OAAO,QAAQ;AAAA,UACf,QAAQ;AAAA,UACR,cACE;AAAA,UACF,QAAQ,WAAW;AAAA,YACjB,eAAe,MAAM,MAAM,GAAG,MAAM,KAAK;AAAA,YACzC,aAAa,MAAM,KAAK;AAAA,UAC1B,CAAC;AAAA,UACD,aAAa,KAAK;AAAA,UAClB,SAAS,OAAO;AAAA,QAClB,CAAC;AAAA,MACH;AACA,YAAM,YAAY,UAAU,IAAI,CAAC,OAAO,WAAW;AAAA,QACjD,WAAW,MAAM;AAAA,QACjB,OAAO,iBAAiB,KAAK,GAAG,KAAK,SAAS,CAAC;AAAA,MACjD,EAAE;AACF,YAAM,kBAAkB,MACrB,IAAI,CAAC,MAAM,WAAW,EAAE,MAAM,MAAM,EAAE,EACtC,OAAO,CAAC,UAAU,MAAM,KAAK,SAAS,WAAW,EACjD,IAAI,CAAC,WAAW;AAAA,QACf,GAAG;AAAA,QACH,OAAO,UACJ,OAAO,CAAC,SAAS,KAAK,YAAY,MAAM,KAAK,EAC7C,QAAQ,CAAC,SAAS,KAAK,KAAK;AAAA,MACjC,EAAE,EACD,OAAO,CAAC,UAAU,MAAM,MAAM,SAAS,CAAC;AAC3C,YAAM,iBAAiB,MAAM;AAAA,QAAmB;AAAA,QAAiB;AAAA,QAAa,CAAC,UAC7E,SAAS;AAAA,UACP,OAAO,QAAQ;AAAA,UACf,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,GAAG,QAAQ,EAAE,OAAO,EAAE,CAAC;AAAA,UAC/D,cACE;AAAA,UACF,QAAQ,WAAW,EAAE,YAAY,MAAM,OAAO,gBAAgB,MAAM,KAAK,QAAQ,CAAC;AAAA,UAClF,aAAa,KAAK;AAAA,UAClB,SAAS,OAAO;AAAA,QAClB,CAAC;AAAA,MACH;AACA,YAAM,WAAW,gBAAgB,IAAI,CAAC,OAAO,WAAW;AAAA,QACtD,WAAW,MAAM;AAAA,QACjB,WAAW,eAAe,KAAK,GAAG,KAAK,aAAa;AAAA,QACpD,QAAQ,eAAe,KAAK,GAAG,KAAK,UAAU;AAAA,MAChD,EAAE;AACF,YAAM,QACJ,SAAS,WAAW,IAChB,IACA,SAAS,OAAO,CAAC,YAAY,CAAC,QAAQ,SAAS,EAAE,SAAS,SAAS;AACzE,YAAM,eAAe,MAAM,YAAY;AAAA,QACrC,OAAO,QAAQ;AAAA,QACf,eAAe,OAAO;AAAA,QACtB,SAAS,OAAO;AAAA,QAChB,QAAQ;AAAA,QACR,aAAa,KAAK;AAAA,QAClB;AAAA,QACA,UAAU,EAAE,SAAS;AAAA,MACvB,CAAC;AACD,YAAM,QAAQ;AAAA,QACZ,GAAG,iBAAiB,IAAI,CAAC,WAAW,OAAO,KAAK;AAAA,QAChD,GAAG,eAAe,IAAI,CAAC,WAAW,OAAO,KAAK;AAAA,QAC9C,aAAa;AAAA,MACf;AACA,aAAO,cAAc;AAAA,QACnB;AAAA,QACA,WAAW,OAAO;AAAA,QAClB,YAAY,OAAO;AAAA,QACnB,SAAS,aAAa;AAAA,QACtB,SAAS,EAAE,aAAa,WAAW,SAAS;AAAA,QAC5C;AAAA,MACF,CAAC;AAAA,IACH,SAAS,OAAO;AACd,aAAO,YAAY,UAAU,KAAK;AAAA,IACpC;AAAA,EACF,CAAC;AACH;AAEA,SAAS,cACP,MACA,QACA,WACA,UACmD;AACnD,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB;AAAA,IACA,WACE,OAAO,eAAe,OAAQ,cAAc,qBAAqB,IAAI,IAAK,OAAO;AAAA,IACnF,UAAU;AAAA,IACV,MAAM,SAAS,MAAM;AACnB,UAAI;AACF,uBAAe,KAAK,MAAM;AAC1B,cAAM,UAAU,MAAM,SAAS,IAAI;AACnC,uBAAe,KAAK,MAAM;AAC1B,eAAO;AAAA,MACT,SAAS,OAAO;AACd,eAAO,YAAY,UAAU,KAAK;AAAA,MACpC;AAAA,IACF;AAAA,EACF;AACF;AAWA,SAAS,aACP,SAQA,aACoB;AACpB,SAAO;AAAA,IACL,MAAO,QAAQ,QAAQ;AAAA,IACvB,WAAW,kBAAkB,QAAQ,aAAa,GAAG;AAAA,IACrD,YAAY,QAAQ,cAAc;AAAA,IAClC,eAAe,QAAQ,iBAAiB;AAAA,IACxC,SAAS,gBAAgB,QAAQ,WAAW,CAAC;AAAA,IAC7C,UAAU,QAAQ,YAAY;AAAA,EAChC;AACF;AAEA,SAAS,cAAc,MAOpB;AACD,QAAM,QAAQ,KAAK,aAAc,KAAK,UAAU,IAAI,IAAI,IAAK,KAAK;AAClE,QAAM,YAAY,KAAK,aAAa,IAAI,KAAK;AAC7C,QAAM,UAAU;AAAA,IACd,SAAS,KAAK;AAAA,IACd,OAAO,KAAK;AAAA,IACZ,UAAU;AAAA,MACR;AAAA,QACE,GAAG,KAAK;AAAA,QACR,gBAAgB;AAAA,QAChB;AAAA,QACA,YAAY,KAAK;AAAA,MACnB;AAAA,MACA,KAAK;AAAA,IACP;AAAA,EACF;AACA,SAAO,SAAS,YAAY,YAAY,KAAK,OAAO,OAAO,IAAI,YAAY,KAAK,OAAO,OAAO;AAChG;AAEA,SAAS,aAAa,MAOnB;AACD,QAAM,QAAQ,KAAK,aAAc,KAAK,UAAU,IAAI,IAAI,IAAK,KAAK;AAClE,QAAM,YAAY,KAAK,aAAa,IAAI,KAAK;AAC7C,QAAM,UAAU;AAAA,IACd,SAAS,KAAK;AAAA,IACd,OAAO,KAAK;AAAA,IACZ,UAAU;AAAA,MACR;AAAA,QACE,GAAG,KAAK;AAAA,QACR,gBAAgB;AAAA,QAChB;AAAA,QACA,YAAY,KAAK;AAAA,MACnB;AAAA,MACA,KAAK;AAAA,IACP;AAAA,EACF;AACA,SAAO,SAAS,YAAY,YAAY,KAAK,OAAO,OAAO,IAAI,YAAY,KAAK,OAAO,OAAO;AAChG;AAEA,eAAe,YAAY,MAQgC;AACzD,iBAAe,KAAK,WAAW;AAC/B,MAAI,CAAC,KAAK,cAAe,QAAO,EAAE,OAAO,MAAW,MAAM,EAAE;AAC5D,QAAM,SAAS,MAAM,SAAS;AAAA,IAC5B,OAAO,KAAK;AAAA,IACZ,QAAQ;AAAA,IACR,cAAc,6CAA6C,KAAK,MAAM;AAAA,IACtE,QAAQ,WAAW,EAAE,OAAO,KAAK,OAAO,UAAU,KAAK,SAAS,CAAC;AAAA,IACjE,aAAa,KAAK;AAAA,IAClB,SAAS,KAAK;AAAA,EAChB,CAAC;AACD,SAAO,EAAE,QAAQ,OAAO,KAAK,QAAQ,OAAO,OAAO,MAAM;AAC3D;AAEA,eAAe,aACb,UACA,MACiB;AACjB,SAAO,aAAa,SAAY,YAAY,KAAK,KAAK,KAAK,IAAI,SAAS,IAAI;AAC9E;AAEA,eAAe,kBACb,iBACA,UACA,MACA,OACmB;AACnB,QAAM,QACJ,oBAAoB,SAChB,WACA,OAAO,oBAAoB,aACzB,MAAO,gBAAqE,IAAI,IAChF;AACR,MACE,CAAC,MAAM,QAAQ,KAAK,KACpB,MAAM,WAAW,KACjB,MAAM,KAAK,CAAC,SAAS,OAAO,SAAS,QAAQ,GAC7C;AACA,UAAM,IAAI,UAAU,GAAG,KAAK,wCAAwC;AAAA,EACtE;AACA,SAAO;AACT;AAEA,eAAe,uBACb,SACA,MACqB;AACrB,QAAM,aAAyB,CAAC;AAChC,aAAW,aAAa,QAAQ,kBAAkB;AAChD,QAAI,cAAc,QAAS,YAAW,QAAQ,MAAM,aAAa,QAAQ,OAAO,IAAI;AACpF,QAAI,cAAc,gBAAgB;AAChC,iBAAW,eAAe,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAAA,IACxE;AACA,QAAI,cAAc,kBAAkB;AAClC,YAAM,WACJ,QAAQ,aAAa,SAAY,KAAK,KAAK,WAAW,MAAM,QAAQ,SAAS,IAAI;AACnF,UAAI,aAAa,OAAW,OAAM,IAAI,MAAM,mCAAmC;AAC/E,iBAAW,iBAAiB,YAAY,QAAQ;AAAA,IAClD;AACA,QAAI,cAAc,WAAW;AAC3B,iBAAW,UAAU,MAAM;AAAA,QACzB,QAAQ;AAAA,QACR,KAAK,KAAK;AAAA,QACV;AAAA,QACA;AAAA,MACF;AAAA,IACF;AACA,QAAI,cAAc,oBAAoB;AACpC,iBAAW,mBAAmB,MAAM;AAAA,QAClC,QAAQ;AAAA,QACR,KAAK,KAAK;AAAA,QACV;AAAA,QACA;AAAA,MACF;AAAA,IACF;AACA,QAAI,cAAc,WAAY,YAAW,WAAW,KAAK,KAAK,YAAY,CAAC;AAAA,EAC7E;AACA,SAAO;AACT;AAEA,SAAS,eAAe,QAAkD;AACxE,MAAI,WAAW,UAAa,OAAO,WAAW,EAAG,QAAO,CAAC;AACzD,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,MAAM,UAAU,KAAK,WAAW,CAAC,IAAI,MAAM,WAAW,CAAC,CAAC;AACzF,aAAW,CAAC,OAAO,KAAK,KAAK,OAAO,QAAQ,GAAG;AAC7C,UAAM,CAAC,OAAO,GAAG,IAAI,MAAM;AAC3B,QACE,CAAC,OAAO,UAAU,KAAK,KACvB,CAAC,OAAO,UAAU,GAAG,KACrB,QAAQ,KACR,MAAM,MACN,QAAQ,KACR;AACA,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,QAAI,MAAM,gBAAgB,KAAK,EAAE,WAAW,GAAG;AAC7C,YAAM,IAAI,UAAU,kDAAkD;AAAA,IACxE;AACA,UAAM,OAAO,OAAO,QAAQ,CAAC;AAC7B,QAAI,SAAS,UAAa,OAAO,KAAK,WAAW,CAAC,GAAG;AACnD,YAAM,IAAI,WAAW,8CAA8C;AAAA,IACrE;AAAA,EACF;AACA,QAAM,QAAQ,OAAO,CAAC;AACtB,QAAM,OAAO,OAAO,GAAG,EAAE;AACzB,MAAI,UAAU,UAAa,SAAS,UAAa,MAAM,WAAW,CAAC,MAAM,KAAK,WAAW,CAAC,GAAG;AAC3F,UAAM,IAAI,WAAW,0DAA0D;AAAA,EACjF;AACA,SAAO;AACT;AAEA,SAAS,6BAA6B,OAAmC;AACvE,SAAO,UAAU,SACb,0IACA,mBAAmB,KAAK;AAC9B;AAEA,SAAS,YAAY,QAAkB,OAAqC;AAC1E,SAAO,UAAU,SAAY,SAAS,OAAO,MAAM,GAAG,KAAK;AAC7D;AAEA,SAAS,kBAAkB,OAAuB;AAChD,MAAI,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,KAAK,QAAQ,GAAG;AACrD,UAAM,IAAI,WAAW,gDAAgD;AAAA,EACvE;AACA,SAAO;AACT;AAEA,SAAS,gBAAgB,OAAuB;AAC9C,MAAI,CAAC,OAAO,UAAU,KAAK,KAAK,QAAQ,GAAG;AACzC,UAAM,IAAI,WAAW,qDAAqD;AAAA,EAC5E;AACA,SAAO;AACT;AAEA,SAAS,wBAAwB,OAAe,OAAuB;AACrE,MAAI,CAAC,OAAO,UAAU,KAAK,KAAK,QAAQ,GAAG;AACzC,UAAM,IAAI,WAAW,GAAG,KAAK,8BAA8B;AAAA,EAC7D;AACA,SAAO;AACT;AAEA,SAAS,mCACP,OACA,OACoB;AACpB,MAAI,UAAU,OAAW,QAAO;AAChC,MAAI,CAAC,OAAO,UAAU,KAAK,KAAK,QAAQ,GAAG;AACzC,UAAM,IAAI,WAAW,GAAG,KAAK,kCAAkC;AAAA,EACjE;AACA,SAAO;AACT;AAEA,SAAS,iBAAiB,OAAe,QAAmB,SAA0B;AACpF,MAAI,OAAO,WAAW,QAAQ,QAAQ;AACpC,UAAM,IAAI,MAAM,GAAG,KAAK,UAAU,QAAQ,MAAM,8BAA8B,OAAO,MAAM,GAAG;AAAA,EAChG;AACF;AAEA,SAAS,kBAAkB,UAA4C;AACrE,SAAO,SAAS,IAAI,CAAC,YAAY;AAC/B,UAAM,aAAyB,EAAE,SAAS,QAAQ,QAAQ;AAC1D,QAAI,QAAQ,WAAW,OAAW,YAAW,SAAS,QAAQ;AAC9D,WAAO;AAAA,EACT,CAAC;AACH;AAEA,SAAS,WAAW,OAAwB;AAC1C,SAAO,KAAK,UAAU,OAAO,MAAM,CAAC;AACtC;AAEA,SAAS,YAAY,OAA2B;AAC9C,MAAI,CAAC,YAAY,KAAK,GAAG;AACvB,UAAM,IAAI,UAAU,6CAA6C;AAAA,EACnE;AACA,SAAO,eAAe,KAAK;AAC7B;AAEA,SAAS,eAAe,OAA6B;AACnD,MAAI,UAAU,QAAQ,OAAO,UAAU,SAAU,QAAO;AACxD,MAAI,MAAM,QAAQ,KAAK,EAAG,QAAO,MAAM,IAAI,cAAc;AACzD,SAAO,OAAO;AAAA,IACZ,OAAO,QAAQ,KAAK,EAAE,IAAI,CAAC,CAAC,KAAK,IAAI,MAAM;AACzC,UAAI,SAAS,QAAW;AACtB,cAAM,IAAI,UAAU,6CAA6C;AAAA,MACnE;AACA,aAAO,CAAC,KAAK,eAAe,IAAI,CAAC;AAAA,IACnC,CAAC;AAAA,EACH;AACF;AAIA,SAAS,mBAAmB,OAAwC;AAClE,QAAM,SAAS,kBAAkB,KAAK;AACtC,MAAI,WAAW,OAAW,QAAO;AACjC,QAAM,QAAoB,CAAC;AAC3B,aAAW,SAAS,QAAQ;AAC1B,QAAI,OAAO,UAAU,YAAY,UAAU,KAAM;AACjD,UAAM,OAAQ,MAA6B;AAC3C,QAAI,SAAS,UAAU,SAAS,YAAa;AAC7C,UAAM,UAAW,MAAgC;AACjD,UAAM,OAAO,YAAY,OAAO;AAChC,QAAI,KAAK,WAAW,EAAG;AACvB,UAAM,WAAY,MAAiC;AACnD,UAAM;AAAA,MACJ,OAAO,aAAa,YAAY,aAAa,QAAQ,CAAC,MAAM,QAAQ,QAAQ,IACxE,EAAE,MAAM,SAAS,MAAM,SAA2C,IAClE,EAAE,MAAM,SAAS,KAAK;AAAA,IAC5B;AAAA,EACF;AACA,SAAO,MAAM,WAAW,IAAI,SAAY;AAC1C;AAEA,eAAe,aACb,UACA,MACqB;AACrB,QAAM,SAAS,aAAa,SAAY,KAAK,SAAS,MAAM,SAAS,IAAI;AACzE,QAAM,QAAQ,mBAAmB,MAAM;AACvC,MAAI,UAAU,QAAW;AACvB,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,iBAAiB,OAAiC;AACzD,QAAM,eAA6B,CAAC;AACpC,MAAI,UAAsB,CAAC;AAC3B,MAAI,UAAU;AACd,aAAW,QAAQ,OAAO;AACxB,QAAI,QAAQ,GAAG,EAAE,GAAG,SAAS,eAAe,KAAK,SAAS,UAAU,SAAS;AAC3E,mBAAa,KAAK,OAAO;AACzB,gBAAU,CAAC,IAAI;AACf,gBAAU;AACV;AAAA,IACF;AACA,YAAQ,KAAK,IAAI;AACjB,QAAI,KAAK,SAAS,OAAQ,WAAU;AAAA,EACtC;AACA,MAAI,QAAQ,SAAS,KAAK,QAAQ,GAAG,EAAE,GAAG,SAAS,eAAe,SAAS;AACzE,iBAAa,KAAK,OAAO;AAAA,EAC3B;AACA,SAAO;AACT;AAEA,SAAS,kBAAkB,OAAuC;AAChE,MAAI,MAAM,QAAQ,KAAK,EAAG,QAAO;AACjC,MAAI,OAAO,UAAU,YAAY,UAAU,QAAQ,cAAc,OAAO;AACtE,UAAM,WAAY,MAAiC;AACnD,WAAO,MAAM,QAAQ,QAAQ,IAAI,WAAW;AAAA,EAC9C;AACA,SAAO;AACT;AAEA,SAAS,YAAY,SAA0B;AAC7C,MAAI,OAAO,YAAY,SAAU,QAAO;AACxC,MAAI,CAAC,MAAM,QAAQ,OAAO,EAAG,QAAO;AACpC,SAAO,QACJ;AAAA,IAAQ,CAAC,SACR,OAAO,SAAS,YAChB,SAAS,QACT,UAAU,QACV,KAAK,SAAS,UACd,UAAU,QACV,OAAO,KAAK,SAAS,WACjB,CAAC,KAAK,IAAI,IACV,CAAC;AAAA,EACP,EACC,KAAK,IAAI;AACd;","names":["epoch"]}