oh-my-knowledge 0.19.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/README.md +205 -8
  2. package/README.zh.md +200 -7
  3. package/dist/src/analysis/failure-clusterer.d.ts +96 -0
  4. package/dist/src/analysis/failure-clusterer.d.ts.map +1 -0
  5. package/dist/src/analysis/failure-clusterer.js +298 -0
  6. package/dist/src/analysis/failure-clusterer.js.map +1 -0
  7. package/dist/src/analysis/sample-diagnostics.d.ts +78 -0
  8. package/dist/src/analysis/sample-diagnostics.d.ts.map +1 -0
  9. package/dist/src/analysis/sample-diagnostics.js +259 -0
  10. package/dist/src/analysis/sample-diagnostics.js.map +1 -0
  11. package/dist/src/analysis/saturation.d.ts +85 -0
  12. package/dist/src/analysis/saturation.d.ts.map +1 -0
  13. package/dist/src/analysis/saturation.js +174 -0
  14. package/dist/src/analysis/saturation.js.map +1 -0
  15. package/dist/src/cli.js +751 -5
  16. package/dist/src/cli.js.map +1 -1
  17. package/dist/src/eval-core/bootstrap.d.ts +72 -0
  18. package/dist/src/eval-core/bootstrap.d.ts.map +1 -0
  19. package/dist/src/eval-core/bootstrap.js +174 -0
  20. package/dist/src/eval-core/bootstrap.js.map +1 -0
  21. package/dist/src/eval-core/evaluation-execution.d.ts +15 -1
  22. package/dist/src/eval-core/evaluation-execution.d.ts.map +1 -1
  23. package/dist/src/eval-core/evaluation-execution.js +37 -3
  24. package/dist/src/eval-core/evaluation-execution.js.map +1 -1
  25. package/dist/src/eval-core/evaluation-job.d.ts +8 -2
  26. package/dist/src/eval-core/evaluation-job.d.ts.map +1 -1
  27. package/dist/src/eval-core/evaluation-job.js +7 -1
  28. package/dist/src/eval-core/evaluation-job.js.map +1 -1
  29. package/dist/src/eval-core/evaluation-reporting.d.ts.map +1 -1
  30. package/dist/src/eval-core/evaluation-reporting.js +90 -0
  31. package/dist/src/eval-core/evaluation-reporting.js.map +1 -1
  32. package/dist/src/eval-core/schema.d.ts.map +1 -1
  33. package/dist/src/eval-core/schema.js +69 -0
  34. package/dist/src/eval-core/schema.js.map +1 -1
  35. package/dist/src/eval-core/verdict.d.ts +74 -0
  36. package/dist/src/eval-core/verdict.d.ts.map +1 -0
  37. package/dist/src/eval-core/verdict.js +283 -0
  38. package/dist/src/eval-core/verdict.js.map +1 -0
  39. package/dist/src/eval-workflows/each-evaluation-workflow.d.ts +10 -1
  40. package/dist/src/eval-workflows/each-evaluation-workflow.d.ts.map +1 -1
  41. package/dist/src/eval-workflows/each-evaluation-workflow.js +4 -1
  42. package/dist/src/eval-workflows/each-evaluation-workflow.js.map +1 -1
  43. package/dist/src/eval-workflows/evaluation-pipeline.d.ts +13 -1
  44. package/dist/src/eval-workflows/evaluation-pipeline.d.ts.map +1 -1
  45. package/dist/src/eval-workflows/evaluation-pipeline.js +41 -3
  46. package/dist/src/eval-workflows/evaluation-pipeline.js.map +1 -1
  47. package/dist/src/eval-workflows/run-evaluation.d.ts +17 -2
  48. package/dist/src/eval-workflows/run-evaluation.d.ts.map +1 -1
  49. package/dist/src/eval-workflows/run-evaluation.js +97 -5
  50. package/dist/src/eval-workflows/run-evaluation.js.map +1 -1
  51. package/dist/src/grading/assertions.d.ts +16 -0
  52. package/dist/src/grading/assertions.d.ts.map +1 -1
  53. package/dist/src/grading/assertions.js +385 -111
  54. package/dist/src/grading/assertions.js.map +1 -1
  55. package/dist/src/grading/debias-validate.d.ts +84 -0
  56. package/dist/src/grading/debias-validate.d.ts.map +1 -0
  57. package/dist/src/grading/debias-validate.js +173 -0
  58. package/dist/src/grading/debias-validate.js.map +1 -0
  59. package/dist/src/grading/gold-cli.d.ts +88 -0
  60. package/dist/src/grading/gold-cli.d.ts.map +1 -0
  61. package/dist/src/grading/gold-cli.js +251 -0
  62. package/dist/src/grading/gold-cli.js.map +1 -0
  63. package/dist/src/grading/gold-dataset.d.ts +73 -0
  64. package/dist/src/grading/gold-dataset.d.ts.map +1 -0
  65. package/dist/src/grading/gold-dataset.js +161 -0
  66. package/dist/src/grading/gold-dataset.js.map +1 -0
  67. package/dist/src/grading/human-gold.d.ts +102 -0
  68. package/dist/src/grading/human-gold.d.ts.map +1 -0
  69. package/dist/src/grading/human-gold.js +188 -0
  70. package/dist/src/grading/human-gold.js.map +1 -0
  71. package/dist/src/grading/index.d.ts +27 -2
  72. package/dist/src/grading/index.d.ts.map +1 -1
  73. package/dist/src/grading/index.js +36 -18
  74. package/dist/src/grading/index.js.map +1 -1
  75. package/dist/src/grading/judge.d.ts +65 -2
  76. package/dist/src/grading/judge.d.ts.map +1 -1
  77. package/dist/src/grading/judge.js +280 -23
  78. package/dist/src/grading/judge.js.map +1 -1
  79. package/dist/src/inputs/eval-config.js +19 -0
  80. package/dist/src/inputs/eval-config.js.map +1 -1
  81. package/dist/src/renderer/html-renderer.d.ts.map +1 -1
  82. package/dist/src/renderer/html-renderer.js +16 -4
  83. package/dist/src/renderer/html-renderer.js.map +1 -1
  84. package/dist/src/renderer/layout.d.ts.map +1 -1
  85. package/dist/src/renderer/layout.js +32 -4
  86. package/dist/src/renderer/layout.js.map +1 -1
  87. package/dist/src/renderer/summary.d.ts +28 -1
  88. package/dist/src/renderer/summary.d.ts.map +1 -1
  89. package/dist/src/renderer/summary.js +321 -7
  90. package/dist/src/renderer/summary.js.map +1 -1
  91. package/dist/src/renderer/table.d.ts.map +1 -1
  92. package/dist/src/renderer/table.js +63 -2
  93. package/dist/src/renderer/table.js.map +1 -1
  94. package/dist/src/types.d.ts +241 -0
  95. package/dist/src/types.d.ts.map +1 -1
  96. package/package.json +15 -4
@@ -3,13 +3,117 @@ import _Ajv from 'ajv';
3
3
  const Ajv = _Ajv.default ?? _Ajv;
4
4
  const ajv = new Ajv();
5
5
  const CUSTOM_ASSERTION_TIMEOUT_MS = 30_000;
6
- export const ASYNC_ASSERTION_TYPES = new Set(['semantic_similarity', 'custom']);
6
+ export const ASYNC_ASSERTION_TYPES = new Set([
7
+ 'semantic_similarity',
8
+ 'custom',
9
+ // v0.22 — RAG-specific judge metrics. All three go through the LLM-judge path
10
+ // and inherit the same length-debias instruction as the main rubric judge.
11
+ 'faithfulness',
12
+ 'answer_relevancy',
13
+ 'context_recall',
14
+ ]);
7
15
  function getErrorMessage(err) {
8
16
  return err instanceof Error ? err.message : String(err);
9
17
  }
10
18
  export function ratioToScore(ratio) {
11
19
  return Number((1 + ratio * 4).toFixed(2));
12
20
  }
21
+ // ===========================================================================
22
+ // v0.21 Phase 5b — deterministic similarity metrics
23
+ // ===========================================================================
24
+ /** Tokenizer: Latin words/numbers as units; CJK chars as single-char units. */
25
+ function tokenize(s) {
26
+ const m = s.toLowerCase().match(/[a-z0-9]+|[一-龥]/g);
27
+ return m ?? [];
28
+ }
29
+ function ngrams(tokens, n) {
30
+ if (n <= 0 || tokens.length < n)
31
+ return [];
32
+ const out = new Array(tokens.length - n + 1);
33
+ for (let i = 0; i <= tokens.length - n; i++) {
34
+ out[i] = tokens.slice(i, i + n).join(' ');
35
+ }
36
+ return out;
37
+ }
38
+ function clippedOverlap(candNgrams, refNgrams) {
39
+ if (candNgrams.length === 0 || refNgrams.length === 0)
40
+ return 0;
41
+ const candCount = new Map();
42
+ const refCount = new Map();
43
+ for (const g of candNgrams)
44
+ candCount.set(g, (candCount.get(g) ?? 0) + 1);
45
+ for (const g of refNgrams)
46
+ refCount.set(g, (refCount.get(g) ?? 0) + 1);
47
+ let total = 0;
48
+ for (const [g, cc] of candCount) {
49
+ total += Math.min(cc, refCount.get(g) ?? 0);
50
+ }
51
+ return total;
52
+ }
53
+ /**
54
+ * ROUGE-N (recall-oriented). N-gram overlap divided by the reference's
55
+ * n-gram count. Multi-set form per Lin 2004 — repeated n-grams are clipped
56
+ * to their reference count, so a candidate that repeats a single matching
57
+ * n-gram many times can't game the score above 1.
58
+ */
59
+ export function rougeN(candidate, reference, n) {
60
+ const cn = ngrams(tokenize(candidate), n);
61
+ const rn = ngrams(tokenize(reference), n);
62
+ if (rn.length === 0)
63
+ return 0;
64
+ return clippedOverlap(cn, rn) / rn.length;
65
+ }
66
+ /** Standard Levenshtein edit distance with O(min(m,n)) memory. */
67
+ export function levenshtein(a, b) {
68
+ if (a === b)
69
+ return 0;
70
+ if (a.length === 0)
71
+ return b.length;
72
+ if (b.length === 0)
73
+ return a.length;
74
+ // Ensure b is the shorter one for memory.
75
+ if (a.length < b.length)
76
+ [a, b] = [b, a];
77
+ const m = a.length, n = b.length;
78
+ let prev = new Array(n + 1);
79
+ let curr = new Array(n + 1);
80
+ for (let j = 0; j <= n; j++)
81
+ prev[j] = j;
82
+ for (let i = 1; i <= m; i++) {
83
+ curr[0] = i;
84
+ for (let j = 1; j <= n; j++) {
85
+ const cost = a[i - 1] === b[j - 1] ? 0 : 1;
86
+ curr[j] = Math.min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + cost);
87
+ }
88
+ [prev, curr] = [curr, prev];
89
+ }
90
+ return prev[n];
91
+ }
92
+ /**
93
+ * BLEU score (corpus-level form, single reference). Geometric mean of
94
+ * precision at n=1..maxN, multiplied by the brevity penalty for short
95
+ * candidates. Unsmoothed — returns 0 if any n-gram precision is 0, which
96
+ * is a known sharpness of the original BLEU; threshold accordingly.
97
+ */
98
+ export function bleu(candidate, reference, maxN = 4) {
99
+ const ct = tokenize(candidate);
100
+ const rt = tokenize(reference);
101
+ if (ct.length === 0 || rt.length === 0)
102
+ return 0;
103
+ let logSum = 0;
104
+ for (let n = 1; n <= maxN; n++) {
105
+ const cn = ngrams(ct, n);
106
+ const rn = ngrams(rt, n);
107
+ if (cn.length === 0)
108
+ return 0;
109
+ const overlap = clippedOverlap(cn, rn);
110
+ if (overlap === 0)
111
+ return 0;
112
+ logSum += Math.log(overlap / cn.length);
113
+ }
114
+ const bp = ct.length >= rt.length ? 1 : Math.exp(1 - rt.length / ct.length);
115
+ return bp * Math.exp(logSum / maxN);
116
+ }
13
117
  export function validateJsonSchema(data, schema) {
14
118
  if (!schema || typeof schema !== 'object')
15
119
  return true;
@@ -21,122 +125,131 @@ export function validateJsonSchema(data, schema) {
21
125
  return false;
22
126
  }
23
127
  }
128
+ /**
129
+ * Evaluate a single assertion against the output. Returns the raw pass/fail
130
+ * BEFORE applying `not`. Pulled out of `runAssertions` so `assert-set` can
131
+ * recurse on children without rebuilding the context.
132
+ *
133
+ * v0.21 Phase 5a — added `not: true` (universal negation) and `assert-set`
134
+ * (any/all combinator). Legacy `not_contains` / `not_equals` / etc. still
135
+ * work; their negation is hard-coded inline as before.
136
+ */
137
+ function evalAssertion(output, assertion, ctx) {
138
+ const { outputLower, toolCalls, toolNames } = ctx;
139
+ if (assertion.type === 'assert-set') {
140
+ const children = assertion.children ?? [];
141
+ if (children.length === 0)
142
+ return false;
143
+ const mode = assertion.mode ?? 'all';
144
+ const childPasses = children.map((c) => {
145
+ const raw = evalAssertion(output, c, ctx);
146
+ return c.not ? !raw : raw;
147
+ });
148
+ return mode === 'any' ? childPasses.some(Boolean) : childPasses.every(Boolean);
149
+ }
150
+ switch (assertion.type) {
151
+ case 'contains':
152
+ return outputLower.includes(String(assertion.value).toLowerCase());
153
+ case 'not_contains':
154
+ return !outputLower.includes(String(assertion.value).toLowerCase());
155
+ case 'regex': {
156
+ const flags = assertion.flags || 'i';
157
+ const re = new RegExp(assertion.pattern, flags);
158
+ return re.test(output);
159
+ }
160
+ case 'min_length':
161
+ return output.length >= assertion.value;
162
+ case 'max_length':
163
+ return output.length <= assertion.value;
164
+ case 'json_valid':
165
+ try {
166
+ JSON.parse(output);
167
+ return true;
168
+ }
169
+ catch {
170
+ return false;
171
+ }
172
+ case 'json_schema':
173
+ try {
174
+ const data = JSON.parse(output);
175
+ return validateJsonSchema(data, assertion.schema);
176
+ }
177
+ catch {
178
+ return false;
179
+ }
180
+ case 'starts_with':
181
+ return outputLower.startsWith(String(assertion.value).toLowerCase());
182
+ case 'ends_with':
183
+ return outputLower.endsWith(String(assertion.value).toLowerCase());
184
+ case 'equals':
185
+ return output.trim() === String(assertion.value).trim();
186
+ case 'not_equals':
187
+ return output.trim() !== String(assertion.value).trim();
188
+ case 'word_count_min':
189
+ return output.split(/\s+/).filter(Boolean).length >= assertion.value;
190
+ case 'word_count_max':
191
+ return output.split(/\s+/).filter(Boolean).length <= assertion.value;
192
+ case 'contains_all':
193
+ return (assertion.values || []).every((v) => outputLower.includes(String(v).toLowerCase()));
194
+ case 'contains_any':
195
+ return (assertion.values || []).some((v) => outputLower.includes(String(v).toLowerCase()));
196
+ case 'cost_max':
197
+ return (ctx.costUSD ?? Infinity) <= assertion.value;
198
+ case 'latency_max':
199
+ return (ctx.durationMs ?? Infinity) <= assertion.value;
200
+ case 'turns_max':
201
+ return (ctx.numTurns ?? Infinity) <= assertion.value;
202
+ case 'turns_min':
203
+ return (ctx.numTurns ?? 0) >= assertion.value;
204
+ case 'tools_called':
205
+ return (assertion.values || []).every((v) => toolNames.includes(String(v).toLowerCase()));
206
+ case 'tools_not_called':
207
+ return (assertion.values || []).every((v) => !toolNames.includes(String(v).toLowerCase()));
208
+ case 'tools_count_max':
209
+ return toolCalls.length <= assertion.value;
210
+ case 'tools_count_min':
211
+ return toolCalls.length >= assertion.value;
212
+ case 'tool_output_contains': {
213
+ const sep = String(assertion.value).indexOf(':');
214
+ if (sep <= 0)
215
+ return false;
216
+ const targetTool = String(assertion.value).slice(0, sep).toLowerCase();
217
+ const expected = String(assertion.value).slice(sep + 1).toLowerCase();
218
+ return toolCalls.some((tc) => tc.tool.toLowerCase() === targetTool &&
219
+ String(tc.output || '').toLowerCase().includes(expected));
220
+ }
221
+ case 'tool_input_contains': {
222
+ const sep = String(assertion.value).indexOf(':');
223
+ if (sep <= 0)
224
+ return false;
225
+ const targetTool = String(assertion.value).slice(0, sep).toLowerCase();
226
+ const expected = String(assertion.value).slice(sep + 1).toLowerCase();
227
+ return toolCalls.some((tc) => tc.tool.toLowerCase() === targetTool &&
228
+ JSON.stringify(tc.input || '').toLowerCase().includes(expected));
229
+ }
230
+ case 'rouge_n_min':
231
+ return rougeN(output, String(assertion.reference ?? assertion.value ?? ''), assertion.n ?? 1)
232
+ >= (assertion.threshold ?? 0.5);
233
+ case 'levenshtein_max':
234
+ return levenshtein(output, String(assertion.reference ?? assertion.value ?? ''))
235
+ <= (assertion.value ?? Infinity);
236
+ case 'bleu_min':
237
+ return bleu(output, String(assertion.reference ?? assertion.value ?? ''))
238
+ >= (assertion.threshold ?? 0.5);
239
+ default:
240
+ return false;
241
+ }
242
+ }
24
243
  export function runAssertions(output, assertions, context = {}) {
25
244
  const outputLower = output.toLowerCase();
26
- const details = [];
27
245
  const toolCalls = context.toolCalls || [];
28
246
  const toolNames = toolCalls.map((tc) => tc.tool.toLowerCase());
247
+ const ctx = { outputLower, toolCalls, toolNames, costUSD: context.costUSD, durationMs: context.durationMs, numTurns: context.numTurns };
248
+ const details = [];
29
249
  for (const assertion of assertions) {
30
250
  const weight = assertion.weight ?? 1;
31
- let passed = false;
32
- switch (assertion.type) {
33
- case 'contains':
34
- passed = outputLower.includes(String(assertion.value).toLowerCase());
35
- break;
36
- case 'not_contains':
37
- passed = !outputLower.includes(String(assertion.value).toLowerCase());
38
- break;
39
- case 'regex': {
40
- const flags = assertion.flags || 'i';
41
- const re = new RegExp(assertion.pattern, flags);
42
- passed = re.test(output);
43
- break;
44
- }
45
- case 'min_length':
46
- passed = output.length >= assertion.value;
47
- break;
48
- case 'max_length':
49
- passed = output.length <= assertion.value;
50
- break;
51
- case 'json_valid':
52
- try {
53
- JSON.parse(output);
54
- passed = true;
55
- }
56
- catch {
57
- passed = false;
58
- }
59
- break;
60
- case 'json_schema':
61
- try {
62
- const data = JSON.parse(output);
63
- passed = validateJsonSchema(data, assertion.schema);
64
- }
65
- catch {
66
- passed = false;
67
- }
68
- break;
69
- case 'starts_with':
70
- passed = outputLower.startsWith(String(assertion.value).toLowerCase());
71
- break;
72
- case 'ends_with':
73
- passed = outputLower.endsWith(String(assertion.value).toLowerCase());
74
- break;
75
- case 'equals':
76
- passed = output.trim() === String(assertion.value).trim();
77
- break;
78
- case 'not_equals':
79
- passed = output.trim() !== String(assertion.value).trim();
80
- break;
81
- case 'word_count_min':
82
- passed = output.split(/\s+/).filter(Boolean).length >= assertion.value;
83
- break;
84
- case 'word_count_max':
85
- passed = output.split(/\s+/).filter(Boolean).length <= assertion.value;
86
- break;
87
- case 'contains_all':
88
- passed = (assertion.values || []).every((v) => outputLower.includes(String(v).toLowerCase()));
89
- break;
90
- case 'contains_any':
91
- passed = (assertion.values || []).some((v) => outputLower.includes(String(v).toLowerCase()));
92
- break;
93
- case 'cost_max':
94
- passed = (context.costUSD ?? Infinity) <= assertion.value;
95
- break;
96
- case 'latency_max':
97
- passed = (context.durationMs ?? Infinity) <= assertion.value;
98
- break;
99
- case 'turns_max':
100
- passed = (context.numTurns ?? Infinity) <= assertion.value;
101
- break;
102
- case 'turns_min':
103
- passed = (context.numTurns ?? 0) >= assertion.value;
104
- break;
105
- case 'tools_called':
106
- passed = (assertion.values || []).every((v) => toolNames.includes(String(v).toLowerCase()));
107
- break;
108
- case 'tools_not_called':
109
- passed = (assertion.values || []).every((v) => !toolNames.includes(String(v).toLowerCase()));
110
- break;
111
- case 'tools_count_max':
112
- passed = toolCalls.length <= assertion.value;
113
- break;
114
- case 'tools_count_min':
115
- passed = toolCalls.length >= assertion.value;
116
- break;
117
- case 'tool_output_contains': {
118
- const sep = String(assertion.value).indexOf(':');
119
- if (sep > 0) {
120
- const targetTool = String(assertion.value).slice(0, sep).toLowerCase();
121
- const expected = String(assertion.value).slice(sep + 1).toLowerCase();
122
- passed = toolCalls.some((tc) => tc.tool.toLowerCase() === targetTool &&
123
- String(tc.output || '').toLowerCase().includes(expected));
124
- }
125
- break;
126
- }
127
- case 'tool_input_contains': {
128
- const sep = String(assertion.value).indexOf(':');
129
- if (sep > 0) {
130
- const targetTool = String(assertion.value).slice(0, sep).toLowerCase();
131
- const expected = String(assertion.value).slice(sep + 1).toLowerCase();
132
- passed = toolCalls.some((tc) => tc.tool.toLowerCase() === targetTool &&
133
- JSON.stringify(tc.input || '').toLowerCase().includes(expected));
134
- }
135
- break;
136
- }
137
- default:
138
- passed = false;
139
- }
251
+ const raw = evalAssertion(output, assertion, ctx);
252
+ const passed = assertion.not ? !raw : raw;
140
253
  details.push({
141
254
  type: assertion.type,
142
255
  value: assertion.value ?? assertion.pattern ?? assertion.values?.join(', ') ?? '',
@@ -203,6 +316,14 @@ export async function runAsyncAssertions(output, assertions, { executor, judgeMo
203
316
  }
204
317
  }
205
318
  }
319
+ else if (assertion.type === 'faithfulness' ||
320
+ assertion.type === 'answer_relevancy' ||
321
+ assertion.type === 'context_recall') {
322
+ const ragResult = await runRagJudge(assertion, output, sample, executor, judgeModel);
323
+ asyncCostUSD += ragResult.costUSD;
324
+ passed = ragResult.passed;
325
+ message = ragResult.message;
326
+ }
206
327
  else if (assertion.type === 'custom') {
207
328
  try {
208
329
  const fnPath = resolve(samplesDir, assertion.fn);
@@ -236,4 +357,157 @@ export async function runAsyncAssertions(output, assertions, { executor, judgeMo
236
357
  const ratio = totalWeight > 0 ? passedWeight / totalWeight : 0;
237
358
  return { passed: passedCount, total: details.length, score: ratioToScore(ratio), details, judgeCostUSD: asyncCostUSD };
238
359
  }
360
+ // ===========================================================================
361
+ // v0.22 — RAG-specific judge metrics
362
+ // ===========================================================================
363
+ //
364
+ // Three metrics, all running through the LLM judge:
365
+ //
366
+ // - faithfulness: does the output's content stay grounded in the
367
+ // reference context? Anti-hallucination check.
368
+ // - answer_relevancy: does the output directly answer the user's question?
369
+ // Catches verbose dodges and topic drift.
370
+ // - context_recall: are the key facts from the gold context actually
371
+ // used in the output? Catches retrieved-but-ignored
372
+ // context (a common RAG bug).
373
+ //
374
+ // Implementation notes:
375
+ //
376
+ // 1. Each prompt is a SINGLE-CALL judge (1-5 score) rather than the multi-step
377
+ // statement-decomposition that RAGAS uses. This is honest tradeoff: simpler,
378
+ // faster, less rigorous than RAGAS but consistent with omk's other LLM-judge
379
+ // assertions. Users who need RAGAS-grade decomposition can drop down to a
380
+ // custom assertion.
381
+ // 2. The prompt includes the SAME length-debias paragraph as the main judge
382
+ // prompt (v3-cot-length) — output verbosity is not a quality signal here
383
+ // either. This is the "auto-inherit length-debias" claim from the v0.22 plan.
384
+ // 3. Reference resolution:
385
+ // faithfulness: sample.context (or assertion.reference override)
386
+ // context_recall: assertion.reference (or sample.context fallback)
387
+ // answer_relevancy: sample.prompt — no reference needed
388
+ // 4. Threshold defaults to 3 (same as semantic_similarity). User can override.
389
+ const RAG_LENGTH_DEBIAS = [
390
+ '## 重要:长度不是质量信号',
391
+ '评分时聚焦内容实质,不要因输出更长就给更高分。',
392
+ '简洁正确的回答与冗长正确的回答应得相同分数。',
393
+ ].join('\n');
394
+ async function runRagJudge(assertion, output, sample, executor, judgeModel) {
395
+ const threshold = assertion.threshold ?? 3;
396
+ let prompt;
397
+ let system;
398
+ if (assertion.type === 'faithfulness') {
399
+ const context = assertion.reference || sample.context || '';
400
+ if (!context) {
401
+ return { passed: false, message: 'faithfulness: 缺少 sample.context 或 assertion.reference', costUSD: 0 };
402
+ }
403
+ system = '你是 RAG 评审员,专注判断输出是否被参考 context 支持。只返回 JSON。';
404
+ prompt = [
405
+ '请判断"待评估输出"中的事实性陈述是否被"参考 context"支持。',
406
+ '',
407
+ '## 参考 context',
408
+ context,
409
+ '',
410
+ '## 待评估输出',
411
+ output,
412
+ '',
413
+ RAG_LENGTH_DEBIAS,
414
+ '',
415
+ '## 评分流程',
416
+ '1. 列出待评估输出中所有事实性陈述',
417
+ '2. 逐条判断是否能在 context 中找到支持',
418
+ '3. 给出 1-5 分:',
419
+ ' 5 = 全部陈述都有 context 支持,无编造',
420
+ ' 4 = 多数有支持,有 1-2 处不重要的编造',
421
+ ' 3 = 一半有支持',
422
+ ' 2 = 多数无支持',
423
+ ' 1 = 完全编造或与 context 矛盾',
424
+ '',
425
+ '请返回 JSON(不要 markdown 代码块):',
426
+ '{"score": <1-5的整数>, "reason": "<简短理由>"}',
427
+ ].join('\n');
428
+ }
429
+ else if (assertion.type === 'answer_relevancy') {
430
+ system = '你是答题切题度评审员。只返回 JSON。';
431
+ prompt = [
432
+ '请判断"AI 输出"是否直接、切题地回答了"用户问题"。',
433
+ '',
434
+ '## 用户问题',
435
+ sample.prompt,
436
+ '',
437
+ '## AI 输出',
438
+ output,
439
+ '',
440
+ RAG_LENGTH_DEBIAS,
441
+ '',
442
+ '## 评分',
443
+ '5 = 完整切题回答,无冗余无遗漏',
444
+ '4 = 切题但有少量冗余或小遗漏',
445
+ '3 = 部分切题,部分跑题或避而不答',
446
+ '2 = 大部分跑题',
447
+ '1 = 完全跑题或拒答',
448
+ '',
449
+ '请返回 JSON(不要 markdown 代码块):',
450
+ '{"score": <1-5的整数>, "reason": "<简短理由>"}',
451
+ ].join('\n');
452
+ }
453
+ else {
454
+ // context_recall
455
+ const reference = assertion.reference || sample.context || '';
456
+ if (!reference) {
457
+ return { passed: false, message: 'context_recall: 缺少 assertion.reference 或 sample.context', costUSD: 0 };
458
+ }
459
+ system = '你是 context 覆盖率评审员。只返回 JSON。';
460
+ prompt = [
461
+ '请判断"参考 gold"中的关键事实在"AI 输出"中被覆盖的程度。',
462
+ '',
463
+ '## 参考 gold',
464
+ reference,
465
+ '',
466
+ '## AI 输出',
467
+ output,
468
+ '',
469
+ RAG_LENGTH_DEBIAS,
470
+ '',
471
+ '## 评分流程',
472
+ '1. 列出参考中的关键事实(忽略修饰性内容)',
473
+ '2. 检查每条是否在输出中被提及/使用',
474
+ '3. 给出 1-5 分:',
475
+ ' 5 = 全部关键事实被覆盖',
476
+ ' 4 = 大部分覆盖,缺 1-2 条次要事实',
477
+ ' 3 = 一半覆盖',
478
+ ' 2 = 仅覆盖少量',
479
+ ' 1 = 完全未覆盖',
480
+ '',
481
+ '请返回 JSON(不要 markdown 代码块):',
482
+ '{"score": <1-5的整数>, "reason": "<简短理由>"}',
483
+ ].join('\n');
484
+ }
485
+ const result = await executor({ model: judgeModel, system, prompt });
486
+ if (!result.ok) {
487
+ return {
488
+ passed: false,
489
+ message: `${assertion.type} judge error: ${result.error}`,
490
+ costUSD: result.costUSD || 0,
491
+ };
492
+ }
493
+ try {
494
+ const text = result.output.trim();
495
+ const jsonMatch = text.match(/\{[\s\S]*\}/);
496
+ if (!jsonMatch) {
497
+ process.stderr.write(`[omk] ${assertion.type} judge returned non-JSON: ${text.slice(0, 100)}\n`);
498
+ return { passed: false, message: 'judge returned non-JSON', costUSD: result.costUSD || 0 };
499
+ }
500
+ const parsed = JSON.parse(jsonMatch[0]);
501
+ const score = Number(parsed.score) || 0;
502
+ return {
503
+ passed: score >= threshold,
504
+ message: parsed.reason ? String(parsed.reason) : '',
505
+ costUSD: result.costUSD || 0,
506
+ };
507
+ }
508
+ catch (parseErr) {
509
+ process.stderr.write(`[omk] ${assertion.type} judge parse error: ${getErrorMessage(parseErr)}\n`);
510
+ return { passed: false, message: 'failed to parse judge response', costUSD: result.costUSD || 0 };
511
+ }
512
+ }
239
513
  //# sourceMappingURL=assertions.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"assertions.js","sourceRoot":"","sources":["../../../src/grading/assertions.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,IAAI,MAAM,KAAK,CAAC;AAGvB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC;AACjC,MAAM,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;AACtB,MAAM,2BAA2B,GAAG,MAAM,CAAC;AAE3C,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAC,CAAC,qBAAqB,EAAE,QAAQ,CAAC,CAAC,CAAC;AA2BhF,SAAS,eAAe,CAAC,GAAY;IACnC,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAa;IACxC,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,IAAa,EAAE,MAA+B;IAC/E,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IACvD,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACrC,OAAO,QAAQ,CAAC,IAAI,CAAY,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAc,EAAE,UAAuB,EAAE,UAAoG,EAAE;IAC3K,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;IACzC,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC;IAC1C,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;IAE/D,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,IAAI,CAAC,CAAC;QACrC,IAAI,MAAM,GAAG,KAAK,CAAC;QAEnB,QAAQ,SAAS,CAAC,IAAI,EAAE,CAAC;YACvB,KAAK,UAAU;gBACb,MAAM,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;gBACrE,MAAM;YACR,KAAK,cAAc;gBACjB,MAAM,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;gBACtE,MAAM;YACR,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,IAAI,GAAG,CAAC;gBACrC,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,SAAS,CAAC,OAAQ,EAAE,KAAK,CAAC,CAAC;gBACjD,MAAM,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACzB,MAAM;YACR,CAAC;YACD,KAAK,YAAY;gBACf,MAAM,GAAG,MAAM,CAAC,MAAM,IAAK,SAAS,CAAC,KAAgB,CAAC;gBACtD,MAAM;YACR,KAAK,YAAY;gBACf,MAAM,GAAG,MAAM,CAAC,MAAM,IAAK,SAAS,CAAC,KAAgB,CAAC;gBACtD,MAAM;YACR,KAAK,YAAY;gBACf,IAAI,CAAC;oBAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;oBAAC,MAAM,GAAG,IAAI,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC;oBAAC,MAAM,GAAG,KAAK,CAAC;gBAAC,CAAC;gBACpE,MAAM;YACR,KAAK,aAAa;gBAChB,IAAI,CAAC;oBACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;oBAChC,MAAM,GAAG,kBAAkB,CAAC,IAAI,EAAE,SAAS,CAAC,MAAO,CAAC,CAAC;gBACvD,CAAC;gBAAC,MAAM,CAAC;oBAAC,MAAM,GAAG,KAAK,CAAC;gBAAC,CAAC;gBAC3B,MAAM;YACR,KAAK,aAAa;gBAChB,MAAM,GAAG,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;gBACvE,MAAM;YACR,KAAK,WAAW;gBACd,MAAM,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;gBACrE,MAAM;YACR,KAAK,QAAQ;gBACX,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC1D,MAAM;YACR,KAAK,YAAY;gBACf,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC1D,MAAM;YACR,KAAK,gBAAgB;gBACnB,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,IAAK,SAAS,CAAC,KAAgB,CAAC;gBACnF,MAAM;YACR,KAAK,gBAAgB;gBACnB,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,IAAK,SAAS,CAAC,KAAgB,CAAC;gBACnF,MAAM;YACR,KAAK,cAAc;gBACjB,MAAM,GAAG,CAAC,SAAS,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;gBAC9F,MAAM;YACR,KAAK,cAAc;gBACjB,MAAM,GAAG,CAAC,SAAS,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;gBAC7F,MAAM;YACR,KAAK,UAAU;gBACb,MAAM,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAC,IAAK,SAAS,CAAC,KAAgB,CAAC;gBACtE,MAAM;YACR,KAAK,aAAa;gBAChB,MAAM,GAAG,CAAC,OAAO,CAAC,UAAU,IAAI,QAAQ,CAAC,IAAK,SAAS,CAAC,KAAgB,CAAC;gBACzE,MAAM;YACR,KAAK,WAAW;gBACd,MAAM,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,QAAQ,CAAC,IAAK,SAAS,CAAC,KAAgB,CAAC;gBACvE,MAAM;YACR,KAAK,WAAW;gBACd,MAAM,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,IAAK,SAAS,CAAC,KAAgB,CAAC;gBAChE,MAAM;YACR,KAAK,cAAc;gBACjB,MAAM,GAAG,CAAC,SAAS,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;gBAC5F,MAAM;YACR,KAAK,kBAAkB;gBACrB,MAAM,GAAG,CAAC,SAAS,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;gBAC7F,MAAM;YACR,KAAK,iBAAiB;gBACpB,MAAM,GAAG,SAAS,CAAC,MAAM,IAAK,SAAS,CAAC,KAAgB,CAAC;gBACzD,MAAM;YACR,KAAK,iBAAiB;gBACpB,MAAM,GAAG,SAAS,CAAC,MAAM,IAAK,SAAS,CAAC,KAAgB,CAAC;gBACzD,MAAM;YACR,KAAK,sBAAsB,CAAC,CAAC,CAAC;gBAC5B,MAAM,GAAG,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACjD,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;oBACZ,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;oBACvE,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;oBACtE,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAC7B,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,UAAU;wBACpC,MAAM,CAAC,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CACzD,CAAC;gBACJ,CAAC;gBACD,MAAM;YACR,CAAC;YACD,KAAK,qBAAqB,CAAC,CAAC,CAAC;gBAC3B,MAAM,GAAG,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACjD,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;oBACZ,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;oBACvE,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;oBACtE,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAC7B,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,UAAU;wBACpC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAChE,CAAC;gBACJ,CAAC;gBACD,MAAM;YACR,CAAC;YACD;gBACE,MAAM,GAAG,KAAK,CAAC;QACnB,CAAC;QAED,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,SAAS,CAAC,OAAO,IAAI,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YACjF,MAAM;YACN,MAAM;SACP,CAAC,CAAC;IACL,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC9D,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACvF,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC3D,MAAM,KAAK,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IAE/D,OAAO;QACL,MAAM,EAAE,WAAW;QACnB,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC;QAC1B,OAAO;KACR,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,MAAc,EAAE,UAAuB,EAAE,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAyB;IACnJ,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,IAAI,CAAC,CAAC;QACrC,IAAI,MAAM,GAAG,KAAK,CAAC;QACnB,IAAI,OAAO,GAAG,EAAE,CAAC;QAEjB,IAAI,SAAS,CAAC,IAAI,KAAK,qBAAqB,EAAE,CAAC;YAC7C,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,IAAI,EAAE,CAAC;YAC5C,MAAM,WAAW,GAAG;gBAClB,kBAAkB;gBAClB,EAAE;gBACF,SAAS;gBACT,SAAS;gBACT,EAAE;gBACF,UAAU;gBACV,MAAM;gBACN,EAAE;gBACF,gCAAgC;gBAChC,yCAAyC;gBACzC,EAAE;gBACF,2CAA2C;aAC5C,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAEb,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC;gBAC5B,KAAK,EAAE,UAAU;gBACjB,MAAM,EAAE,6BAA6B;gBACrC,MAAM,EAAE,WAAW;aACpB,CAAC,CAAC;YAEH,YAAY,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;YACpC,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;gBACd,IAAI,CAAC;oBACH,MAAM,SAAS,GAAG,MAAM,CAAC,MAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;oBAC7D,IAAI,SAAS,EAAE,CAAC;wBACd,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAkB,CAAC;wBACzD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;wBACxC,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,IAAI,CAAC,CAAC;wBAC3C,MAAM,GAAG,KAAK,IAAI,SAAS,CAAC;wBAC5B,OAAO,GAAG,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;oBAChC,CAAC;yBAAM,CAAC;wBACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sDAAsD,MAAM,CAAC,MAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;oBAC/G,CAAC;gBACH,CAAC;gBAAC,OAAO,QAAiB,EAAE,CAAC;oBAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACtG,CAAC;YACH,CAAC;QACH,CAAC;aAAM,IAAI,SAAS,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,SAAS,CAAC,EAAG,CAAC,CAAC;gBAClD,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,MAAM,CAA0B,CAAC;gBAC1D,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,KAAK,CAAC;gBACpC,IAAI,CAAC,EAAE;oBAAE,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;gBACjF,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAwB;oBACvD,EAAE,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;oBACjC,IAAI,OAAO,CAAwB,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,+BAA+B,2BAA2B,GAAG,IAAI,IAAI,CAAC,CAAC,EAAE,2BAA2B,CAAC,CAAC;iBAC3L,CAAC,CAAC;gBACH,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC9B,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;YACjC,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACtB,MAAM,GAAG,KAAK,CAAC;gBACf,OAAO,GAAG,2BAA2B,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9D,CAAC;QACH,CAAC;QAED,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,KAAK,EAAE,SAAS,CAAC,SAAS,IAAI,SAAS,CAAC,EAAE,IAAI,EAAE;YAChD,MAAM;YACN,MAAM;YACN,GAAG,CAAC,OAAO,IAAI,EAAE,OAAO,EAAE,CAAC;SAC5B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC9D,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACvF,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC3D,MAAM,KAAK,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IAE/D,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,CAAC;AACzH,CAAC"}
1
+ {"version":3,"file":"assertions.js","sourceRoot":"","sources":["../../../src/grading/assertions.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,IAAI,MAAM,KAAK,CAAC;AAGvB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC;AACjC,MAAM,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;AACtB,MAAM,2BAA2B,GAAG,MAAM,CAAC;AAE3C,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAC;IAC3C,qBAAqB;IACrB,QAAQ;IACR,8EAA8E;IAC9E,2EAA2E;IAC3E,cAAc;IACd,kBAAkB;IAClB,gBAAgB;CACjB,CAAC,CAAC;AA2BH,SAAS,eAAe,CAAC,GAAY;IACnC,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAa;IACxC,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5C,CAAC;AAED,8EAA8E;AAC9E,oDAAoD;AACpD,8EAA8E;AAE9E,+EAA+E;AAC/E,SAAS,QAAQ,CAAC,CAAS;IACzB,MAAM,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACpD,OAAO,CAAC,IAAI,EAAE,CAAC;AACjB,CAAC;AAED,SAAS,MAAM,CAAC,MAAgB,EAAE,CAAS;IACzC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IAC3C,MAAM,GAAG,GAAa,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IACvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,cAAc,CAAC,UAAoB,EAAE,SAAmB;IAC/D,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAChE,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC5C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,UAAU;QAAE,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1E,KAAK,MAAM,CAAC,IAAI,SAAS;QAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACvE,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QAChC,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,MAAM,CAAC,SAAiB,EAAE,SAAiB,EAAE,CAAS;IACpE,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1C,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1C,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC9B,OAAO,cAAc,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;AAC5C,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,WAAW,CAAC,CAAS,EAAE,CAAS;IAC9C,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtB,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC,MAAM,CAAC;IACpC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC,MAAM,CAAC;IACpC,0CAA0C;IAC1C,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM;QAAE,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,IAAI,IAAI,GAAG,IAAI,KAAK,CAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IACpC,IAAI,IAAI,GAAG,IAAI,KAAK,CAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;QAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3C,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;QACvE,CAAC;QACD,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC9B,CAAC;IACD,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,IAAI,CAAC,SAAiB,EAAE,SAAiB,EAAE,IAAI,GAAG,CAAC;IACjE,MAAM,EAAE,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;IAC/B,MAAM,EAAE,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;IAC/B,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEjD,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACzB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACzB,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAC9B,MAAM,OAAO,GAAG,cAAc,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QACvC,IAAI,OAAO,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAC5B,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC;IAC5E,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,IAAa,EAAE,MAA+B;IAC/E,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IACvD,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACrC,OAAO,QAAQ,CAAC,IAAI,CAAY,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,aAAa,CACpB,MAAc,EACd,SAAoB,EACpB,GAAsI;IAEtI,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,GAAG,CAAC;IAElD,IAAI,SAAS,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;QACpC,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,IAAI,EAAE,CAAC;QAC1C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACxC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,IAAI,KAAK,CAAC;QACrC,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACrC,MAAM,GAAG,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;YAC1C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAC5B,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACjF,CAAC;IAED,QAAQ,SAAS,CAAC,IAAI,EAAE,CAAC;QACvB,KAAK,UAAU;YACb,OAAO,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QACrE,KAAK,cAAc;YACjB,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QACtE,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,IAAI,GAAG,CAAC;YACrC,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,SAAS,CAAC,OAAQ,EAAE,KAAK,CAAC,CAAC;YACjD,OAAO,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzB,CAAC;QACD,KAAK,YAAY;YACf,OAAO,MAAM,CAAC,MAAM,IAAK,SAAS,CAAC,KAAgB,CAAC;QACtD,KAAK,YAAY;YACf,OAAO,MAAM,CAAC,MAAM,IAAK,SAAS,CAAC,KAAgB,CAAC;QACtD,KAAK,YAAY;YACf,IAAI,CAAC;gBAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAAC,OAAO,IAAI,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC;gBAAC,OAAO,KAAK,CAAC;YAAC,CAAC;QAClE,KAAK,aAAa;YAChB,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAChC,OAAO,kBAAkB,CAAC,IAAI,EAAE,SAAS,CAAC,MAAO,CAAC,CAAC;YACrD,CAAC;YAAC,MAAM,CAAC;gBAAC,OAAO,KAAK,CAAC;YAAC,CAAC;QAC3B,KAAK,aAAa;YAChB,OAAO,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QACvE,KAAK,WAAW;YACd,OAAO,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QACrE,KAAK,QAAQ;YACX,OAAO,MAAM,CAAC,IAAI,EAAE,KAAK,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1D,KAAK,YAAY;YACf,OAAO,MAAM,CAAC,IAAI,EAAE,KAAK,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1D,KAAK,gBAAgB;YACnB,OAAO,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,IAAK,SAAS,CAAC,KAAgB,CAAC;QACnF,KAAK,gBAAgB;YACnB,OAAO,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,IAAK,SAAS,CAAC,KAAgB,CAAC;QACnF,KAAK,cAAc;YACjB,OAAO,CAAC,SAAS,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAC9F,KAAK,cAAc;YACjB,OAAO,CAAC,SAAS,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAC7F,KAAK,UAAU;YACb,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,QAAQ,CAAC,IAAK,SAAS,CAAC,KAAgB,CAAC;QAClE,KAAK,aAAa;YAChB,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,QAAQ,CAAC,IAAK,SAAS,CAAC,KAAgB,CAAC;QACrE,KAAK,WAAW;YACd,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,IAAK,SAAS,CAAC,KAAgB,CAAC;QACnE,KAAK,WAAW;YACd,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC,IAAK,SAAS,CAAC,KAAgB,CAAC;QAC5D,KAAK,cAAc;YACjB,OAAO,CAAC,SAAS,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAC5F,KAAK,kBAAkB;YACrB,OAAO,CAAC,SAAS,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAC7F,KAAK,iBAAiB;YACpB,OAAO,SAAS,CAAC,MAAM,IAAK,SAAS,CAAC,KAAgB,CAAC;QACzD,KAAK,iBAAiB;YACpB,OAAO,SAAS,CAAC,MAAM,IAAK,SAAS,CAAC,KAAgB,CAAC;QACzD,KAAK,sBAAsB,CAAC,CAAC,CAAC;YAC5B,MAAM,GAAG,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACjD,IAAI,GAAG,IAAI,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC3B,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;YACvE,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACtE,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAC3B,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,UAAU;gBACpC,MAAM,CAAC,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CACzD,CAAC;QACJ,CAAC;QACD,KAAK,qBAAqB,CAAC,CAAC,CAAC;YAC3B,MAAM,GAAG,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACjD,IAAI,GAAG,IAAI,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC3B,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;YACvE,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACtE,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAC3B,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,UAAU;gBACpC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAChE,CAAC;QACJ,CAAC;QACD,KAAK,aAAa;YAChB,OAAO,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,SAAS,IAAI,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC;mBACxF,CAAC,SAAS,CAAC,SAAS,IAAI,GAAG,CAAC,CAAC;QACpC,KAAK,iBAAiB;YACpB,OAAO,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,SAAS,IAAI,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;mBAC3E,CAAC,SAAS,CAAC,KAAe,IAAI,QAAQ,CAAC,CAAC;QAC/C,KAAK,UAAU;YACb,OAAO,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,SAAS,IAAI,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;mBACpE,CAAC,SAAS,CAAC,SAAS,IAAI,GAAG,CAAC,CAAC;QACpC;YACE,OAAO,KAAK,CAAC;IACjB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,MAAc,EACd,UAAuB,EACvB,UAAoG,EAAE;IAEtG,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC;IAC1C,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;IAC/D,MAAM,GAAG,GAAG,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;IAExI,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,IAAI,CAAC,CAAC;QACrC,MAAM,GAAG,GAAG,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAC1C,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,SAAS,CAAC,OAAO,IAAI,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YACjF,MAAM;YACN,MAAM;SACP,CAAC,CAAC;IACL,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC9D,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACvF,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC3D,MAAM,KAAK,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IAE/D,OAAO;QACL,MAAM,EAAE,WAAW;QACnB,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC;QAC1B,OAAO;KACR,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,MAAc,EAAE,UAAuB,EAAE,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAyB;IACnJ,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,IAAI,CAAC,CAAC;QACrC,IAAI,MAAM,GAAG,KAAK,CAAC;QACnB,IAAI,OAAO,GAAG,EAAE,CAAC;QAEjB,IAAI,SAAS,CAAC,IAAI,KAAK,qBAAqB,EAAE,CAAC;YAC7C,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,IAAI,EAAE,CAAC;YAC5C,MAAM,WAAW,GAAG;gBAClB,kBAAkB;gBAClB,EAAE;gBACF,SAAS;gBACT,SAAS;gBACT,EAAE;gBACF,UAAU;gBACV,MAAM;gBACN,EAAE;gBACF,gCAAgC;gBAChC,yCAAyC;gBACzC,EAAE;gBACF,2CAA2C;aAC5C,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAEb,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC;gBAC5B,KAAK,EAAE,UAAU;gBACjB,MAAM,EAAE,6BAA6B;gBACrC,MAAM,EAAE,WAAW;aACpB,CAAC,CAAC;YAEH,YAAY,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;YACpC,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;gBACd,IAAI,CAAC;oBACH,MAAM,SAAS,GAAG,MAAM,CAAC,MAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;oBAC7D,IAAI,SAAS,EAAE,CAAC;wBACd,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAkB,CAAC;wBACzD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;wBACxC,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,IAAI,CAAC,CAAC;wBAC3C,MAAM,GAAG,KAAK,IAAI,SAAS,CAAC;wBAC5B,OAAO,GAAG,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;oBAChC,CAAC;yBAAM,CAAC;wBACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sDAAsD,MAAM,CAAC,MAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;oBAC/G,CAAC;gBACH,CAAC;gBAAC,OAAO,QAAiB,EAAE,CAAC;oBAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACtG,CAAC;YACH,CAAC;QACH,CAAC;aAAM,IACL,SAAS,CAAC,IAAI,KAAK,cAAc;YACjC,SAAS,CAAC,IAAI,KAAK,kBAAkB;YACrC,SAAS,CAAC,IAAI,KAAK,gBAAgB,EACnC,CAAC;YACD,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;YACrF,YAAY,IAAI,SAAS,CAAC,OAAO,CAAC;YAClC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;YAC1B,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC;QAC9B,CAAC;aAAM,IAAI,SAAS,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,SAAS,CAAC,EAAG,CAAC,CAAC;gBAClD,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,MAAM,CAA0B,CAAC;gBAC1D,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,KAAK,CAAC;gBACpC,IAAI,CAAC,EAAE;oBAAE,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;gBACjF,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAwB;oBACvD,EAAE,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;oBACjC,IAAI,OAAO,CAAwB,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,+BAA+B,2BAA2B,GAAG,IAAI,IAAI,CAAC,CAAC,EAAE,2BAA2B,CAAC,CAAC;iBAC3L,CAAC,CAAC;gBACH,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC9B,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;YACjC,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACtB,MAAM,GAAG,KAAK,CAAC;gBACf,OAAO,GAAG,2BAA2B,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9D,CAAC;QACH,CAAC;QAED,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,KAAK,EAAE,SAAS,CAAC,SAAS,IAAI,SAAS,CAAC,EAAE,IAAI,EAAE;YAChD,MAAM;YACN,MAAM;YACN,GAAG,CAAC,OAAO,IAAI,EAAE,OAAO,EAAE,CAAC;SAC5B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC9D,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACvF,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC3D,MAAM,KAAK,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IAE/D,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,CAAC;AACzH,CAAC;AAED,8EAA8E;AAC9E,qCAAqC;AACrC,8EAA8E;AAC9E,EAAE;AACF,oDAAoD;AACpD,EAAE;AACF,uEAAuE;AACvE,qEAAqE;AACrE,6EAA6E;AAC7E,gEAAgE;AAChE,yEAAyE;AACzE,0EAA0E;AAC1E,oDAAoD;AACpD,EAAE;AACF,wBAAwB;AACxB,EAAE;AACF,gFAAgF;AAChF,iFAAiF;AACjF,iFAAiF;AACjF,8EAA8E;AAC9E,wBAAwB;AACxB,6EAA6E;AAC7E,6EAA6E;AAC7E,kFAAkF;AAClF,4BAA4B;AAC5B,2EAA2E;AAC3E,0EAA0E;AAC1E,8DAA8D;AAC9D,gFAAgF;AAEhF,MAAM,iBAAiB,GAAG;IACxB,gBAAgB;IAChB,yBAAyB;IACzB,wBAAwB;CACzB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAQb,KAAK,UAAU,WAAW,CACxB,SAAoB,EACpB,MAAc,EACd,MAAc,EACd,QAAoB,EACpB,UAAkB;IAElB,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,IAAI,CAAC,CAAC;IAE3C,IAAI,MAAc,CAAC;IACnB,IAAI,MAAc,CAAC;IAEnB,IAAI,SAAS,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;QAC5D,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,uDAAuD,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QACzG,CAAC;QACD,MAAM,GAAG,6CAA6C,CAAC;QACvD,MAAM,GAAG;YACP,qCAAqC;YACrC,EAAE;YACF,eAAe;YACf,OAAO;YACP,EAAE;YACF,UAAU;YACV,MAAM;YACN,EAAE;YACF,iBAAiB;YACjB,EAAE;YACF,SAAS;YACT,oBAAoB;YACpB,2BAA2B;YAC3B,cAAc;YACd,8BAA8B;YAC9B,4BAA4B;YAC5B,cAAc;YACd,cAAc;YACd,0BAA0B;YAC1B,EAAE;YACF,4BAA4B;YAC5B,yCAAyC;SAC1C,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC;SAAM,IAAI,SAAS,CAAC,IAAI,KAAK,kBAAkB,EAAE,CAAC;QACjD,MAAM,GAAG,sBAAsB,CAAC;QAChC,MAAM,GAAG;YACP,8BAA8B;YAC9B,EAAE;YACF,SAAS;YACT,MAAM,CAAC,MAAM;YACb,EAAE;YACF,UAAU;YACV,MAAM;YACN,EAAE;YACF,iBAAiB;YACjB,EAAE;YACF,OAAO;YACP,mBAAmB;YACnB,kBAAkB;YAClB,oBAAoB;YACpB,WAAW;YACX,aAAa;YACb,EAAE;YACF,4BAA4B;YAC5B,yCAAyC;SAC1C,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC;SAAM,CAAC;QACN,iBAAiB;QACjB,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;QAC9D,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,yDAAyD,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAC3G,CAAC;QACD,MAAM,GAAG,6BAA6B,CAAC;QACvC,MAAM,GAAG;YACP,oCAAoC;YACpC,EAAE;YACF,YAAY;YACZ,SAAS;YACT,EAAE;YACF,UAAU;YACV,MAAM;YACN,EAAE;YACF,iBAAiB;YACjB,EAAE;YACF,SAAS;YACT,wBAAwB;YACxB,qBAAqB;YACrB,cAAc;YACd,kBAAkB;YAClB,0BAA0B;YAC1B,aAAa;YACb,cAAc;YACd,cAAc;YACd,EAAE;YACF,4BAA4B;YAC5B,yCAAyC;SAC1C,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACrE,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO;YACL,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,GAAG,SAAS,CAAC,IAAI,iBAAiB,MAAM,CAAC,KAAK,EAAE;YACzD,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,CAAC;SAC7B,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,CAAC,MAAO,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC5C,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,SAAS,CAAC,IAAI,6BAA6B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YACjG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,yBAAyB,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,CAAC,EAAE,CAAC;QAC7F,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAkB,CAAC;QACzD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,OAAO;YACL,MAAM,EAAE,KAAK,IAAI,SAAS;YAC1B,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;YACnD,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,CAAC;SAC7B,CAAC;IACJ,CAAC;IAAC,OAAO,QAAiB,EAAE,CAAC;QAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,SAAS,CAAC,IAAI,uBAAuB,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAClG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,gCAAgC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,CAAC,EAAE,CAAC;IACpG,CAAC;AACH,CAAC"}