@chatcops/cli 0.0.0-stage → 0.1.0

package/dist/test/judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/test/judge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAyD,MAAM,gBAAgB,CAAC;AACrH,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAE3C,OAAO,EAAE,MAAM,EAAe,MAAM,WAAW,CAAC;AAGhD,0DAA0D;AAC1D,MAAM,CAAC,MAAM,OAAO,GAA6B;IAC/C,MAAM,EAAE,CAAC,mBAAmB,CAAC;IAC7B,MAAM,EAAE,CAAC,gBAAgB,CAAC;IAC1B,MAAM,EAAE,CAAC,gBAAgB,EAAE,gBAAgB,CAAC;CAC7C,CAAC;AAEF,MAAM,aAAa,GAAG;;;;;;;qEAO+C,CAAC;AActE;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAAqB;IACtD,IAAI,QAAoB,CAAC;IACzB,IAAI,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC9H,QAAQ,GAAG,MAAM,KAAK,CAAC,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;IAC9E,CAAC;SAAM,IAAI,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5C,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC;IAChC,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;QAClC,QAAQ,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;IAC/G,CAAC;IACD,MAAM,EAAE,GAAG,GAAG,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,KAAK,IAAI,SAAS,EAAE,CAAC;IAC7D,OAAO;QACL,EAAE;QACF,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,KAAK,EAAE,CAAC,MAAM,EAAE,YAAY,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC;KACvF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,CAAC,IAAI,EAAE,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,WAAW,CAAC,iBAAiB,IAAI,wFAAwF,CAAC,CAAC;IACzJ,OAAO,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;AACvF,CAAC;AAED,SAAS,MAAM,CAAC,IAAY,EAAE,OAAqB;IACjD,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC;QACxC,IAAI,KAAK;YAAE,OAAO,KAAK,CAAC;IAC1B,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,iBAAiB,CAAC;IACxD,MAAM,IAAI,WAAW,CACnB,8CAA8C,IAAI,UAAU,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,aAAa,0EAA0E,CAC1K,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,MAAsB;IACzC,IAAI,CAAC;QACH,OAAO,MAAM,cAAc,CAAC,MAAM,CAAC,CAAC;IACtC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,WAAW,CAAC,0BAA2B,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;IAC9E,CAAC;AACH,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,QAAoB,EAAE,MAAc,EAAE,YAA0B,EAAE,MAAe;IACpG,MAAM,UAAU,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,IAAI,CAAC,IAAI,gBAAgB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC7G,MAAM,OAAO,GAAG,aAAa,MAAM,kCAAkC,UAAU,2EAA2E,CAAC;IAC3J,MAAM,QAAQ,GAAG,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,MAAe,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;IAEtF,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,IAAI,KAA6B,CAAC;IAClC,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;QAClB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;YACnG,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY;gBAAE,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC;iBAChD,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;gBAAE,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;QACxD,CAAC;IACH,CAAC;SAAM,CAAC;QACN,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IAC5F,CAAC;IAED,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACtD,OAAO,EAAE,GAAG,OAAO,EAAE,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;AACvE,CAAC;AAED,qDAAqD;AACrD,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5C,IAAI,MAAyD,CAAC;IAC9D,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,GAAG,SAAS,CAAC;IACrB,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,EAAE,KAAK,CAAC;IAC5B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACxD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,qDAAqD,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,WAAW,EAAE,CAAC,CAAC;IAC7I,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,MAAM,EAAE,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,mBAAmB,EAAE,CAAC;AACpI,CAAC"}

package/dist/test/load.d.ts

@@ -0,0 +1,9 @@
+import type { Fixture, TestCase } from './types.js';
+export declare const DEFAULT_PATTERN = "evals/**/*.chat.yaml";
+/** A fixture file, checked, with defaults applied to each test. */
+export declare function loadFixture(file: string): Promise<Fixture>;
+/** Files under `cwd` matching any of the globs (relative to `cwd`). */
+export declare function findFixtures(cwd: string, patterns: string[]): Promise<string[]>;
+/** `--filter`: a substring of the test name, or `tag:<name>`. */
+export declare function matchesFilter(test: TestCase, filter: string | undefined): boolean;
+//# sourceMappingURL=load.d.ts.map

package/dist/test/load.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"load.d.ts","sourceRoot":"","sources":["../../src/test/load.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEpD,eAAO,MAAM,eAAe,yBAAyB,CAAC;AAiCtD,mEAAmE;AACnE,wBAAsB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAkBhE;AAWD,uEAAuE;AACvE,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAerF;AAED,iEAAiE;AACjE,wBAAgB,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAIjF"}

package/dist/test/load.js

@@ -0,0 +1,98 @@
+import { readdir, readFile } from 'node:fs/promises';
+import { join, relative, resolve, sep } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { globToRegExp } from '@chatcops/ingest';
+import { parse as parseYaml } from 'yaml';
+import { z } from 'zod';
+import { ConfigError } from '../config.js';
+export const DEFAULT_PATTERN = 'evals/**/*.chat.yaml';
+const SKIP_DIRS = new Set(['node_modules', '.git', 'dist', '__cassettes__']);
+const assertion = z.record(z.unknown()).refine((value) => Object.keys(value).length === 1, {
+    message: 'an assertion has exactly one key, such as `contains: "$19"`',
+});
+const pageContext = z.object({
+    url: z.string().url(),
+    title: z.string(),
+    description: z.string().optional(),
+    contentSnippet: z.string().optional(),
+});
+const toolResult = z.object({ success: z.boolean(), data: z.unknown().optional(), message: z.string().optional() });
+const common = {
+    tags: z.array(z.string()).optional(),
+    locale: z.string().optional(),
+    pageContext: pageContext.optional(),
+    userData: z.record(z.string()).optional(),
+    mocks: z.object({ tools: z.record(toolResult).optional() }).strict().optional(),
+    repeat: z.number().int().min(1).max(100).optional(),
+    passRate: z.number().min(0).max(1).optional(),
+};
+const turn = z.object({ user: z.string().min(1), expect: z.array(assertion).optional() }).strict();
+const testCase = z.object({ name: z.string().min(1), turns: z.array(turn).min(1), ...common }).strict();
+const fixtureSchema = z
+    .object({
+    version: z.literal(1),
+    name: z.string().min(1),
+    defaults: z.object(common).strict().optional(),
+    tests: z.array(testCase).min(1),
+})
+    .strict();
+/** A fixture file, checked, with defaults applied to each test. */
+export async function loadFixture(file) {
+    let raw;
+    try {
+        if (/\.ya?ml$/i.test(file)) {
+            raw = parseYaml(await readFile(file, 'utf8'));
+        }
+        else {
+            raw = (await import(pathToFileURL(file).href)).default;
+        }
+    }
+    catch (error) {
+        throw new ConfigError(`${file}: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    const parsed = fixtureSchema.safeParse(raw);
+    if (!parsed.success) {
+        const issues = parsed.error.issues.map((issue) => `  ${issue.path.join('.') || '(root)'}: ${issue.message}`);
+        throw new ConfigError(`${file} isn't a valid eval file:\n${issues.join('\n')}`);
+    }
+    const fixture = parsed.data;
+    return { ...fixture, tests: fixture.tests.map((test) => withDefaults(test, fixture.defaults)) };
+}
+function withDefaults(test, defaults) {
+    if (!defaults)
+        return test;
+    return {
+        ...defaults,
+        ...test,
+        mocks: { tools: { ...defaults.mocks?.tools, ...test.mocks?.tools } },
+    };
+}
+/** Files under `cwd` matching any of the globs (relative to `cwd`). */
+export async function findFixtures(cwd, patterns) {
+    const regexes = patterns.map((pattern) => globToRegExp(pattern.replace(/^\.\//, '')));
+    const found = [];
+    async function walk(dir) {
+        for (const entry of await readdir(dir, { withFileTypes: true })) {
+            if (entry.isDirectory()) {
+                if (!SKIP_DIRS.has(entry.name))
+                    await walk(join(dir, entry.name));
+            }
+            else if (entry.isFile()) {
+                const path = relative(cwd, join(dir, entry.name)).split(sep).join('/');
+                if (regexes.some((regex) => regex.test(path)))
+                    found.push(resolve(cwd, path));
+            }
+        }
+    }
+    await walk(cwd);
+    return found.sort();
+}
+/** `--filter`: a substring of the test name, or `tag:<name>`. */
+export function matchesFilter(test, filter) {
+    if (!filter)
+        return true;
+    if (filter.startsWith('tag:'))
+        return test.tags?.includes(filter.slice(4)) ?? false;
+    return test.name.toLowerCase().includes(filter.toLowerCase());
+}
+//# sourceMappingURL=load.js.map

package/dist/test/load.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"load.js","sourceRoot":"","sources":["../../src/test/load.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAG3C,MAAM,CAAC,MAAM,eAAe,GAAG,sBAAsB,CAAC;AACtD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,CAAC,CAAC,CAAC;AAE7E,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE;IACzF,OAAO,EAAE,6DAA6D;CACvE,CAAC,CAAC;AACH,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;IACrB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACtC,CAAC,CAAC;AACH,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AACpH,MAAM,MAAM,GAAG;IACb,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACpC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,WAAW,EAAE,WAAW,CAAC,QAAQ,EAAE;IACnC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACzC,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/E,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;IACnD,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CAC9C,CAAC;AACF,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;AACnG,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;AACxG,MAAM,aAAa,GAAG,CAAC;KACpB,MAAM,CAAC;IACN,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACrB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9C,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CAChC,CAAC;KACD,MAAM,EAAE,CAAC;AAEZ,mEAAmE;AACnE,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAY;IAC5C,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,GAAG,GAAG,SAAS,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,GAAG,GAAI,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAA2B,CAAC,OAAO,CAAC;QACpF,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,WAAW,CAAC,GAAG,IAAI,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC9F,CAAC;IACD,MAAM,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC5C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7G,MAAM,IAAI,WAAW,CAAC,GAAG,IAAI,8BAA8B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAClF,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAe,CAAC;IACvC,OAAO,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;AAClG,CAAC;AAED,SAAS,YAAY,CAAC,IAAc,EAAE,QAA6B;IACjE,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3B,OAAO;QACL,GAAG,QAAQ;QACX,GAAG,IAAI;QACP,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,GAAG,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;KACrE,CAAC;AACJ,CAAC;AAED,uEAAuE;AACvE,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,GAAW,EAAE,QAAkB;IAChE,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IACtF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,UAAU,IAAI,CAAC,GAAW;QAC7B,KAAK,MAAM,KAAK,IAAI,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YAChE,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;oBAAE,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YACpE,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC1B,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACvE,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAChF,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,IAAI,CAAC,GAAG,CAAC,CAAC;IAChB,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;AACtB,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,aAAa,CAAC,IAAc,EAAE,MAA0B;IACtE,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IACzB,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;IACpF,OAAO,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;AAChE,CAAC"}

package/dist/test/reporters/index.d.ts

@@ -0,0 +1,35 @@
+import type { CassetteMode } from '../cassette.js';
+import type { TestResult } from '../types.js';
+/** Where a reporter writes: stdout, unless the reporter is given a file. */
+export interface ReporterOutput {
+    write(text: string): unknown;
+}
+/** About the whole run, for the end of a report. */
+export interface RunSummary {
+    mode: CassetteMode;
+    /** The bot's model id, when available. */
+    model?: string;
+    /** Total time spent running the evals, in milliseconds. */
+    elapsedMs: number;
+    /** Whether some calls could not be priced. */
+    costIncomplete: boolean;
+    /** The judge's `provider:model`, when the run used one. */
+    judge?: string;
+    /** Turns and judge calls with a price, in US dollars. Unset when nothing had a price. */
+    costUsd?: number;
+}
+/** Gets results as tests finish, and all of them at the end. */
+export interface Reporter {
+    onResult?(result: TestResult): void;
+    onEnd?(results: TestResult[], summary: RunSummary): void | Promise<void>;
+}
+export type ReporterFactory = (output: ReporterOutput, options: {
+    verbose?: boolean;
+    env?: Record<string, string | undefined>;
+}) => Reporter;
+/**
+ * The reporters `--reporter` can name. To add one, write it in its own file
+ * in this folder and add it here.
+ */
+export declare const REPORTERS: Map<string, ReporterFactory>;
+//# sourceMappingURL=index.d.ts.map

package/dist/test/reporters/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/test/reporters/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAI9C,4EAA4E;AAC5E,MAAM,WAAW,cAAc;IAC7B,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;CAC9B;AAED,oDAAoD;AACpD,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,YAAY,CAAC;IACnB,0CAA0C;IAC1C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,SAAS,EAAE,MAAM,CAAC;IAClB,8CAA8C;IAC9C,cAAc,EAAE,OAAO,CAAC;IACxB,2DAA2D;IAC3D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yFAAyF;IACzF,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,gEAAgE;AAChE,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI,CAAC;IACpC,KAAK,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,EAAE,OAAO,EAAE,UAAU,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1E;AAED,MAAM,MAAM,eAAe,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE;IAAE,OAAO,CAAC,EAAE,OAAO,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAA;CAAE,KAAK,QAAQ,CAAC;AAE7I;;;GAGG;AACH,eAAO,MAAM,SAAS,8BAGpB,CAAC"}

package/dist/test/reporters/index.js

@@ -0,0 +1,11 @@
+import { jsonReporter } from './json.js';
+import { prettyReporter } from './pretty.js';
+/**
+ * The reporters `--reporter` can name. To add one, write it in its own file
+ * in this folder and add it here.
+ */
+export const REPORTERS = new Map([
+    ['pretty', prettyReporter],
+    ['json', jsonReporter],
+]);
+//# sourceMappingURL=index.js.map

package/dist/test/reporters/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/test/reporters/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AA8B7C;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,IAAI,GAAG,CAA0B;IACxD,CAAC,QAAQ,EAAE,cAAc,CAAC;IAC1B,CAAC,MAAM,EAAE,YAAY,CAAC;CACvB,CAAC,CAAC"}

package/dist/test/reporters/json.d.ts

@@ -0,0 +1,4 @@
+import type { ReporterFactory } from './index.js';
+/** One JSON object at the end: counts, and every run with every turn. */
+export declare const jsonReporter: ReporterFactory;
+//# sourceMappingURL=json.d.ts.map

package/dist/test/reporters/json.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../../src/test/reporters/json.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD,yEAAyE;AACzE,eAAO,MAAM,YAAY,EAAE,eAKzB,CAAC"}

package/dist/test/reporters/json.js

@@ -0,0 +1,8 @@
+/** One JSON object at the end: counts, and every run with every turn. */
+export const jsonReporter = (output) => ({
+    onEnd(results, summary) {
+        const failed = results.filter((result) => !result.pass).length;
+        output.write(`${JSON.stringify({ passed: results.length - failed, failed, summary, results })}\n`);
+    },
+});
+//# sourceMappingURL=json.js.map

package/dist/test/reporters/json.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"json.js","sourceRoot":"","sources":["../../../src/test/reporters/json.ts"],"names":[],"mappings":"AAEA,yEAAyE;AACzE,MAAM,CAAC,MAAM,YAAY,GAAoB,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACxD,KAAK,CAAC,OAAO,EAAE,OAAO;QACpB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QAC/D,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC;IACrG,CAAC;CACF,CAAC,CAAC"}

package/dist/test/reporters/pretty.d.ts

@@ -0,0 +1,3 @@
+import type { ReporterFactory } from './index.js';
+export declare const prettyReporter: ReporterFactory;
+//# sourceMappingURL=pretty.d.ts.map

package/dist/test/reporters/pretty.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pretty.d.ts","sourceRoot":"","sources":["../../../src/test/reporters/pretty.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,eAAe,EAAc,MAAM,YAAY,CAAC;AAE9D,eAAO,MAAM,cAAc,EAAE,eAU5B,CAAC"}

package/dist/test/reporters/pretty.js

@@ -0,0 +1,119 @@
+import { styleText } from 'node:util';
+import { dollars } from '../cost.js';
+export const prettyReporter = (output, options = {}) => {
+    const env = options.env ?? process.env;
+    const colors = 'isTTY' in output && output.isTTY === true && env.NO_COLOR === undefined;
+    const paint = (color, text) => (colors ? styleText(color, text, { validateStream: false }) : text);
+    return {
+        onEnd(results, summary) {
+            output.write(prettyReport(results, summary, options.verbose === true, paint));
+        },
+    };
+};
+function prettyReport(results, summary, verbose, paint) {
+    const lines = [
+        '',
+        paint('cyan', 'ChatCops evaluations'),
+        paint('gray', '────────────────────────────────────────'),
+        paint('gray', `Model: ${summary.model ?? 'unavailable'}`),
+        ...(summary.judge ? [paint('gray', `Judge: ${summary.judge}`)] : []),
+        paint('gray', `Mode: ${summary.mode === 'record' ? 'recorded' : summary.mode}`),
+        '',
+    ];
+    const groups = new Map();
+    for (const result of results) {
+        const group = groups.get(result.fixture) ?? [];
+        group.push(result);
+        groups.set(result.fixture, group);
+    }
+    for (const [fixture, tests] of groups) {
+        lines.push(paint('cyan', fixture));
+        for (const result of tests) {
+            const status = result.pass ? paint('green', '✓') : paint('red', '✗');
+            const rate = result.runs.length > 1 ? ` (${result.runs.filter((run) => run.pass).length}/${result.runs.length} runs)` : '';
+            const elapsed = result.durationMs;
+            const duration = elapsed === undefined ? '' : `  ${formatDuration(elapsed)}`;
+            lines.push(`  ${status} ${result.test}${rate}${duration}`);
+            if (!result.pass || verbose) {
+                appendDetails(lines, result, result.runs.length > 1, paint);
+                lines.push('');
+            }
+        }
+        if (lines.at(-1) !== '')
+            lines.push('');
+    }
+    const failed = results.filter((result) => !result.pass).length;
+    const assertions = results.flatMap((result) => result.runs.flatMap((run) => run.turns.flatMap(({ results: checks }) => checks)));
+    const passedAssertions = assertions.filter((check) => check.pass).length;
+    const cost = summary.mode === 'replay'
+        ? ' Replayed, nothing spent.'
+        : summary.costUsd === undefined
+            ? ''
+            : summary.costIncomplete
+                ? ` Spent about ${dollars(summary.costUsd)} (partial; some calls could not be priced).`
+                : ` Spent about ${dollars(summary.costUsd)}.`;
+    const judge = summary.judge ? ` Judged by ${summary.judge}.` : '';
+    lines.push(paint('cyan', 'Summary'), paint('gray', '────────────────────────────────────────'), `${paint('green', `${results.length - failed} passed`)}, ${paint(failed ? 'red' : 'gray', `${failed} failed`)}.${cost}${judge}`, `Assertions: ${passedAssertions} passed, ${assertions.length - passedAssertions} failed`, formatUsage(results), paint('gray', `Elapsed: ${formatDuration(summary.elapsedMs)}`));
+    return `${lines.join('\n')}\n`;
+}
+function appendDetails(lines, result, showRun, paint) {
+    for (const run of result.runs) {
+        for (const [index, { turn, results: checks }] of run.turns.entries()) {
+            if (showRun && index === 0)
+                lines.push(paint('gray', `    Run ${run.run}`));
+            if (!checks.length)
+                lines.push(paint('gray', '      No assertions for this turn.'));
+            for (const check of checks) {
+                const status = check.pass ? paint('green', '✓') : paint('red', '✗');
+                const wrapped = wrapText(`turn ${index + 1}: ${reason(check)}`, 76);
+                lines.push(`      ${status} ${wrapped[0]}`);
+                for (const continuation of wrapped.slice(1))
+                    lines.push(`        ${continuation}`);
+            }
+            if (checks.some((check) => !check.pass)) {
+                lines.push(paint('gray', `      reply: ${clip(turn.text)}`));
+            }
+        }
+        if (run.error)
+            lines.push(`    Run ${run.run}: ${run.error}`);
+    }
+}
+/** Wrap long explanations without losing words or relying on ANSI string widths. */
+function wrapText(text, width) {
+    const lines = [];
+    let line = '';
+    for (const word of text.split(/\s+/)) {
+        if (line && line.length + word.length + 1 > width) {
+            lines.push(line);
+            line = word;
+        }
+        else {
+            line = line ? `${line} ${word}` : word;
+        }
+    }
+    lines.push(line);
+    return lines;
+}
+function reason(check) {
+    return check.reason ?? Object.keys(check.assertion)[0];
+}
+function formatUsage(results) {
+    const turns = results.flatMap((result) => result.runs.flatMap((run) => run.turns.map(({ turn }) => turn)));
+    const reported = turns.flatMap((turn) => (turn.usage ? [turn.usage] : []));
+    if (!reported.length)
+        return 'Bot tokens: unavailable (not reported)';
+    const input = reported.reduce((total, usage) => total + usage.inputTokens, 0);
+    const output = reported.reduce((total, usage) => total + usage.outputTokens, 0);
+    const partial = reported.length < turns.length ? ` (partial; reported for ${reported.length} of ${turns.length} turns)` : '';
+    return `Bot tokens${partial}: ${input} input + ${output} output = ${input + output} total`;
+}
+function formatDuration(milliseconds) {
+    if (milliseconds === undefined)
+        return 'unavailable';
+    return milliseconds < 1000 ? `${Math.round(milliseconds)} ms` : `${(milliseconds / 1000).toFixed(2)} s`;
+}
+function clip(text) {
+    const flat = text.replace(/\s+/g, ' ').trim();
+    return flat ? (flat.length > 200 ? `${flat.slice(0, 199)}…` : flat) : '(empty)';
+}
+//# sourceMappingURL=pretty.js.map

package/dist/test/reporters/pretty.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pretty.js","sourceRoot":"","sources":["../../../src/test/reporters/pretty.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAIrC,MAAM,CAAC,MAAM,cAAc,GAAoB,CAAC,MAAM,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;IACtE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IACvC,MAAM,MAAM,GAAG,OAAO,IAAI,MAAM,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI,IAAI,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC;IACxF,MAAM,KAAK,GAAG,CAAC,KAAwC,EAAE,IAAY,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAE9I,OAAO;QACL,KAAK,CAAC,OAAO,EAAE,OAAO;YACpB,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,KAAK,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;QAChF,CAAC;KACF,CAAC;AACJ,CAAC,CAAC;AAEF,SAAS,YAAY,CACnB,OAAqB,EACrB,OAAmB,EACnB,OAAgB,EAChB,KAAyE;IAEzE,MAAM,KAAK,GAAG;QACZ,EAAE;QACF,KAAK,CAAC,MAAM,EAAE,sBAAsB,CAAC;QACrC,KAAK,CAAC,MAAM,EAAE,0CAA0C,CAAC;QACzD,KAAK,CAAC,MAAM,EAAE,UAAU,OAAO,CAAC,KAAK,IAAI,aAAa,EAAE,CAAC;QACzD,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACpE,KAAK,CAAC,MAAM,EAAE,SAAS,OAAO,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/E,EAAE;KACH,CAAC;IACF,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC/C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/C,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnB,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;QACnC,KAAK,MAAM,MAAM,IAAI,KAAK,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YACrE,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3H,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC;YAClC,MAAM,QAAQ,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7E,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,CAAC,IAAI,GAAG,IAAI,GAAG,QAAQ,EAAE,CAAC,CAAC;YAC3D,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,OAAO,EAAE,CAAC;gBAC5B,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;gBAC5D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;QACD,IAAI,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE;YAAE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAC/D,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACjI,MAAM,gBAAgB,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IACzE,MAAM,IAAI,GACR,OAAO,CAAC,IAAI,KAAK,QAAQ;QACvB,CAAC,CAAC,2BAA2B;QAC7B,CAAC,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS;YAC7B,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,OAAO,CAAC,cAAc;gBACtB,CAAC,CAAC,gBAAgB,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,6CAA6C;gBACvF,CAAC,CAAC,gBAAgB,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC;IACtD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAClE,KAAK,CAAC,IAAI,CACR,KAAK,CAAC,MAAM,EAAE,SAAS,CAAC,EACxB,KAAK,CAAC,MAAM,EAAE,0CAA0C,CAAC,EACzD,GAAG,KAAK,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,GAAG,MAAM,SAAS,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,GAAG,MAAM,SAAS,CAAC,IAAI,IAAI,GAAG,KAAK,EAAE,EAC/H,eAAe,gBAAgB,YAAY,UAAU,CAAC,MAAM,GAAG,gBAAgB,SAAS,EACxF,WAAW,CAAC,OAAO,CAAC,EACpB,KAAK,CAAC,MAAM,EAAE,YAAY,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAC/D,CAAC;IACF,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AACjC,CAAC;AAED,SAAS,aAAa,CACpB,KAAe,EACf,MAAkB,EAClB,OAAgB,EAChB,KAAyE;IAEzE,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YACrE,IAAI,OAAO,IAAI,KAAK,KAAK,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YAC5E,IAAI,CAAC,MAAM,CAAC,MAAM;gBAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,oCAAoC,CAAC,CAAC,CAAC;YACpF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;gBACpE,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,KAAK,GAAG,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;gBACpE,KAAK,CAAC,IAAI,CAAC,SAAS,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC5C,KAAK,MAAM,YAAY,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,WAAW,YAAY,EAAE,CAAC,CAAC;YACrF,CAAC;YACD,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,gBAAgB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC;QACD,IAAI,GAAG,CAAC,KAAK;YAAE,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;IAChE,CAAC;AACH,CAAC;AAED,oFAAoF;AACpF,SAAS,QAAQ,CAAC,IAAY,EAAE,KAAa;IAC3C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;QACrC,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,KAAK,EAAE,CAAC;YAClD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,IAAI,GAAG,IAAI,CAAC;QACd,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACzC,CAAC;IACH,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjB,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,MAAM,CAAC,KAAsB;IACpC,OAAO,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,WAAW,CAAC,OAAqB;IACxC,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3G,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3E,IAAI,CAAC,QAAQ,CAAC,MAAM;QAAE,OAAO,wCAAwC,CAAC;IACtE,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAC9E,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;IAChF,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,2BAA2B,QAAQ,CAAC,MAAM,OAAO,KAAK,CAAC,MAAM,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7H,OAAO,aAAa,OAAO,KAAK,KAAK,YAAY,MAAM,aAAa,KAAK,GAAG,MAAM,QAAQ,CAAC;AAC7F,CAAC;AAED,SAAS,cAAc,CAAC,YAAgC;IACtD,IAAI,YAAY,KAAK,SAAS;QAAE,OAAO,aAAa,CAAC;IACrD,OAAO,YAAY,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AAC1G,CAAC;AAED,SAAS,IAAI,CAAC,IAAY;IACxB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9C,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAClF,CAAC"}

package/dist/test/run.d.ts

@@ -0,0 +1,46 @@
+import { type ChatTool, type ToolResult } from '@chatcops/core';
+import { createChatHandler, type ChatCopsServerConfig } from '@chatcops/server';
+import type { Cassettes } from './cassette.js';
+import { type Prices, type Spend } from './cost.js';
+import type { Fixture, Judge, TestCase, TestResult, TurnResult } from './types.js';
+export interface RunOptions {
+    /** Tests run at once. Default 4. */
+    concurrency?: number;
+    /** Overrides each test's `repeat`. */
+    repeat?: number;
+    /** Called as each test finishes. */
+    onResult?: (result: TestResult) => void;
+    /** Records or replays each test's HTTP calls. */
+    cassettes?: Cassettes;
+    /** For `judge` assertions. */
+    judge?: Judge;
+    /** The bot's model id, to price its turns. */
+    model?: string;
+    prices?: Prices;
+    /** Stops starting turns once `--max-cost` is reached. */
+    spend?: Spend;
+}
+/** An eval file to run: `file` as reported, `path` on disk (for its cassette). */
+export interface FixtureJob {
+    file: string;
+    path?: string;
+    fixture: Fixture;
+    tests: TestCase[];
+}
+/**
+ * Runs every test of the fixtures against the server config, in process: the
+ * real request path (knowledge, page context, locale, tools), with fresh
+ * in-memory storage for every run and the fixture's tool mocks.
+ */
+export declare function runFixtures(server: ChatCopsServerConfig, fixtures: FixtureJob[], options?: RunOptions): Promise<TestResult[]>;
+/** Sends one message and collects the reply's parts as a turn result. */
+export declare function sendTurn(handler: ReturnType<typeof createChatHandler>, body: Record<string, unknown>): Promise<TurnResult>;
+/** The config's tools, with mocked ones answering their fixed result instead of running. */
+export declare function withMocks(tools: ChatTool[], mocks: Record<string, ToolResult> | undefined): ChatTool[];
+/**
+ * In replay nothing reaches the network, so CI needs no keys. The built-in
+ * providers refuse to start without one, so give them a stand-in. Headers
+ * aren't part of what a recording is matched on.
+ */
+export declare function withoutKeys(server: ChatCopsServerConfig): ChatCopsServerConfig;
+//# sourceMappingURL=run.d.ts.map

package/dist/test/run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/test/run.ts"],"names":[],"mappings":"AACA,OAAO,EAA+B,KAAK,QAAQ,EAAE,KAAK,UAAU,EAAiB,MAAM,gBAAgB,CAAC;AAC5G,OAAO,EAAE,iBAAiB,EAAE,KAAK,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAGhF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,EAAmB,KAAK,MAAM,EAAE,KAAK,KAAK,EAAE,MAAM,WAAW,CAAC;AACrE,OAAO,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAiB,UAAU,EAAE,MAAM,YAAY,CAAC;AAElG,MAAM,WAAW,UAAU;IACzB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sCAAsC;IACtC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,oCAAoC;IACpC,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,IAAI,CAAC;IACxC,iDAAiD;IACjD,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,8CAA8C;IAC9C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yDAAyD;IACzD,KAAK,CAAC,EAAE,KAAK,CAAC;CACf;AAED,kFAAkF;AAClF,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,QAAQ,EAAE,CAAC;CACnB;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,MAAM,EAAE,oBAAoB,EAC5B,QAAQ,EAAE,UAAU,EAAE,EACtB,OAAO,GAAE,UAAe,GACvB,OAAO,CAAC,UAAU,EAAE,CAAC,CA2BvB;AAgED,yEAAyE;AACzE,wBAAsB,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC,OAAO,iBAAiB,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC,CA8BhI;AAED,4FAA4F;AAC5F,wBAAgB,SAAS,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,GAAG,SAAS,GAAG,QAAQ,EAAE,CAQtG;AAED;;;;GAIG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,oBAAoB,GAAG,oBAAoB,CAI9E"}

package/dist/test/run.js

@@ -0,0 +1,157 @@
+import { randomUUID } from 'node:crypto';
+import { isAIProvider, memoryStorage } from '@chatcops/core';
+import { createChatHandler } from '@chatcops/server';
+import { ConfigError } from '../config.js';
+import { check } from './assertions/index.js';
+import { costOf, dollars } from './cost.js';
+/**
+ * Runs every test of the fixtures against the server config, in process: the
+ * real request path (knowledge, page context, locale, tools), with fresh
+ * in-memory storage for every run and the fixture's tool mocks.
+ */
+export async function runFixtures(server, fixtures, options = {}) {
+    if (options.cassettes?.mode === 'replay')
+        server = withoutKeys(server);
+    const jobs = fixtures.flatMap(({ file, path, tests }) => tests.map((test) => ({ file, path: path ?? file, test })));
+    const results = new Array(jobs.length);
+    let next = 0;
+    const worker = async () => {
+        while (next < jobs.length) {
+            const index = next++;
+            const { file, path, test } = jobs[index];
+            const started = performance.now();
+            results[index] = await runTest(file, test, options.repeat, (run) => {
+                const once = () => runOnce(server, { file, path, test, run }, options);
+                if (!options.cassettes)
+                    return once();
+                return options.cassettes.run(path, test.name, once).then(({ value, problems }) => {
+                    if (problems.length) {
+                        value.pass = false;
+                        value.error = problems.length > 1 ? `${problems[0]} (and ${problems.length - 1} more)` : problems[0];
+                    }
+                    return value;
+                });
+            });
+            results[index].durationMs = performance.now() - started;
+            options.onResult?.(results[index]);
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(Math.max(1, options.concurrency ?? 4), jobs.length) }, worker));
+    return results;
+}
+async function runTest(fixture, test, repeatOverride, once) {
+    const repeat = repeatOverride ?? test.repeat ?? 1;
+    const runs = [];
+    for (let run = 1; run <= repeat; run++)
+        runs.push(await once(run));
+    const passed = runs.filter((result) => result.pass).length;
+    const passRate = passed / runs.length;
+    return { fixture, test: test.name, runs, passRate, pass: passRate >= (test.passRate ?? 1) };
+}
+async function runOnce(server, { file, path, test, run }, options) {
+    const result = { fixture: file, test: test.name, run, pass: true, turns: [] };
+    const { spend } = options;
+    const context = { fixture: path, ...(options.judge ? { judge: options.judge } : {}) };
+    // Usage on `finish` is what usage assertions read.
+    const handler = createChatHandler({ ...server, exposeUsage: true, storage: memoryStorage(), tools: withMocks(server.tools ?? [], test.mocks?.tools) });
+    const conversationId = `eval-${randomUUID()}`;
+    const conversation = [];
+    for (const turnSpec of test.turns) {
+        if (spend?.exceeded) {
+            result.pass = false;
+            result.error = `stopped before turn ${conversation.length + 1}: --max-cost ${dollars(spend.max ?? 0)} reached (${dollars(spend.spent)} spent)`;
+            break;
+        }
+        const turn = await sendTurn(handler, {
+            conversationId,
+            message: turnSpec.user,
+            ...(test.locale ? { locale: test.locale } : {}),
+            ...(test.pageContext ? { pageContext: test.pageContext } : {}),
+            ...(test.userData ? { userData: test.userData } : {}),
+        });
+        if (options.model)
+            turn.model = options.model;
+        const costUsd = costOf(options.model, turn.usage, options.prices);
+        if (costUsd !== undefined)
+            turn.costUsd = costUsd;
+        spend?.add(costUsd);
+        conversation.push(turn);
+        const checks = [];
+        for (const assertion of turnSpec.expect ?? []) {
+            const checked = await check(assertion, turn, conversation, context);
+            spend?.add(checked.costUsd);
+            checks.push(checked);
+        }
+        result.turns.push({ turn, results: checks });
+        if (checks.some((c) => !c.pass))
+            result.pass = false;
+        if (turn.error) {
+            result.pass = false;
+            result.error = `turn ${conversation.length} failed: ${turn.error}`;
+            break;
+        }
+    }
+    return result;
+}
+/** Sends one message and collects the reply's parts as a turn result. */
+export async function sendTurn(handler, body) {
+    const started = performance.now();
+    const turn = { user: String(body.message), text: '', toolCalls: [], sources: [], leads: [], latencyMs: 0 };
+    const start = await handler.startChat(body, undefined, { format: 'v1' });
+    if (!start.ok) {
+        turn.error = start.error.message ? `${start.error.code}: ${start.error.message}` : start.error.code;
+        turn.latencyMs = performance.now() - started;
+        return turn;
+    }
+    try {
+        for await (const part of start.parts) {
+            if (part.type === 'text-delta')
+                turn.text += part.delta;
+            else if (part.type === 'tool-call')
+                turn.toolCalls.push({ name: part.toolName, input: part.input });
+            else if (part.type === 'source')
+                turn.sources.push({ url: part.url, ...(part.title ? { title: part.title } : {}) });
+            else if (part.type === 'lead-captured')
+                turn.leads.push(part.lead);
+            else if (part.type === 'error')
+                turn.error = part.code;
+            else if (part.type === 'finish') {
+                turn.finishReason = part.reason;
+                if (part.usage)
+                    turn.usage = { inputTokens: part.usage.inputTokens, outputTokens: part.usage.outputTokens };
+                turn.latencyMs = performance.now() - started;
+            }
+        }
+    }
+    catch (error) {
+        // The server's HTTP layer turns this into an internal_error part; in
+        // process it arrives as a throw, such as a knowledge source that failed.
+        // It fails this test instead of stopping the whole run.
+        turn.error = `internal_error: ${error instanceof Error ? error.message : String(error)}`;
+    }
+    if (!turn.latencyMs)
+        turn.latencyMs = performance.now() - started;
+    return turn;
+}
+/** The config's tools, with mocked ones answering their fixed result instead of running. */
+export function withMocks(tools, mocks) {
+    if (!mocks || !Object.keys(mocks).length)
+        return tools;
+    const names = new Set(tools.map((tool) => tool.name));
+    const unknown = Object.keys(mocks).filter((name) => !names.has(name));
+    if (unknown.length)
+        throw new ConfigError(`Mocks for tools the server config doesn't have: ${unknown.join(', ')}`);
+    return tools.map((tool) => mocks[tool.name] ? Object.assign(Object.create(tool), { execute: async () => structuredClone(mocks[tool.name]) }) : tool);
+}
+/**
+ * In replay nothing reaches the network, so CI needs no keys. The built-in
+ * providers refuse to start without one, so give them a stand-in. Headers
+ * aren't part of what a recording is matched on.
+ */
+export function withoutKeys(server) {
+    const { provider } = server;
+    if (isAIProvider(provider) || provider.apiKey?.trim())
+        return server;
+    return { ...server, provider: { ...provider, apiKey: 'chatcops-replay' } };
+}
+//# sourceMappingURL=run.js.map

package/dist/test/run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/test/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAiD,MAAM,gBAAgB,CAAC;AAC5G,OAAO,EAAE,iBAAiB,EAA6B,MAAM,kBAAkB,CAAC;AAChF,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAE9C,OAAO,EAAE,MAAM,EAAE,OAAO,EAA2B,MAAM,WAAW,CAAC;AA6BrE;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAA4B,EAC5B,QAAsB,EACtB,UAAsB,EAAE;IAExB,IAAI,OAAO,CAAC,SAAS,EAAE,IAAI,KAAK,QAAQ;QAAE,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IACvE,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACpH,MAAM,OAAO,GAAiB,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;QACxB,OAAO,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,IAAI,EAAE,CAAC;YACrB,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;YACzC,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YAClC,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE;gBACjE,MAAM,IAAI,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,OAAO,CAAC,CAAC;gBACvE,IAAI,CAAC,OAAO,CAAC,SAAS;oBAAE,OAAO,IAAI,EAAE,CAAC;gBACtC,OAAO,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE;oBAC/E,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;wBACpB,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC;wBACnB,KAAK,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,SAAS,QAAQ,CAAC,MAAM,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;oBACvG,CAAC;oBACD,OAAO,KAAK,CAAC;gBACf,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;YACH,OAAO,CAAC,KAAK,CAAC,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;YACxD,OAAO,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QACrC,CAAC;IACH,CAAC,CAAC;IACF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;IAChH,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,OAAO,CACpB,OAAe,EACf,IAAc,EACd,cAAkC,EAClC,IAA6C;IAE7C,MAAM,MAAM,GAAG,cAAc,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;IAClD,MAAM,IAAI,GAAoB,EAAE,CAAC;IACjC,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,MAAM,EAAE,GAAG,EAAE;QAAE,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAC3D,MAAM,QAAQ,GAAG,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IACtC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,EAAE,CAAC;AAC9F,CAAC;AAED,KAAK,UAAU,OAAO,CACpB,MAA4B,EAC5B,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAA+D,EACtF,OAAmB;IAEnB,MAAM,MAAM,GAAkB,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IAC7F,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;IAC1B,MAAM,OAAO,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IACtF,mDAAmD;IACnD,MAAM,OAAO,GAAG,iBAAiB,CAAC,EAAE,GAAG,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;IACvJ,MAAM,cAAc,GAAG,QAAQ,UAAU,EAAE,EAAE,CAAC;IAC9C,MAAM,YAAY,GAAiB,EAAE,CAAC;IAEtC,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAClC,IAAI,KAAK,EAAE,QAAQ,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC;YACpB,MAAM,CAAC,KAAK,GAAG,uBAAuB,YAAY,CAAC,MAAM,GAAG,CAAC,gBAAgB,OAAO,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,aAAa,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC;YAC/I,MAAM;QACR,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE;YACnC,cAAc;YACd,OAAO,EAAE,QAAQ,CAAC,IAAI;YACtB,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/C,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9D,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtD,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,KAAK;YAAE,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC9C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAClE,IAAI,OAAO,KAAK,SAAS;YAAE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QAClD,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;QACpB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,MAAM,MAAM,GAAG,EAAE,CAAC;QAClB,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;YAC9C,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;YACpE,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvB,CAAC;QACD,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC7C,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAAE,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC;QACrD,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC;YACpB,MAAM,CAAC,KAAK,GAAG,QAAQ,YAAY,CAAC,MAAM,YAAY,IAAI,CAAC,KAAK,EAAE,CAAC;YACnE,MAAM;QACR,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,yEAAyE;AACzE,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAA6C,EAAE,IAA6B;IACzG,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,IAAI,GAAe,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IACvH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IACzE,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QACd,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC;QACpG,IAAI,CAAC,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QAC7C,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,CAAC;QACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK,CAAC,KAAgC,EAAE,CAAC;YAChE,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY;gBAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC;iBACnD,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW;gBAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;iBAC/F,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ;gBAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;iBAC/G,IAAI,IAAI,CAAC,IAAI,KAAK,eAAe;gBAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAA+B,CAAC,CAAC;iBACzF,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO;gBAAE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC;iBAClD,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAChC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC;gBAChC,IAAI,IAAI,CAAC,KAAK;oBAAE,IAAI,CAAC,KAAK,GAAG,EAAE,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;gBAC5G,IAAI,CAAC,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;YAC/C,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,qEAAqE;QACrE,yEAAyE;QACzE,wDAAwD;QACxD,IAAI,CAAC,KAAK,GAAG,mBAAmB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IAC3F,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,SAAS;QAAE,IAAI,CAAC,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;IAClE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,4FAA4F;AAC5F,MAAM,UAAU,SAAS,CAAC,KAAiB,EAAE,KAA6C;IACxF,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IACvD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACtD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IACtE,IAAI,OAAO,CAAC,MAAM;QAAE,MAAM,IAAI,WAAW,CAAC,mDAAmD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACnH,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACxB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAa,EAAE,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CACrI,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,MAA4B;IACtD,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;IAC5B,IAAI,YAAY,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE;QAAE,OAAO,MAAM,CAAC;IACrE,OAAO,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,EAAE,GAAG,QAAQ,EAAE,MAAM,EAAE,iBAAiB,EAAE,EAAE,CAAC;AAC7E,CAAC"}