oh-my-knowledge 0.0.1 → 0.19.0

package/dist/src/eval-workflows/evaluation-pipeline.js

@@ -0,0 +1,218 @@
+import { createHash } from 'node:crypto';
+import { existsSync, readFileSync } from 'node:fs';
+import { analyzeResults } from '../analysis/report-diagnostics.js';
+import { computeReportCoverage } from '../analysis/coverage-analyzer.js';
+import { computeReportGapRates } from '../analysis/gap-analyzer.js';
+import { aggregateReport, applyBlindMode, DEFAULT_OUTPUT_DIR, generateRunId, persistReport } from '../eval-core/evaluation-reporting.js';
+import { executeTasks, preflight } from '../eval-core/evaluation-execution.js';
+import { preflightDependencies, formatDependencyErrors } from '../eval-core/dependency-checker.js';
+import { createFileJobStore, DEFAULT_JOBS_DIR, } from '../server/job-store.js';
+import { stopAllServers } from '../inputs/mcp-resolver.js';
+import { buildEvaluationRequest, createFailedJob, createEvaluationRun, createQueuedJob, createSucceededJob, finalizeEvaluationRun, markJobRunning, failEvaluationRun, } from '../eval-core/evaluation-job.js';
+async function initializeEvaluationRunState({ samplesPath, skillDir, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, concurrency, timeoutMs, noCache, blind, project, owner, tags, runId, jobStore, persistJob, repeat, each, }) {
+    const request = buildEvaluationRequest({
+        samplesPath,
+        skillDir,
+        artifacts,
+        model,
+        judgeModel: noJudge ? null : judgeModel,
+        executor: executorName,
+        judgeExecutor: judgeExecutorName,
+        noJudge,
+        concurrency,
+        timeoutMs,
+        noCache,
+        dryRun: false,
+        blind,
+        project,
+        owner,
+        tags,
+        repeat,
+        each,
+    });
+    const createdAt = new Date().toISOString();
+    const { run: initialRun, startedAt } = createEvaluationRun(runId, createdAt);
+    const jobId = `job-${runId}`;
+    const resolvedJobStore = persistJob ? (jobStore ?? createFileJobStore(DEFAULT_JOBS_DIR)) : null;
+    const queuedJob = createQueuedJob({ jobId, request, createdAt });
+    if (resolvedJobStore)
+        await resolvedJobStore.save(jobId, queuedJob);
+    const runningJob = markJobRunning(queuedJob, runId, startedAt);
+    if (resolvedJobStore)
+        await resolvedJobStore.save(jobId, runningJob);
+    return { request, runId, jobId, createdAt, startedAt, initialRun, runningJob, resolvedJobStore };
+}
+function finalizeSuccessfulRun(state) {
+    const finishedAt = new Date().toISOString();
+    const run = finalizeEvaluationRun(state.initialRun, finishedAt);
+    const job = createSucceededJob({
+        jobId: state.jobId,
+        runId: state.runId,
+        reportId: state.runId,
+        request: state.request,
+        createdAt: state.createdAt,
+        startedAt: state.startedAt,
+        finishedAt,
+    });
+    return { run, job };
+}
+async function persistSuccessfulJob(state, job) {
+    if (state.resolvedJobStore) {
+        await state.resolvedJobStore.save(state.jobId, job);
+    }
+}
+async function persistFailedJob(state, err) {
+    const finishedAt = new Date().toISOString();
+    const failedJob = createFailedJob({
+        job: { ...state.runningJob, runId: state.runId, startedAt: state.startedAt, finishedAt: undefined },
+        error: err instanceof Error ? err.message : String(err),
+        finishedAt,
+    });
+    void failEvaluationRun(state.initialRun, finishedAt);
+    if (state.resolvedJobStore) {
+        await state.resolvedJobStore.save(state.jobId, failedJob);
+    }
+}
+/**
+ * Compute the mandatory test set watermark hash (spec §7.1). Returns the first
+ * 12 hex chars of SHA-256 over the samples file contents, or null if the file
+ * can't be read.
+ */
+function computeTestSetHash(samplesPath) {
+    if (!samplesPath || !existsSync(samplesPath))
+        return null;
+    try {
+        const content = readFileSync(samplesPath, 'utf-8');
+        return createHash('sha256').update(content).digest('hex').slice(0, 12);
+    }
+    catch {
+        return null;
+    }
+}
+function finalizeEvaluationReport({ report, results, artifacts, variantNames, blind, samplesPath, }) {
+    report.analysis = analyzeResults(report);
+    const hasToolData = Object.values(results).some((sampleResults) => (Object.values(sampleResults).some((variantResult) => variantResult.toolCalls && variantResult.toolCalls.length > 0)));
+    if (hasToolData) {
+        const artifactContents = Object.fromEntries(artifacts.map((artifact) => [artifact.name, artifact.content]));
+        const artifactCwds = Object.fromEntries(artifacts.map((artifact) => [artifact.name, artifact.cwd || null]));
+        const coverage = computeReportCoverage(report, artifactContents, artifactCwds);
+        if (Object.keys(coverage).length > 0) {
+            report.analysis.coverage = coverage;
+        }
+    }
+    // Gap rate computation runs on every successful report regardless of whether
+    // tool trace data is present — text-based signals (markers, hedging) still
+    // apply. The samples-file SHA is the mandatory watermark required by spec §7.1.
+    const gapReports = computeReportGapRates(report.results, variantNames);
+    if (Object.keys(gapReports).length > 0) {
+        const testSetHash = computeTestSetHash(samplesPath);
+        for (const variant of variantNames) {
+            const gr = gapReports[variant];
+            if (!gr)
+                continue;
+            gr.testSetPath = samplesPath;
+            gr.testSetHash = testSetHash;
+        }
+        report.analysis.gapReports = gapReports;
+    }
+    if (blind) {
+        applyBlindMode(report, variantNames, `${variantNames.join(',')}:${samplesPath}`);
+    }
+    return report;
+}
+export async function executeEvaluationPipeline({ samplesPath, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, blind = false, concurrency = 1, timeoutMs, noCache = false, jobStore = null, persistJob = true, onProgress = null, skipPreflight = false, verbose = false, retry = 0, existingResults, requires, layeredStats = false, repeat, each, }) {
+    const variantNames = artifacts.map((artifact) => artifact.name);
+    const runState = await initializeEvaluationRunState({
+        samplesPath,
+        skillDir,
+        artifacts,
+        model,
+        judgeModel,
+        noJudge,
+        executorName,
+        judgeExecutorName,
+        concurrency,
+        timeoutMs,
+        noCache,
+        blind,
+        project,
+        owner,
+        tags,
+        runId: generateRunId(variantNames),
+        jobStore,
+        persistJob,
+        repeat,
+        each,
+    });
+    try {
+        if (!skipPreflight) {
+            if (onProgress)
+                onProgress({ phase: 'preflight', jobId: runState.jobId });
+            await preflight(executor, model);
+            if (!noJudge)
+                await preflight(judgeExecutor, judgeModel);
+            // Dependency check: auto-extract from skill contents + merge explicit requires
+            const skillContents = artifacts.map((a) => a.content).filter((c) => typeof c === 'string');
+            const cwd = artifacts.find((a) => a.cwd)?.cwd || skillDir || process.cwd();
+            const depResult = await preflightDependencies(skillContents, samples, cwd, requires, artifacts);
+            if (!depResult.ok) {
+                throw new Error(formatDependencyErrors(depResult.missing));
+            }
+        }
+        const { results, totalCostUSD, skipped } = await executeTasks({
+            tasks,
+            executor,
+            judgeExecutor,
+            model,
+            judgeModel,
+            noJudge,
+            samplesPath,
+            concurrency,
+            timeoutMs,
+            noCache,
+            verbose,
+            onProgress,
+            retry,
+            existingResults,
+        });
+        if (skipped > 0 && onProgress) {
+            onProgress({ phase: 'done', completed: tasks.length, total: tasks.length, sample_id: '', variant: '', skipped: true });
+        }
+        const { run, job } = finalizeSuccessfulRun(runState);
+        const report = finalizeEvaluationReport({
+            report: aggregateReport({
+                runId: runState.runId,
+                variants: variantNames,
+                model,
+                judgeModel,
+                noJudge,
+                executorName,
+                samples,
+                tasks,
+                results,
+                totalCostUSD,
+                artifacts,
+                request: runState.request,
+                run,
+                job,
+                layeredStats,
+            }),
+            results,
+            artifacts,
+            variantNames,
+            blind,
+            samplesPath,
+        });
+        const filePath = persistReport(report, outputDir);
+        await persistSuccessfulJob(runState, job);
+        return { report, filePath };
+    }
+    catch (err) {
+        await persistFailedJob(runState, err);
+        throw err;
+    }
+    finally {
+        await stopAllServers();
+    }
+}
+//# sourceMappingURL=evaluation-pipeline.js.map

package/dist/src/eval-workflows/evaluation-pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluation-pipeline.js","sourceRoot":"","sources":["../../../src/eval-workflows/evaluation-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AACzE,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,kBAAkB,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AACzI,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,sCAAsC,CAAC;AAC/E,OAAO,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,oCAAoC,CAAC;AAEnG,OAAO,EACL,kBAAkB,EAClB,gBAAgB,GACjB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EACL,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,eAAe,EACf,kBAAkB,EAClB,qBAAqB,EACrB,cAAc,EACd,iBAAiB,GAClB,MAAM,gCAAgC,CAAC;AA4BxC,KAAK,UAAU,4BAA4B,CAAC,EAC1C,WAAW,EACX,QAAQ,EACR,SAAS,EACT,KAAK,EACL,UAAU,EACV,OAAO,EACP,YAAY,EACZ,iBAAiB,EACjB,WAAW,EACX,SAAS,EACT,OAAO,EACP,KAAK,EACL,OAAO,EACP,KAAK,EACL,IAAI,EACJ,KAAK,EACL,QAAQ,EACR,UAAU,EACV,MAAM,EACN,IAAI,GAsBL;IACC,MAAM,OAAO,GAAG,sBAAsB,CAAC;QACrC,WAAW;QACX,QAAQ;QACR,SAAS;QACT,KAAK;QACL,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU;QACvC,QAAQ,EAAE,YAAY;QACtB,aAAa,EAAE,iBAAiB;QAChC,OAAO;QACP,WAAW;QACX,SAAS;QACT,OAAO;QACP,MAAM,EAAE,KAAK;QACb,KAAK;QACL,OAAO;QACP,KAAK;QACL,IAAI;QACJ,MAAM;QACN,IAAI;KACL,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,mBAAmB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC7E,MAAM,KAAK,GAAG,OAAO,KAAK,EAAE,CAAC;IAC7B,MAAM,gBAAgB,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,kBAAkB,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAChG,MAAM,SAAS,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;IACjE,IAAI,gBAAgB;QAAE,MAAM,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IACpE,MAAM,UAAU,GAAG,cAAc,CAAC,SAAS,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;IAC/D,IAAI,gBAAgB;QAAE,MAAM,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IACrE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,CAAC;AACnG,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAyB;IACtD,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC5C,MAAM,GAAG,GAAG,qBAAqB,CAAC,KAAK,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IAChE,MAAM,GAAG,GAAG,kBAAkB,CAAC;QAC7B,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,QAAQ,EAAE,KAAK,CAAC,KAAK;QACrB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,UAAU;KACX,CAAC,CAAC;IACH,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;AACtB,CAAC;AAED,KAAK,UAAU,oBAAoB,CAAC,KAAyB,EAAE,GAAkB;IAC/E,IAAI,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC3B,MAAM,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACtD,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,KAAyB,EAAE,GAAY;IACrE,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC5C,MAAM,SAAS,GAAG,eAAe,CAAC;QAChC,GAAG,EAAE,EAAE,GAAG,KAAK,CAAC,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE;QACnG,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;QACvD,UAAU;KACX,CAAC,CAAC;IACH,KAAK,iBAAiB,CAAC,KAAK,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IACrD,IAAI,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC3B,MAAM,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC5D,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,WAAmB;IAC7C,IAAI,CAAC,WAAW,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1D,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACnD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,wBAAwB,CAAC,EAChC,MAAM,EACN,OAAO,EACP,SAAS,EACT,YAAY,EACZ,KAAK,EACL,WAAW,GAQZ;IACC,MAAM,CAAC,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IAEzC,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,EAAE,CAAC,CACjE,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,EAAE,CAAC,aAAa,CAAC,SAAS,IAAI,aAAa,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CACpH,CAAC,CAAC;IACH,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,gBAAgB,GAAG,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC5G,MAAM,YAAY,GAAG,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5G,MAAM,QAAQ,GAAG,qBAAqB,CAAC,MAAM,EAAE,gBAAgB,EAAE,YAAY,CAAC,CAAC;QAC/E,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,MAAM,CAAC,QAAS,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACvC,CAAC;IACH,CAAC;IAED,6EAA6E;IAC7E,2EAA2E;IAC3E,gFAAgF;IAChF,MAAM,UAAU,GAAG,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACvE,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,WAAW,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;QACpD,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;YACnC,MAAM,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;YAC/B,IAAI,CAAC,EAAE;gBAAE,SAAS;YAClB,EAAE,CAAC,WAAW,GAAG,WAAW,CAAC;YAC7B,EAAE,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,CAAC;QACD,MAAM,CAAC,QAAS,CAAC,UAAU,GAAG,UAAU,CAAC;IAC3C,CAAC;IAED,IAAI,KAAK,EAAE,CAAC;QACV,cAAc,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC,CAAC;IACnF,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAsCD,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,EAC9C,WAAW,EACX,QAAQ,EACR,OAAO,EACP,KAAK,EACL,SAAS,EACT,KAAK,EACL,UAAU,EACV,OAAO,EACP,YAAY,EACZ,iBAAiB,EACjB,QAAQ,EACR,aAAa,EACb,SAAS,GAAG,kBAAkB,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,KAAK,GAAG,KAAK,EACb,WAAW,GAAG,CAAC,EACf,SAAS,EACT,OAAO,GAAG,KAAK,EACf,QAAQ,GAAG,IAAI,EACf,UAAU,GAAG,IAAI,EACjB,UAAU,GAAG,IAAI,EACjB,aAAa,GAAG,KAAK,EACrB,OAAO,GAAG,KAAK,EACf,KAAK,GAAG,CAAC,EACT,eAAe,EACf,QAAQ,EACR,YAAY,GAAG,KAAK,EACpB,MAAM,EACN,IAAI,GACsB;IAC1B,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,MAAM,4BAA4B,CAAC;QAClD,WAAW;QACX,QAAQ;QACR,SAAS;QACT,KAAK;QACL,UAAU;QACV,OAAO;QACP,YAAY;QACZ,iBAAiB;QACjB,WAAW;QACX,SAAS;QACT,OAAO;QACP,KAAK;QACL,OAAO;QACP,KAAK;QACL,IAAI;QACJ,KAAK,EAAE,aAAa,CAAC,YAAY,CAAC;QAClC,QAAQ;QACR,UAAU;QACV,MAAM;QACN,IAAI;KACL,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,IAAI,UAAU;gBAAE,UAAU,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;YAC1E,MAAM,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACjC,IAAI,CAAC,OAAO;gBAAE,MAAM,SAAS,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC;YAEzD,+EAA+E;YAC/E,MAAM,aAAa,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;YACxG,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,QAAQ,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;YAC3E,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC,aAAa,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;YAChG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAED,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,GAAG,MAAM,YAAY,CAAC;YAC5D,KAAK;YACL,QAAQ;YACR,aAAa;YACb,KAAK;YACL,UAAU;YACV,OAAO;YACP,WAAW;YACX,WAAW;YACX,SAAS;YACT,OAAO;YACP,OAAO;YACP,UAAU;YACV,KAAK;YACL,eAAe;SAChB,CAAC,CAAC;QACH,IAAI,OAAO,GAAG,CAAC,IAAI,UAAU,EAAE,CAAC;YAC9B,UAAU,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACzH,CAAC;QAED,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,wBAAwB,CAAC;YACtC,MAAM,EAAE,eAAe,CAAC;gBACtB,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,QAAQ,EAAE,YAAY;gBACtB,KAAK;gBACL,UAAU;gBACV,OAAO;gBACP,YAAY;gBACZ,OAAO;gBACP,KAAK;gBACL,OAAO;gBACP,YAAY;gBACZ,SAAS;gBACT,OAAO,EAAE,QAAQ,CAAC,OAAO;gBACzB,GAAG;gBACH,GAAG;gBACH,YAAY;aACb,CAAC;YACF,OAAO;YACP,SAAS;YACT,YAAY;YACZ,KAAK;YACL,WAAW;SACZ,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAClD,MAAM,oBAAoB,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC1C,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IAC9B,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,gBAAgB,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACtC,MAAM,GAAG,CAAC;IACZ,CAAC;YAAS,CAAC;QACT,MAAM,cAAc,EAAE,CAAC;IACzB,CAAC;AACH,CAAC"}

package/dist/src/eval-workflows/evaluation-preparation.d.ts

@@ -0,0 +1,64 @@
+import type { DependencyRequirements } from '../eval-core/dependency-checker.js';
+import type { Artifact, Sample, Task, VariantSpec } from '../types.js';
+export interface PreparedEvaluationRun {
+    samples: Sample[];
+    artifacts: Artifact[];
+    tasks: Task[];
+    variantNames: string[];
+    requires?: DependencyRequirements;
+}
+export declare function prepareEvaluationRun({ samplesPath, skillDir, variantSpecs, artifacts, dryRun, mcpConfig, }: {
+    samplesPath: string;
+    skillDir: string;
+    variantSpecs: VariantSpec[];
+    artifacts?: Artifact[];
+    dryRun: boolean;
+    mcpConfig?: string;
+}): Promise<PreparedEvaluationRun>;
+export declare function buildDryRunTaskReport({ model, judgeModel, executorName, samplesPath, skillDir, tasks, variantNames, }: {
+    model: string;
+    judgeModel: string;
+    executorName: string;
+    samplesPath: string;
+    skillDir: string;
+    tasks: Task[];
+    variantNames: string[];
+}): {
+    dryRun: true;
+    model: string;
+    judgeModel: string;
+    variants: string[];
+    executor: string;
+    samplesPath: string;
+    skillDir: string;
+    totalTasks: number;
+    tasks: {
+        sample_id: string;
+        variant: string;
+        artifactKind: import("../types.js").ArtifactKind;
+        artifactSource: "baseline" | "variant-name" | "file-path" | "git" | "inline" | "custom";
+        executionStrategy: import("../types.js").ExecutionStrategyKind;
+        experimentType: import("../types.js").ExperimentType;
+        experimentRole: import("../types.js").ExperimentRole;
+        cwd: string | null;
+        promptPreview: string;
+        hasRubric: boolean;
+        hasAssertions: boolean;
+        hasDimensions: boolean;
+        hasSystem: boolean;
+    }[];
+};
+export declare function buildDryRunEachArtifacts(skillEntries: Array<{
+    name: string;
+    skillPath: string;
+    samplesPath: string;
+}>): {
+    artifacts: {
+        name: string;
+        samplesPath: string;
+        sampleCount: number;
+        taskCount: number;
+    }[];
+    totalTasks: number;
+};
+//# sourceMappingURL=evaluation-preparation.d.ts.map

package/dist/src/eval-workflows/evaluation-preparation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluation-preparation.d.ts","sourceRoot":"","sources":["../../../src/eval-workflows/evaluation-preparation.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,oCAAoC,CAAC;AACjF,OAAO,KAAK,EAAE,QAAQ,EAAc,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAEnF,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,QAAQ,CAAC,EAAE,sBAAsB,CAAC;CACnC;AAED,wBAAsB,oBAAoB,CAAC,EACzC,WAAW,EACX,QAAQ,EACR,YAAY,EACZ,SAAS,EACT,MAAM,EACN,SAAS,GACV,EAAE;IACD,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IACvB,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAyCjC;AAED,wBAAgB,qBAAqB,CAAC,EACpC,KAAK,EACL,UAAU,EACV,YAAY,EACZ,WAAW,EACX,QAAQ,EACR,KAAK,EACL,YAAY,GACb,EAAE;IACD,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;;;;;;;;;;;;;;;;;;;;;;;;EA6BA;AAED,wBAAgB,wBAAwB,CAAC,YAAY,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,CAAC;;;;;;;;EAerH"}

package/dist/src/eval-workflows/evaluation-preparation.js

@@ -0,0 +1,90 @@
+import { resolve } from 'node:path';
+import { buildTasksFromArtifacts } from '../eval-core/task-planner.js';
+import { loadSamples } from '../inputs/load-samples.js';
+import { resolveArtifacts } from '../inputs/skill-loader.js';
+import { buildVariantConfig } from '../eval-core/execution-strategy.js';
+import { loadMcpConfig, resolveMcpUrls } from '../inputs/mcp-resolver.js';
+import { resolveUrls } from '../inputs/url-fetcher.js';
+export async function prepareEvaluationRun({ samplesPath, skillDir, variantSpecs, artifacts, dryRun, mcpConfig, }) {
+    const { samples, requires } = loadSamples(samplesPath);
+    // Build expressions from specs (preserving order) and resolve to artifacts.
+    const variantExpressions = variantSpecs.map((spec) => spec.expr);
+    const resolvedArtifacts = artifacts || resolveArtifacts(resolve(skillDir), variantExpressions);
+    // Attach experimentRole to each artifact by matching spec.name to artifact.name.
+    const roleByName = {};
+    for (const spec of variantSpecs)
+        roleByName[spec.name] = spec.role;
+    for (const artifact of resolvedArtifacts) {
+        if (artifact.experimentRole)
+            continue; // preserve if already set (e.g. each-workflow)
+        const role = roleByName[artifact.name];
+        if (role)
+            artifact.experimentRole = role;
+    }
+    if (!dryRun) {
+        const mcpServers = loadMcpConfig(mcpConfig);
+        const mcpResolved = mcpServers ? await resolveMcpUrls(samples, mcpServers) : new Set();
+        await resolveUrls(samples, mcpResolved);
+    }
+    if (resolvedArtifacts.length === 0) {
+        throw new Error(`未发现任何 variant。请检查：\n`
+            + `  1. skill 目录是否存在：${resolve(skillDir)}\n`
+            + `  2. 目录下是否有 .md 文件或含 SKILL.md 的子目录\n`
+            + `  3. 通过 --control / --treatment 显式声明 variant 与角色，或用 --config eval.yaml`);
+    }
+    const tasks = buildTasksFromArtifacts(samples, resolvedArtifacts);
+    const variantNames = resolvedArtifacts.map((artifact) => artifact.name);
+    return {
+        samples,
+        artifacts: resolvedArtifacts,
+        tasks,
+        variantNames,
+        requires,
+    };
+}
+export function buildDryRunTaskReport({ model, judgeModel, executorName, samplesPath, skillDir, tasks, variantNames, }) {
+    return {
+        dryRun: true,
+        model,
+        judgeModel,
+        variants: variantNames,
+        executor: executorName,
+        samplesPath,
+        skillDir,
+        totalTasks: tasks.length,
+        tasks: tasks.map((task) => {
+            const config = buildVariantConfig(task.artifact);
+            return {
+                sample_id: task.sample_id,
+                variant: task.variant,
+                artifactKind: task.artifact.kind,
+                artifactSource: task.artifact.source,
+                executionStrategy: config.executionStrategy,
+                experimentType: config.experimentType,
+                experimentRole: config.experimentRole,
+                cwd: task.cwd,
+                promptPreview: task.prompt.slice(0, 100),
+                hasRubric: Boolean(task.rubric),
+                hasAssertions: Boolean(task.assertions?.length),
+                hasDimensions: Boolean(task.dimensions && Object.keys(task.dimensions).length),
+                hasSystem: Boolean(task.artifactContent),
+            };
+        }),
+    };
+}
+export function buildDryRunEachArtifacts(skillEntries) {
+    const artifacts = skillEntries.map((entry) => {
+        const { samples } = loadSamples(entry.samplesPath);
+        return {
+            name: entry.name,
+            samplesPath: entry.samplesPath,
+            sampleCount: samples.length,
+            taskCount: samples.length * 2,
+        };
+    });
+    return {
+        artifacts,
+        totalTasks: artifacts.reduce((sum, artifact) => sum + artifact.taskCount, 0),
+    };
+}
+//# sourceMappingURL=evaluation-preparation.js.map

package/dist/src/eval-workflows/evaluation-preparation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluation-preparation.js","sourceRoot":"","sources":["../../../src/eval-workflows/evaluation-preparation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,oCAAoC,CAAC;AACxE,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC1E,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAYvD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,EACzC,WAAW,EACX,QAAQ,EACR,YAAY,EACZ,SAAS,EACT,MAAM,EACN,SAAS,GAQV;IACC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;IAEvD,4EAA4E;IAC5E,MAAM,kBAAkB,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjE,MAAM,iBAAiB,GAAG,SAAS,IAAI,gBAAgB,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,kBAAkB,CAAC,CAAC;IAE/F,iFAAiF;IACjF,MAAM,UAAU,GAAwC,EAAE,CAAC;IAC3D,KAAK,MAAM,IAAI,IAAI,YAAY;QAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;IACnE,KAAK,MAAM,QAAQ,IAAI,iBAAiB,EAAE,CAAC;QACzC,IAAI,QAAQ,CAAC,cAAc;YAAE,SAAS,CAAE,+CAA+C;QACvF,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,IAAI;YAAE,QAAQ,CAAC,cAAc,GAAG,IAAI,CAAC;IAC3C,CAAC;IAED,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,UAAU,GAAsB,aAAa,CAAC,SAAS,CAAC,CAAC;QAC/D,MAAM,WAAW,GAAG,UAAU,CAAC,CAAC,CAAC,MAAM,cAAc,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,EAAU,CAAC;QAC/F,MAAM,WAAW,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CACb,sBAAsB;cACpB,qBAAqB,OAAO,CAAC,QAAQ,CAAC,IAAI;cAC1C,sCAAsC;cACtC,wEAAwE,CAC3E,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,uBAAuB,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;IAClE,MAAM,YAAY,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAExE,OAAO;QACL,OAAO;QACP,SAAS,EAAE,iBAAiB;QAC5B,KAAK;QACL,YAAY;QACZ,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,EACpC,KAAK,EACL,UAAU,EACV,YAAY,EACZ,WAAW,EACX,QAAQ,EACR,KAAK,EACL,YAAY,GASb;IACC,OAAO;QACL,MAAM,EAAE,IAAa;QACrB,KAAK;QACL,UAAU;QACV,QAAQ,EAAE,YAAY;QACtB,QAAQ,EAAE,YAAY;QACtB,WAAW;QACX,QAAQ;QACR,UAAU,EAAE,KAAK,CAAC,MAAM;QACxB,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACxB,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACjD,OAAO;gBACL,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;gBAChC,cAAc,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBACpC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;gBAC3C,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;gBACxC,SAAS,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;gBAC/B,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC;gBAC/C,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,UAAU,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;gBAC9E,SAAS,EAAE,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC;aACzC,CAAC;QACJ,CAAC,CAAC;KACH,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,YAA6E;IACpH,MAAM,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QAC3C,MAAM,EAAE,OAAO,EAAE,GAAG,WAAW,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACnD,OAAO;YACL,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,WAAW,EAAE,OAAO,CAAC,MAAM;YAC3B,SAAS,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;SAC9B,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,SAAS;QACT,UAAU,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;KAC7E,CAAC;AACJ,CAAC"}

package/dist/src/eval-workflows/run-evaluation.d.ts

@@ -0,0 +1,111 @@
+import type { Artifact, JobStore, ProgressCallback, Report, VarianceData, VariantSpec } from '../types.js';
+export interface SkillProgressInfo {
+    phase: string;
+    skill: string;
+    current: number;
+    total: number;
+}
+interface CommonEvaluationOptions {
+    model?: string;
+    judgeModel?: string;
+    outputDir?: string | null;
+    project?: string;
+    owner?: string;
+    tags?: string[];
+    noJudge?: boolean;
+    concurrency?: number;
+    timeoutMs?: number;
+    executorName?: string;
+    judgeExecutorName?: string;
+    jobStore?: JobStore | null;
+    persistJob?: boolean;
+    onProgress?: ProgressCallback | null;
+    skipPreflight?: boolean;
+    mcpConfig?: string;
+    verbose?: boolean;
+    layeredStats?: boolean;
+    /** --repeat N. 1 表示单次(默认); > 1 时在 runMultiple 层聚合 variance。
+     *  记入 report.meta.request.repeat 让 meta 如实反映用户输入。 */
+    repeat?: number;
+    /** --each 模式标记, true 表示当前评测是 each 批量流程(每个 skill 独立对比 baseline)。
+     *  记入 report.meta.request.each。 */
+    each?: boolean;
+}
+export interface RunEvaluationOptions extends CommonEvaluationOptions {
+    samplesPath: string;
+    skillDir: string;
+    variantSpecs?: VariantSpec[];
+    artifacts?: Artifact[];
+    dryRun?: boolean;
+    blind?: boolean;
+    noCache?: boolean;
+    retry?: number;
+    resume?: string;
+}
+export interface RunEachEvaluationOptions extends CommonEvaluationOptions {
+    skillDir: string;
+    dryRun?: boolean;
+    onSkillProgress?: ((info: SkillProgressInfo) => void) | null;
+}
+export interface RunMultipleOptions extends RunEvaluationOptions {
+    repeat?: number;
+    onRepeatProgress?: ((info: {
+        run: number;
+        total: number;
+    }) => void) | null;
+}
+interface DryRunTask {
+    sample_id: string;
+    variant: string;
+    artifactKind: Artifact['kind'];
+    artifactSource: Artifact['source'];
+    executionStrategy: string;
+    experimentType: string;
+    experimentRole: string;
+    cwd: string | null;
+    promptPreview: string;
+    hasRubric: boolean;
+    hasAssertions: boolean;
+    hasDimensions: boolean;
+    hasSystem: boolean;
+}
+interface DryRunBase {
+    dryRun: true;
+    model: string;
+    judgeModel: string;
+    executor: string;
+    skillDir: string;
+    totalTasks: number;
+}
+export interface DryRunReport extends DryRunBase {
+    variants: string[];
+    samplesPath: string;
+    tasks: DryRunTask[];
+}
+export declare function runEvaluation({ samplesPath, skillDir, variantSpecs, artifacts, model, judgeModel, outputDir, project, owner, tags, noJudge, dryRun, blind, concurrency, timeoutMs, noCache, executorName, judgeExecutorName, jobStore, persistJob, onProgress, skipPreflight, mcpConfig, verbose, retry, resume, layeredStats, repeat, each, }: RunEvaluationOptions): Promise<{
+    report: Report | DryRunReport;
+    filePath: string | null;
+}>;
+interface DryRunEachSkill {
+    name: string;
+    samplesPath: string;
+    sampleCount: number;
+    taskCount: number;
+}
+export interface DryRunEachReport extends DryRunBase {
+    each: true;
+    totalArtifacts: number;
+    artifacts: DryRunEachSkill[];
+}
+export declare function buildVarianceData(runs: Report[]): VarianceData | null;
+export declare function runEachEvaluation({ skillDir, model, judgeModel, outputDir, project, owner, tags, noJudge, dryRun, concurrency, timeoutMs, executorName, judgeExecutorName, jobStore, persistJob, onProgress, onSkillProgress, skipPreflight, mcpConfig, verbose, repeat, }: RunEachEvaluationOptions): Promise<{
+    report: Report | DryRunEachReport;
+    filePath: string | null;
+}>;
+export declare function runMultiple({ repeat, onRepeatProgress, ...config }: RunMultipleOptions): Promise<{
+    report: Report;
+    aggregated: VarianceData | null;
+    filePath: string | null;
+}>;
+export {};
+//# sourceMappingURL=run-evaluation.d.ts.map

package/dist/src/eval-workflows/run-evaluation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-evaluation.d.ts","sourceRoot":"","sources":["../../../src/eval-workflows/run-evaluation.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EACV,QAAQ,EAER,QAAQ,EACR,gBAAgB,EAChB,MAAM,EAGN,YAAY,EAGZ,WAAW,EAGZ,MAAM,aAAa,CAAC;AAErB,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,UAAU,uBAAuB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC;IAC3B,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACrC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAIlB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;yDACqD;IACrD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;uCACmC;IACnC,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,oBAAqB,SAAQ,uBAAuB;IACnE,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,WAAW,EAAE,CAAC;IAC7B,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IACvB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,wBAAyB,SAAQ,uBAAuB;IACvE,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,eAAe,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,iBAAiB,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;CAC9D;AAED,MAAM,WAAW,kBAAmB,SAAQ,oBAAoB;IAC9D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gBAAgB,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;CAC5E;AAED,UAAU,UAAU;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC/B,cAAc,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACnC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,OAAO,CAAC;IACvB,aAAa,EAAE,OAAO,CAAC;IACvB,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,UAAU,UAAU;IAClB,MAAM,EAAE,IAAI,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,YAAa,SAAQ,UAAU;IAC9C,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB;AAED,wBAAsB,aAAa,CAAC,EAClC,WAAW,EACX,QAAQ,EACR,YAAiB,EACjB,SAAS,EACT,KAAqB,EACrB,UAAwB,EACxB,SAA8B,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,OAAe,EACf,MAAc,EACd,KAAa,EACb,WAAe,EACf,SAAS,EACT,OAAe,EACf,YAAuB,EACvB,iBAAiB,EACjB,QAAe,EACf,UAAiB,EACjB,UAAiB,EACjB,aAAqB,EACrB,SAAS,EACT,OAAe,EACf,KAAS,EACT,MAAM,EACN,YAAoB,EACpB,MAAM,EACN,IAAI,GACL,EAAE,oBAAoB,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAgF5F;AAED,UAAU,eAAe;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,gBAAiB,SAAQ,UAAU;IAClD,IAAI,EAAE,IAAI,CAAC;IACX,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,eAAe,EAAE,CAAC;CAC9B;AA6CD,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,YAAY,GAAG,IAAI,CAuErE;AAED,wBAAsB,iBAAiB,CAAC,EACtC,QAAQ,EACR,KAAqB,EACrB,UAAwB,EACxB,SAA8B,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,OAAe,EACf,MAAc,EACd,WAAe,EACf,SAAS,EACT,YAAuB,EACvB,iBAAiB,EACjB,QAAe,EACf,UAAiB,EACjB,UAAiB,EACjB,eAAsB,EACtB,aAAqB,EACrB,SAAS,EACT,OAAe,EACf,MAAM,GACP,EAAE,wBAAwB,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,GAAG,gBAAgB,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAuDpG;AAED,wBAAsB,WAAW,CAAC,EAAE,MAAU,EAAE,gBAAgB,EAAE,GAAG,MAAM,EAAE,EAAE,kBAAkB;;;;GAyBhG"}