@superblocksteam/vite-plugin-file-sync 2.0.18-next.3 → 2.0.18-next.31

package/dist/ai-service/evals/helpers/index.js

@@ -3,18 +3,25 @@ import { readFile } from "node:fs/promises";
 import { mkdir, mkdtemp, rm, writeFile, symlink } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
+import { parse } from "@babel/parser";
 import superblocksLinter from "@superblocksteam/linter";
 import { generateText } from "ai";
 import ddTrace, {} from "dd-trace";
 import { ESLint } from "eslint";
-import { expect, beforeEach, afterEach } from "vitest";
+import { expect } from "vitest";
 import packageJson from "../../../../package.json" with { type: "json" };
+import { traverse } from "../../../traverse.js";
 import { OperationQueue } from "../../../util/operation-queue.js";
 import { AppShell } from "../../app-interface/shell.js";
 import { BoltArtifactProcessor } from "../../artifacts/bolt.js";
 import { Paths } from "../../const.js";
 import { matchMultiplePaths, matchPath } from "../../transform/shared.js";
 export const waitForNextTick = () => new Promise((resolve) => setTimeout(resolve, 1000));
+export const logLLMResponse = (result) => {
+    console.log("====== LLM Response text:");
+    console.log(result.text);
+    console.log("====== End LLM Response text:");
+};
 // Helper function to sanitize errors and prevent logging system prompts
 export function sanitizeErrorForTests(error) {
     if (typeof error !== "object" || error === null) {
@@ -51,12 +58,35 @@ export async function wrapLLM(fn, options) {
         process.env.CI !== "true") {
         console.log(`Using dev fixture: ${options.devFixture} for test: ${expect.getState().currentTestName}`);
         const result = await readFile(path.resolve(__dirname, "../", options.devFixture), "utf8");
-        return { text: result };
+        // Return without spanContext when using fixtures - no Datadog reporting
+        return { result: { text: result } };
     }
     const { llmobs } = ddTrace;
     const testName = expect.getState().currentTestName ?? "unknown-vitest-test";
-    const wrappedFn = llmobs.wrap({ name: `test:${testName}`, kind: "llm", ...options?.spanOptions }, fn);
-    return await wrappedFn();
+    let capturedSpanContext = undefined;
+    // Use llmobs.trace instead of wrap to get proper span context
+    const result = await llmobs.trace({ name: `test:${testName}`, kind: "llm", ...options?.spanOptions }, async () => {
+        capturedSpanContext = llmobs.exportSpan();
+        const fnResult = await fn();
+        // Annotate the span with input/output data if callback provided
+        if (options?.onResult) {
+            const annotationData = options.onResult(fnResult);
+            llmobs.annotate({
+                inputData: annotationData.inputData,
+                outputData: annotationData.outputData,
+                metrics: annotationData.metrics,
+                ...(annotationData.metadata
+                    ? { metadata: annotationData.metadata }
+                    : {}),
+            });
+        }
+        // Run evaluation inside the span context
+        if (options?.onEvaluation) {
+            await options.onEvaluation(fnResult);
+        }
+        return fnResult;
+    });
+    return { result, spanContext: capturedSpanContext };
 }
 export function extractArtifacts(text) {
     const artifactProcessor = new BoltArtifactProcessor();
@@ -71,6 +101,98 @@ export async function readDemoAppArtifacts(patterns = ["**/*.{ts,tsx,yaml,css}",
     });
     return await appShell.readFiles(patterns);
 }
+export async function readAppArtifacts(appPath, patterns = ["**/*.{ts,tsx,yaml,css}", "package.json"]) {
+    // Get default app artifacts first
+    const defaultArtifacts = await readDemoAppArtifacts(patterns);
+    // Get specific app artifacts
+    const fullAppPath = path.resolve(__dirname, "../input-context", appPath);
+    const fsOperationQueue = new OperationQueue();
+    const appShell = new AppShell({
+        appRootDirPath: fullAppPath,
+        fsOperationQueue,
+    });
+    const specificArtifacts = await appShell.readFiles(patterns);
+    // Create a map for easy merging, keyed by file path
+    const artifactMap = new Map();
+    // Add default artifacts first
+    for (const artifact of defaultArtifacts) {
+        artifactMap.set(artifact.filePath, artifact);
+    }
+    // Add specific artifacts, overriding defaults if they exist
+    for (const artifact of specificArtifacts) {
+        artifactMap.set(artifact.filePath, artifact);
+    }
+    // Return merged artifacts as array
+    return Array.from(artifactMap.values());
+}
+export async function createComponentNodePath(appPath, filePath, criteria) {
+    const artifacts = await readAppArtifacts(appPath);
+    const targetArtifact = artifacts.find((a) => a.filePath === filePath);
+    if (!targetArtifact) {
+        return criteria.findAll ? [] : undefined;
+    }
+    const ast = parse(targetArtifact.content, {
+        sourceType: "module",
+        plugins: ["jsx", "typescript"],
+    });
+    const matches = [];
+    const targetIndex = criteria.matchIndex ?? 0;
+    traverse(ast, {
+        JSXOpeningElement(path) {
+            if (path.node.name.type === "JSXIdentifier" &&
+                path.node.name.name === criteria.tagName) {
+                let isMatch = true;
+                // Check bind attribute if specified
+                if (criteria.bindName) {
+                    const bindAttr = path.node.attributes.find((attr) => attr.type === "JSXAttribute" &&
+                        attr.name.type === "JSXIdentifier" &&
+                        attr.name.name === "bind" &&
+                        attr.value?.type === "JSXExpressionContainer" &&
+                        attr.value.expression.type === "Identifier" &&
+                        attr.value.expression.name === criteria.bindName);
+                    if (!bindAttr)
+                        isMatch = false;
+                }
+                // Check element ID if specified
+                if (criteria.elementId && isMatch) {
+                    const idAttr = path.node.attributes.find((attr) => attr.type === "JSXAttribute" &&
+                        attr.name.type === "JSXIdentifier" &&
+                        attr.name.name === "data-sb-id" &&
+                        attr.value?.type === "StringLiteral" &&
+                        attr.value.value === criteria.elementId);
+                    if (!idAttr)
+                        isMatch = false;
+                }
+                // Check additional attribute filters if specified
+                if (criteria.attributeFilters && isMatch) {
+                    for (const [attrName, attrValue] of Object.entries(criteria.attributeFilters)) {
+                        const attr = path.node.attributes.find((attr) => attr.type === "JSXAttribute" &&
+                            attr.name.type === "JSXIdentifier" &&
+                            attr.name.name === attrName);
+                        if (!attr ||
+                            (attr.type === "JSXAttribute" &&
+                                attr.value?.type === "StringLiteral" &&
+                                attr.value.value !== attrValue)) {
+                            isMatch = false;
+                            break;
+                        }
+                    }
+                }
+                if (isMatch) {
+                    matches.push(path);
+                    // If we're not finding all matches and we've reached our target index, stop
+                    if (!criteria.findAll && matches.length > targetIndex) {
+                        path.stop();
+                    }
+                }
+            }
+        },
+    });
+    if (criteria.findAll) {
+        return matches;
+    }
+    return matches[targetIndex] || undefined;
+}
 async function writeArtifacts(artifacts, tempDir) {
     return Promise.all(artifacts.map(async (artifact) => {
         const fullPath = path.join(tempDir, artifact.filePath);
@@ -88,7 +210,13 @@ export function groupArtifacts(artifacts) {
         const pageScopeMatch = pageScopeMatcher(file.filePath);
         const apiPathMatch = apiPathMatcher(file.filePath);
         const componentPathsMatch = componentPathsMatcher(file.filePath);
-        if (file.filePath === Paths.PackageJson) {
+        if (file.type === "component") {
+            acc.components.push({
+                ...file,
+                params: {},
+            });
+        }
+        else if (file.filePath === Paths.PackageJson) {
             acc.packageJson = file;
         }
         else if (pagePathMatch.matched) {
@@ -110,7 +238,7 @@ export function groupArtifacts(artifacts) {
             });
         }
         else if (componentPathsMatch.matched) {
-            acc.components.push({
+            acc.customComponents.push({
                 ...file,
                 params: componentPathsMatch.params,
             });
@@ -125,25 +253,22 @@ export function groupArtifacts(artifacts) {
         pages: [],
         apis: [],
         components: [],
+        customComponents: [],
         other: [],
     });
 }
-export function useTempDir() {
-    let tempDir;
-    beforeEach(async () => {
-        tempDir = await mkdtemp(path.join(os.tmpdir(), "sb-ai-evals-"));
-    });
-    afterEach(async () => {
-        await rm(tempDir, { recursive: true, force: true });
-    });
-    return (testName) => {
-        if (testName) {
-            // Create unique subdirectory for each test
-            const sanitizedTestName = testName.replace(/[^a-zA-Z0-9-]/g, "-");
-            return path.join(tempDir, sanitizedTestName);
-        }
-        return tempDir;
-    };
+export async function createTempDir(testName) {
+    const tempDir = await mkdtemp(path.join(os.tmpdir(), "sb-ai-evals-"));
+    if (testName) {
+        const sanitizedTestName = testName.replace(/[^a-zA-Z0-9-]/g, "-");
+        const subDir = path.join(tempDir, sanitizedTestName);
+        await mkdir(subDir, { recursive: true });
+        return subDir;
+    }
+    return tempDir;
+}
+export async function cleanupTempDir(tempDir) {
+    await rm(tempDir, { recursive: true, force: true });
 }
 export function setupTracing() {
     // Initialize dd-trace for LLM Observability (like dev-tracer)

package/dist/ai-service/evals/helpers/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/ai-service/evals/helpers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC1E,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,OAAO,EAAE,EAAe,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACvD,OAAO,WAAW,MAAM,0BAA0B,CAAC,OAAO,IAAI,EAAE,MAAM,EAAE,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AAClE,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAG1E,MAAM,CAAC,MAAM,eAAe,GAAG,GAAG,EAAE,CAClC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;AAEtD,wEAAwE;AACxE,MAAM,UAAU,qBAAqB,CAAC,KAAU;IAC9C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QAChD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,sDAAsD;IACtD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAEpD,8DAA8D;IAC9D,IACE,SAAS,CAAC,iBAAiB;QAC3B,OAAO,SAAS,CAAC,iBAAiB,KAAK,QAAQ;QAC/C,SAAS,CAAC,iBAAiB,CAAC,MAAM;QAClC,OAAO,SAAS,CAAC,iBAAiB,CAAC,MAAM,KAAK,QAAQ,EACtD,CAAC;QACD,MAAM,YAAY,GAAG,SAAS,CAAC,iBAAiB,CAAC,MAAM,CAAC;QACxD,IAAI,YAAY,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC9B,SAAS,CAAC,iBAAiB,CAAC,MAAM;gBAChC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,iBAAiB,CAAC;QACnD,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,iDAAiD;AACjD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAAW;IAChD,IAAI,CAAC;QACH,OAAO,MAAM,YAAY,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,cAAc,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;QACpD,MAAM,cAAc,CAAC;IACvB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,EAAoB,EACpB,OAGC;IAED,IACE,OAAO,EAAE,UAAU;QACnB,OAAO,CAAC,GAAG,CAAC,6CAA6C,KAAK,MAAM;QACpE,OAAO,CAAC,GAAG,CAAC,EAAE,KAAK,MAAM,EACzB,CAAC;QACD,OAAO,CAAC,GAAG,CACT,sBAAsB,OAAO,CAAC,UAAU,cAAc,MAAM,CAAC,QAAQ,EAAE,CAAC,eAAe,EAAE,CAC1F,CAAC;QACF,MAAM,MAAM,GAAG,MAAM,QAAQ,CAC3B,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC,EAClD,MAAM,CACP,CAAC;QACF,OAAO,EAAE,IAAI,EAAE,MAAM,EAAO,CAAC;IAC/B,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAE3B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,eAAe,IAAI,qBAAqB,CAAC;IAE5E,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAC3B,EAAE,IAAI,EAAE,QAAQ,QAAQ,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,WAAW,EAAE,EAClE,EAAE,CACH,CAAC;IAEF,OAAO,MAAM,SAAS,EAAE,CAAC;AAC3B,CAAC;AAMD,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,MAAM,iBAAiB,GAAG,IAAI,qBAAqB,EAAE,CAAC;IACtD,OAAO,iBAAiB,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;AACtD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,QAAQ,GAAG,CAAC,wBAAwB,EAAE,cAAc,CAAC;IAErD,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CACjC,SAAS,EACT,wCAAwC,CACzC,CAAC;IACF,MAAM,gBAAgB,GAAG,IAAI,cAAc,EAAE,CAAC;IAC9C,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC;QAC5B,cAAc,EAAE,cAAc;QAC9B,gBAAgB;KACjB,CAAC,CAAC;IAEH,OAAO,MAAM,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;AAC5C,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,SAAyB,EAAE,OAAe;IACtE,OAAO,OAAO,CAAC,GAAG,CAChB,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;QAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACvD,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,MAAM,SAAS,CAAC,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC9C,CAAC,CAAC,CACH,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,SAAyB;IACtD,MAAM,eAAe,GAAG,SAAS,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IACrD,MAAM,gBAAgB,GAAG,SAAS,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACrD,MAAM,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IACtD,MAAM,qBAAqB,GAAG,kBAAkB,CAC9C,KAAK,CAAC,cAAc,EACpB,KAAK,CAAC,gBAAgB,CACvB,CAAC;IAEF,OAAO,SAAS,CAAC,MAAM,CACrB,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QACZ,MAAM,aAAa,GAAG,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrD,MAAM,cAAc,GAAG,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvD,MAAM,YAAY,GAAG,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnD,MAAM,mBAAmB,GAAG,qBAAqB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEjE,IAAI,IAAI,CAAC,QAAQ,KAAK,KAAK,CAAC,WAAW,EAAE,CAAC;YACxC,GAAG,CAAC,WAAW,GAAG,IAAI,CAAC;QACzB,CAAC;aAAM,IAAI,aAAa,CAAC,OAAO,EAAE,CAAC;YACjC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;gBACb,GAAG,IAAI;gBACP,MAAM,EAAE,aAAa,CAAC,MAAM;aAC7B,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,cAAc,CAAC,OAAO,EAAE,CAAC;YAClC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;gBACd,GAAG,IAAI;gBACP,MAAM,EAAE,cAAc,CAAC,MAAM;aAC9B,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC;YAChC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;gBACZ,GAAG,IAAI;gBACP,MAAM,EAAE,YAAY,CAAC,MAAM;aAC5B,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,mBAAmB,CAAC,OAAO,EAAE,CAAC;YACvC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC;gBAClB,GAAG,IAAI;gBACP,MAAM,EAAE,mBAAmB,CAAC,MAAM;aACnC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,EACD;QACE,WAAW,EAAE,IAA2B;QACxC,MAAM,EAAE,EAAyC;QACjD,KAAK,EAAE,EAA0C;QACjD,IAAI,EAAE,EAA4C;QAClD,UAAU,EAAE,EAA6C;QACzD,KAAK,EAAE,EAAoB;KAC5B,CACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,UAAU;IACxB,IAAI,OAAe,CAAC;IAEpB,UAAU,CAAC,KAAK,IAAI,EAAE;QACpB,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,QAAiB,EAAE,EAAE;QAC3B,IAAI,QAAQ,EAAE,CAAC;YACb,2CAA2C;YAC3C,MAAM,iBAAiB,GAAG,QAAQ,CAAC,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC;YAClE,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;QAC/C,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY;IAC1B,8DAA8D;IAC9D,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,eAAe,CAAC;IACtC,OAAO,CAAC,IAAI,CAAC;QACX,GAAG,EAAE,YAAY;QACjB,OAAO,EAAE,kBAAkB;QAC3B,OAAO,EAAE,WAAW,CAAC,OAAO;QAC5B,OAAO,EAAE,KAAK;QACd,IAAI,EAAE;YACJ,YAAY,EAAE,UAAU;YACxB,kBAAkB,EAAE,QAAQ;SAC7B;QACD,MAAM,EAAE;YACN,KAAK,EAAE,sBAAsB;YAC7B,gBAAgB,EAAE,IAAI;SACvB;KACF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,SAAyB,EAAE,OAAe;IACxE,MAAM,cAAc,CAClB,SAAS;SACN,MAAM,CACL,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CACf,SAAS,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,CAAC,OAAO,KAAK,KAAK,CAC7D;SACA,MAAM,CAAC,MAAM,oBAAoB,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,EACxD,OAAO,CACR,CAAC;IAEF,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC;QACxB,kBAAkB,EAAE,IAAI;QACxB,UAAU,EAAE,iBAAiB;aAC1B,MAAM,CACL;YACE,KAAK,EAAE;gBACL,qCAAqC,EAAE,KAAK;gBAC5C,yCAAyC,EAAE,KAAK;gBAChD,mCAAmC,EAAE,KAAK;gBAC1C,4CAA4C,EAAE,KAAK;gBACnD,uCAAuC,EAAE,KAAK;gBAC9C,oCAAoC,EAAE,KAAK;aAC5C;SACF,EACD;YACE,KAAK,EAAE,CAAC,UAAU,CAAC;YACnB,KAAK,EAAE;gBACL,qCAAqC,EAAE,MAAM;gBAC7C,gCAAgC,EAAE,OAAO;gBACzC,+BAA+B,EAAE,MAAM;gBACvC,iCAAiC,EAAE,MAAM;gBACzC,oCAAoC,EAAE,MAAM;gBAC5C,mCAAmC,EAAE,OAAO;gBAC5C,2CAA2C,EAAE,OAAO;gBACpD,+BAA+B,EAAE,MAAM;gBACvC,6CAA6C,EAAE,OAAO;aACvD;SACF,CACF;aACA,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAClB,+DAA+D;YAC/D,+DAA+D;YAC/D,mDAAmD;YACnD,IACE,UAAU;gBACV,OAAO,UAAU,KAAK,QAAQ;gBAC9B,QAAQ,IAAI,UAAU,EACtB,CAAC;gBACD,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,UAAiB,CAAC;gBAC9C,OAAO;oBACL,GAAG,IAAI;oBACP,eAAe,EAAE;wBACf,GAAG,CAAC,IAAI,CAAC,eAAe,IAAI,EAAE,CAAC;wBAC/B,MAAM;qBACP;iBACF,CAAC;YACJ,CAAC;YACD,OAAO,UAAU,CAAC;QACpB,CAAC,CAAC;QAEJ,MAAM,EAAE,KAAK;QACb,GAAG,EAAE,OAAO;KACb,CAAC,CAAC;IACH,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,SAAS,CACxC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CACpC,CAAC;IAEF,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;IACxD,MAAM,YAAY,GAAG,MAAM,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;IACzD,OAAO,SAAS,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;AACxC,CAAC;AAsBD,SAAS,gBAAgB,CAAC,OAAuB;IAC/C,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,MAAM,aAAa,GACjB,EAAE,CAAC;IAEL,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,WAAW,IAAI,MAAM,CAAC,UAAU,CAAC;QACjC,aAAa,IAAI,MAAM,CAAC,YAAY,CAAC;QAErC,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,SAAS,CAAC;YAE3C,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3B,aAAa,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;YACrD,CAAC;YAED,IAAI,OAAO,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;gBAC3B,aAAa,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC;YACjC,CAAC;iBAAM,IAAI,OAAO,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;gBAClC,aAAa,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;YACnC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,UAAU,EAAE,WAAW;QACvB,YAAY,EAAE,aAAa;QAC3B,aAAa;QACb,SAAS,EAAE,WAAW,GAAG,CAAC;KAC3B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,SAAyB,EACzB,OAAe;IAEf,MAAM,cAAc,CAClB,SAAS;SACN,MAAM,CACL,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CACf,SAAS,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,CAAC,OAAO,KAAK,KAAK,CAC7D;SACA,MAAM,CAAC,MAAM,oBAAoB,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,EACxD,OAAO,CACR,CAAC;IAEF,+EAA+E;IAC/E,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;IAC3D,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IACjE,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IAE3D,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,iBAAiB,EAAE,eAAe,EAAE,KAAK,CAAC,CAAC;IAC3D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,0DAA0D;QAC1D,OAAO,CAAC,IAAI,CAAC,wCAAwC,EAAE,KAAK,CAAC,CAAC;IAChE,CAAC;IAED,4EAA4E;IAC5E,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IAC3D,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuEvB,CAAC;IACA,MAAM,SAAS,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;IAE3C,yDAAyD;IACzD,OAAO,IAAI,OAAO,CAAa,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACjD,MAAM,aAAa,GAAG,KAAK,CACzB,KAAK,EACL;YACE,QAAQ;YACR,UAAU;YACV,mBAAmB;YACnB,UAAU;YACV,MAAM;YACN,eAAe;SAChB,EACD;YACE,GAAG,EAAE,OAAO;YACZ,KAAK,EAAE,MAAM;SACd,CACF,CAAC;QAEF,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,MAAM,GAAG,EAAE,CAAC;QAEhB,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACxC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACxC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,aAAa,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YACjC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACf,6BAA6B;gBAC7B,OAAO,CAAC;oBACN,UAAU,EAAE,CAAC;oBACb,YAAY,EAAE,CAAC;oBACf,aAAa,EAAE,EAAE;oBACjB,SAAS,EAAE,KAAK;iBACjB,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACtB,uCAAuC;gBACvC,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAmB,CAAC;oBACrD,MAAM,UAAU,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC7C,OAAO,CAAC,UAAU,CAAC,CAAC;gBACtB,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,CACJ,IAAI,KAAK,CACP,uCAAuC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAChG,CACF,CAAC;gBACJ,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,uBAAuB;gBACvB,MAAM,CAAC,IAAI,KAAK,CAAC,+BAA+B,IAAI,MAAM,MAAM,EAAE,CAAC,CAAC,CAAC;YACvE,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,aAAa,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAClC,MAAM,CAAC,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/ai-service/evals/helpers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC1E,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,iBAAiB,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,OAAO,EAAE,EAAe,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,WAAW,MAAM,0BAA0B,CAAC,OAAO,IAAI,EAAE,MAAM,EAAE,CAAC;AACzE,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AAClE,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAK1E,MAAM,CAAC,MAAM,eAAe,GAAG,GAAG,EAAE,CAClC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;AAEtD,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAW,EAAE,EAAE;IAC5C,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;IACzC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;AAC/C,CAAC,CAAC;AAEF,wEAAwE;AACxE,MAAM,UAAU,qBAAqB,CAAC,KAAU;IAC9C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QAChD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,sDAAsD;IACtD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAEpD,8DAA8D;IAC9D,IACE,SAAS,CAAC,iBAAiB;QAC3B,OAAO,SAAS,CAAC,iBAAiB,KAAK,QAAQ;QAC/C,SAAS,CAAC,iBAAiB,CAAC,MAAM;QAClC,OAAO,SAAS,CAAC,iBAAiB,CAAC,MAAM,KAAK,QAAQ,EACtD,CAAC;QACD,MAAM,YAAY,GAAG,SAAS,CAAC,iBAAiB,CAAC,MAAM,CAAC;QACxD,IAAI,YAAY,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC9B,SAAS,CAAC,iBAAiB,CAAC,MAAM;gBAChC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,iBAAiB,CAAC;QACnD,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,iDAAiD;AACjD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAAW;IAChD,IAAI,CAAC;QACH,OAAO,MAAM,YAAY,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,cAAc,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;QACpD,MAAM,cAAc,CAAC;IACvB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,EAAoB,EACpB,OAUC;IAED,IACE,OAAO,EAAE,UAAU;QACnB,OAAO,CAAC,GAAG,CAAC,6CAA6C,KAAK,MAAM;QACpE,OAAO,CAAC,GAAG,CAAC,EAAE,KAAK,MAAM,EACzB,CAAC;QACD,OAAO,CAAC,GAAG,CACT,sBAAsB,OAAO,CAAC,UAAU,cAAc,MAAM,CAAC,QAAQ,EAAE,CAAC,eAAe,EAAE,CAC1F,CAAC;QACF,MAAM,MAAM,GAAG,MAAM,QAAQ,CAC3B,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC,EAClD,MAAM,CACP,CAAC;QACF,wEAAwE;QACxE,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAO,EAAE,CAAC;IAC3C,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,eAAe,IAAI,qBAAqB,CAAC;IAE5E,IAAI,mBAAmB,GAAQ,SAAS,CAAC;IAEzC,8DAA8D;IAC9D,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAC/B,EAAE,IAAI,EAAE,QAAQ,QAAQ,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,WAAW,EAAE,EAClE,KAAK,IAAI,EAAE;QACT,mBAAmB,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;QAE1C,MAAM,QAAQ,GAAG,MAAM,EAAE,EAAE,CAAC;QAE5B,gEAAgE;QAChE,IAAI,OAAO,EAAE,QAAQ,EAAE,CAAC;YACtB,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAClD,MAAM,CAAC,QAAQ,CAAC;gBACd,SAAS,EAAE,cAAc,CAAC,SAAS;gBACnC,UAAU,EAAE,cAAc,CAAC,UAAU;gBACrC,OAAO,EAAE,cAAc,CAAC,OAAO;gBAC/B,GAAG,CAAC,cAAc,CAAC,QAAQ;oBACzB,CAAC,CAAC,EAAE,QAAQ,EAAE,cAAc,CAAC,QAAQ,EAAE;oBACvC,CAAC,CAAC,EAAE,CAAC;aACR,CAAC,CAAC;QACL,CAAC;QAED,yCAAyC;QACzC,IAAI,OAAO,EAAE,YAAY,EAAE,CAAC;YAC1B,MAAM,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QACvC,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC,CACF,CAAC;IAEF,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE,CAAC;AACtD,CAAC;AAMD,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,MAAM,iBAAiB,GAAG,IAAI,qBAAqB,EAAE,CAAC;IACtD,OAAO,iBAAiB,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;AACtD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,QAAQ,GAAG,CAAC,wBAAwB,EAAE,cAAc,CAAC;IAErD,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CACjC,SAAS,EACT,wCAAwC,CACzC,CAAC;IACF,MAAM,gBAAgB,GAAG,IAAI,cAAc,EAAE,CAAC;IAC9C,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC;QAC5B,cAAc,EAAE,cAAc;QAC9B,gBAAgB;KACjB,CAAC,CAAC;IAEH,OAAO,MAAM,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAAe,EACf,QAAQ,GAAG,CAAC,wBAAwB,EAAE,cAAc,CAAC;IAErD,kCAAkC;IAClC,MAAM,gBAAgB,GAAG,MAAM,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAE9D,6BAA6B;IAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,kBAAkB,EAAE,OAAO,CAAC,CAAC;IACzE,MAAM,gBAAgB,GAAG,IAAI,cAAc,EAAE,CAAC;IAC9C,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC;QAC5B,cAAc,EAAE,WAAW;QAC3B,gBAAgB;KACjB,CAAC,CAAC;IACH,MAAM,iBAAiB,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAE7D,oDAAoD;IACpD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAwB,CAAC;IAEpD,8BAA8B;IAC9B,KAAK,MAAM,QAAQ,IAAI,gBAAgB,EAAE,CAAC;QACxC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED,4DAA4D;IAC5D,KAAK,MAAM,QAAQ,IAAI,iBAAiB,EAAE,CAAC;QACzC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED,mCAAmC;IACnC,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;AAC1C,CAAC;AAuBD,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,OAAe,EACf,QAAgB,EAChB,QAAiC;IAEjC,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAClD,MAAM,cAAc,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC;IAEtE,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,OAAO,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3C,CAAC;IAED,MAAM,GAAG,GAAG,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE;QACxC,UAAU,EAAE,QAAQ;QACpB,OAAO,EAAE,CAAC,KAAK,EAAE,YAAY,CAAC;KAC/B,CAAC,CAAC;IAEH,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,MAAM,WAAW,GAAG,QAAQ,CAAC,UAAU,IAAI,CAAC,CAAC;IAE7C,QAAQ,CAAC,GAAG,EAAE;QACZ,iBAAiB,CAAC,IAAiC;YACjD,IACE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,eAAe;gBACvC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,OAAO,EACxC,CAAC;gBACD,IAAI,OAAO,GAAG,IAAI,CAAC;gBAEnB,oCAAoC;gBACpC,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;oBACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CACxC,CAAC,IAAwB,EAAE,EAAE,CAC3B,IAAI,CAAC,IAAI,KAAK,cAAc;wBAC5B,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,eAAe;wBAClC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM;wBACzB,IAAI,CAAC,KAAK,EAAE,IAAI,KAAK,wBAAwB;wBAC7C,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,KAAK,YAAY;wBAC3C,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ,CAAC,QAAQ,CACnD,CAAC;oBACF,IAAI,CAAC,QAAQ;wBAAE,OAAO,GAAG,KAAK,CAAC;gBACjC,CAAC;gBAED,gCAAgC;gBAChC,IAAI,QAAQ,CAAC,SAAS,IAAI,OAAO,EAAE,CAAC;oBAClC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CACtC,CAAC,IAAwB,EAAE,EAAE,CAC3B,IAAI,CAAC,IAAI,KAAK,cAAc;wBAC5B,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,eAAe;wBAClC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,YAAY;wBAC/B,IAAI,CAAC,KAAK,EAAE,IAAI,KAAK,eAAe;wBACpC,IAAI,CAAC,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC,SAAS,CAC1C,CAAC;oBACF,IAAI,CAAC,MAAM;wBAAE,OAAO,GAAG,KAAK,CAAC;gBAC/B,CAAC;gBAED,kDAAkD;gBAClD,IAAI,QAAQ,CAAC,gBAAgB,IAAI,OAAO,EAAE,CAAC;oBACzC,KAAK,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAChD,QAAQ,CAAC,gBAAgB,CAC1B,EAAE,CAAC;wBACF,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CACpC,CAAC,IAAwB,EAAE,EAAE,CAC3B,IAAI,CAAC,IAAI,KAAK,cAAc;4BAC5B,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,eAAe;4BAClC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAC9B,CAAC;wBAEF,IACE,CAAC,IAAI;4BACL,CAAC,IAAI,CAAC,IAAI,KAAK,cAAc;gCAC3B,IAAI,CAAC,KAAK,EAAE,IAAI,KAAK,eAAe;gCACpC,IAAI,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,EACjC,CAAC;4BACD,OAAO,GAAG,KAAK,CAAC;4BAChB,MAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAEnB,4EAA4E;oBAC5E,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC;wBACtD,IAAI,CAAC,IAAI,EAAE,CAAC;oBACd,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACrB,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,OAAO,OAAO,CAAC,WAAW,CAAC,IAAI,SAAS,CAAC;AAC3C,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,SAAyB,EAAE,OAAe;IACtE,OAAO,OAAO,CAAC,GAAG,CAChB,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;QAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACvD,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,MAAM,SAAS,CAAC,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC9C,CAAC,CAAC,CACH,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,SAAyB;IACtD,MAAM,eAAe,GAAG,SAAS,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IACrD,MAAM,gBAAgB,GAAG,SAAS,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACrD,MAAM,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IACtD,MAAM,qBAAqB,GAAG,kBAAkB,CAC9C,KAAK,CAAC,cAAc,EACpB,KAAK,CAAC,gBAAgB,CACvB,CAAC;IAEF,OAAO,SAAS,CAAC,MAAM,CACrB,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QACZ,MAAM,aAAa,GAAG,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrD,MAAM,cAAc,GAAG,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvD,MAAM,YAAY,GAAG,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnD,MAAM,mBAAmB,GAAG,qBAAqB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEjE,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC9B,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC;gBAClB,GAAG,IAAI;gBACP,MAAM,EAAE,EAAS;aAClB,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,IAAI,CAAC,QAAQ,KAAK,KAAK,CAAC,WAAW,EAAE,CAAC;YAC/C,GAAG,CAAC,WAAW,GAAG,IAAI,CAAC;QACzB,CAAC;aAAM,IAAI,aAAa,CAAC,OAAO,EAAE,CAAC;YACjC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;gBACb,GAAG,IAAI;gBACP,MAAM,EAAE,aAAa,CAAC,MAAM;aAC7B,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,cAAc,CAAC,OAAO,EAAE,CAAC;YAClC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;gBACd,GAAG,IAAI;gBACP,MAAM,EAAE,cAAc,CAAC,MAAM;aAC9B,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC;YAChC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;gBACZ,GAAG,IAAI;gBACP,MAAM,EAAE,YAAY,CAAC,MAAM;aAC5B,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,mBAAmB,CAAC,OAAO,EAAE,CAAC;YACvC,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACxB,GAAG,IAAI;gBACP,MAAM,EAAE,mBAAmB,CAAC,MAAM;aACnC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,EACD;QACE,WAAW,EAAE,IAA2B;QACxC,MAAM,EAAE,EAAyC;QACjD,KAAK,EAAE,EAA0C;QACjD,IAAI,EAAE,EAA4C;QAClD,UAAU,EAAE,EAA4B;QACxC,gBAAgB,EAAE,EAA6C;QAC/D,KAAK,EAAE,EAAoB;KAC5B,CACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,QAAiB;IACnD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;IAEtE,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,iBAAiB,GAAG,QAAQ,CAAC,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC;QAClE,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;QACrD,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAAe;IAClD,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACtD,CAAC;AAED,MAAM,UAAU,YAAY;IAC1B,8DAA8D;IAC9D,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,eAAe,CAAC;IACtC,OAAO,CAAC,IAAI,CAAC;QACX,GAAG,EAAE,YAAY;QACjB,OAAO,EAAE,kBAAkB;QAC3B,OAAO,EAAE,WAAW,CAAC,OAAO;QAC5B,OAAO,EAAE,KAAK;QACd,IAAI,EAAE;YACJ,YAAY,EAAE,UAAU;YACxB,kBAAkB,EAAE,QAAQ;SAC7B;QACD,MAAM,EAAE;YACN,KAAK,EAAE,sBAAsB;YAC7B,gBAAgB,EAAE,IAAI;SACvB;KACF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,SAAyB,EAAE,OAAe;IACxE,MAAM,cAAc,CAClB,SAAS;SACN,MAAM,CACL,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CACf,SAAS,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,CAAC,OAAO,KAAK,KAAK,CAC7D;SACA,MAAM,CAAC,MAAM,oBAAoB,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,EACxD,OAAO,CACR,CAAC;IAEF,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC;QACxB,kBAAkB,EAAE,IAAI;QACxB,UAAU,EAAE,iBAAiB;aAC1B,MAAM,CACL;YACE,KAAK,EAAE;gBACL,qCAAqC,EAAE,KAAK;gBAC5C,yCAAyC,EAAE,KAAK;gBAChD,mCAAmC,EAAE,KAAK;gBAC1C,4CAA4C,EAAE,KAAK;gBACnD,uCAAuC,EAAE,KAAK;gBAC9C,oCAAoC,EAAE,KAAK;aAC5C;SACF,EACD;YACE,KAAK,EAAE,CAAC,UAAU,CAAC;YACnB,KAAK,EAAE;gBACL,qCAAqC,EAAE,MAAM;gBAC7C,gCAAgC,EAAE,OAAO;gBACzC,+BAA+B,EAAE,MAAM;gBACvC,iCAAiC,EAAE,MAAM;gBACzC,oCAAoC,EAAE,MAAM;gBAC5C,mCAAmC,EAAE,OAAO;gBAC5C,2CAA2C,EAAE,OAAO;gBACpD,+BAA+B,EAAE,MAAM;gBACvC,6CAA6C,EAAE,OAAO;aACvD;SACF,CACF;aACA,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAClB,+DAA+D;YAC/D,+DAA+D;YAC/D,mDAAmD;YACnD,IACE,UAAU;gBACV,OAAO,UAAU,KAAK,QAAQ;gBAC9B,QAAQ,IAAI,UAAU,EACtB,CAAC;gBACD,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,UAAiB,CAAC;gBAC9C,OAAO;oBACL,GAAG,IAAI;oBACP,eAAe,EAAE;wBACf,GAAG,CAAC,IAAI,CAAC,eAAe,IAAI,EAAE,CAAC;wBAC/B,MAAM;qBACP;iBACF,CAAC;YACJ,CAAC;YACD,OAAO,UAAU,CAAC;QACpB,CAAC,CAAC;QAEJ,MAAM,EAAE,KAAK;QACb,GAAG,EAAE,OAAO;KACb,CAAC,CAAC;IACH,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,SAAS,CACxC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CACpC,CAAC;IAEF,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;IACxD,MAAM,YAAY,GAAG,MAAM,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;IACzD,OAAO,SAAS,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;AACxC,CAAC;AAsBD,SAAS,gBAAgB,CAAC,OAAuB;IAC/C,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,MAAM,aAAa,GACjB,EAAE,CAAC;IAEL,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,WAAW,IAAI,MAAM,CAAC,UAAU,CAAC;QACjC,aAAa,IAAI,MAAM,CAAC,YAAY,CAAC;QAErC,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,SAAS,CAAC;YAE3C,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3B,aAAa,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;YACrD,CAAC;YAED,IAAI,OAAO,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;gBAC3B,aAAa,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC;YACjC,CAAC;iBAAM,IAAI,OAAO,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;gBAClC,aAAa,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;YACnC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,UAAU,EAAE,WAAW;QACvB,YAAY,EAAE,aAAa;QAC3B,aAAa;QACb,SAAS,EAAE,WAAW,GAAG,CAAC;KAC3B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,SAAyB,EACzB,OAAe;IAEf,MAAM,cAAc,CAClB,SAAS;SACN,MAAM,CACL,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CACf,SAAS,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,CAAC,OAAO,KAAK,KAAK,CAC7D;SACA,MAAM,CAAC,MAAM,oBAAoB,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,EACxD,OAAO,CACR,CAAC;IAEF,+EAA+E;IAC/E,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;IAC3D,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IACjE,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IAE3D,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,iBAAiB,EAAE,eAAe,EAAE,KAAK,CAAC,CAAC;IAC3D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,0DAA0D;QAC1D,OAAO,CAAC,IAAI,CAAC,wCAAwC,EAAE,KAAK,CAAC,CAAC;IAChE,CAAC;IAED,4EAA4E;IAC5E,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IAC3D,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuEvB,CAAC;IACA,MAAM,SAAS,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;IAE3C,yDAAyD;IACzD,OAAO,IAAI,OAAO,CAAa,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACjD,MAAM,aAAa,GAAG,KAAK,CACzB,KAAK,EACL;YACE,QAAQ;YACR,UAAU;YACV,mBAAmB;YACnB,UAAU;YACV,MAAM;YACN,eAAe;SAChB,EACD;YACE,GAAG,EAAE,OAAO;YACZ,KAAK,EAAE,MAAM;SACd,CACF,CAAC;QAEF,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,MAAM,GAAG,EAAE,CAAC;QAEhB,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACxC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACxC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,aAAa,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YACjC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACf,6BAA6B;gBAC7B,OAAO,CAAC;oBACN,UAAU,EAAE,CAAC;oBACb,YAAY,EAAE,CAAC;oBACf,aAAa,EAAE,EAAE;oBACjB,SAAS,EAAE,KAAK;iBACjB,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACtB,uCAAuC;gBACvC,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAmB,CAAC;oBACrD,MAAM,UAAU,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC7C,OAAO,CAAC,UAAU,CAAC,CAAC;gBACtB,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,CACJ,IAAI,KAAK,CACP,uCAAuC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAChG,CACF,CAAC;gBACJ,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,uBAAuB;gBACvB,MAAM,CAAC,IAAI,KAAK,CAAC,+BAA+B,IAAI,MAAM,MAAM,EAAE,CAAC,CAAC,CAAC;YACvE,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,aAAa,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAClC,MAAM,CAAC,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/ai-service/evals/helpers/non-critical-scorers.d.ts

@@ -0,0 +1,6 @@
+import type { ScorerContext, ScorerResult } from "./scorer-types.js";
+export declare function scoreApiCountTolerance(context: ScorerContext): ScorerResult;
+export declare function scoreExpectedUiComponentsInPage(context: ScorerContext): ScorerResult;
+export declare function scoreExpectedUiComponentProperties(context: ScorerContext): ScorerResult;
+export declare function scoreLintIssues(context: ScorerContext): Promise<ScorerResult>;
+//# sourceMappingURL=non-critical-scorers.d.ts.map

package/dist/ai-service/evals/helpers/non-critical-scorers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"non-critical-scorers.d.ts","sourceRoot":"","sources":["../../../../src/ai-service/evals/helpers/non-critical-scorers.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAMrE,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAoC3E;AAED,wBAAgB,+BAA+B,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAmEpF;AAiBD,wBAAgB,kCAAkC,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CA0EvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CA0BnF"}

package/dist/ai-service/evals/helpers/non-critical-scorers.js

@@ -0,0 +1,187 @@
+import { findComponentsInPage, findComponentAtPath } from "../content-matchers/index.js";
+import { SCORING_CONFIG } from "./scorer-types.js";
+// Non-critical scorer functions - these result in smaller point deductions when they fail
+// ------------------------
+export function scoreApiCountTolerance(context) {
+    const expectedRange = context.config.expectedApiCountRange;
+    let passed = true;
+    let apiCountDiff = 0;
+    if (typeof expectedRange === "number") {
+        if (expectedRange !== context.apis.length) {
+            passed = false;
+            apiCountDiff = Math.abs(expectedRange - context.apis.length);
+        }
+    }
+    else {
+        const [min, max] = expectedRange;
+        if (context.apis.length < min || context.apis.length > max) {
+            passed = false;
+            apiCountDiff =
+                context.apis.length < min
+                    ? min - context.apis.length
+                    : context.apis.length - max;
+        }
+    }
+    const scoreDelta = passed
+        ? 0
+        : apiCountDiff * SCORING_CONFIG.NON_CRITICAL_DEDUCTIONS.API_OUTSIDE_RANGE;
+    return {
+        checkName: "hasReasonableApiCount",
+        passed,
+        scoreDelta,
+        isCritical: false,
+        metadata: {
+            expectedRange,
+            actualCount: context.apis.length,
+            apiCountDiff,
+        },
+    };
+}
+export function scoreExpectedUiComponentsInPage(context) {
+    let totalDeduction = 0;
+    let totalIssues = 0;
+    let passed = true;
+    const componentIssues = [];
+    for (const uiComponent of context.config.expectedUiComponents ?? []) {
+        const foundComponents = findComponentsInPage(context.pageContent, uiComponent.type);
+        if (typeof uiComponent.expectedCount === "number") {
+            const tolerance = 1;
+            const diff = Math.abs(foundComponents.length - uiComponent.expectedCount);
+            if (diff > tolerance) {
+                const componentsOutsideRange = diff - tolerance;
+                const deduction = componentsOutsideRange *
+                    SCORING_CONFIG.NON_CRITICAL_DEDUCTIONS
+                        .MISSING_NON_CRITICAL_COMPONENT;
+                totalIssues += componentsOutsideRange;
+                totalDeduction += deduction;
+                passed = false;
+                componentIssues.push({
+                    type: uiComponent.type,
+                    expected: uiComponent.expectedCount,
+                    actual: foundComponents.length,
+                    issueCount: componentsOutsideRange,
+                });
+            }
+        }
+        else {
+            const [min, max] = uiComponent.expectedCount;
+            if (foundComponents.length < min || foundComponents.length > max) {
+                const componentsOutsideRange = foundComponents.length < min
+                    ? min - foundComponents.length
+                    : foundComponents.length - max;
+                const deduction = componentsOutsideRange *
+                    SCORING_CONFIG.NON_CRITICAL_DEDUCTIONS
+                        .MISSING_NON_CRITICAL_COMPONENT;
+                totalIssues += componentsOutsideRange;
+                totalDeduction += deduction;
+                passed = false;
+                componentIssues.push({
+                    type: uiComponent.type,
+                    expected: [min, max],
+                    actual: foundComponents.length,
+                    issueCount: componentsOutsideRange,
+                });
+            }
+        }
+    }
+    return {
+        checkName: "hasReasonableUiComponentInPageCounts",
+        passed,
+        scoreDelta: totalDeduction,
+        isCritical: false,
+        metadata: {
+            totalIssues,
+            componentIssues,
+        },
+    };
+}
+function getComponentForPropertyCheck(foundComponents, config) {
+    if (config.path) {
+        return findComponentAtPath(foundComponents, config.path);
+    }
+    const index = config.index ?? 0;
+    return foundComponents[index];
+}
+export function scoreExpectedUiComponentProperties(context) {
+    let totalDeduction = 0;
+    let totalIssues = 0;
+    let passed = true;
+    const propertyIssues = [];
+    for (const uiComponent of context.config.expectedUiComponentProperties ?? []) {
+        const foundComponents = findComponentsInPage(context.pageContent, uiComponent.type);
+        const component = getComponentForPropertyCheck(foundComponents, uiComponent);
+        if (!component) {
+            const selector = uiComponent.path
+                ? `at path [${uiComponent.path.join(", ")}]`
+                : `at index ${uiComponent.index ?? 0}`;
+            throw new Error(`Component of type ${uiComponent.type} ${selector} to check properties on not found`);
+        }
+        // Check expected properties
+        for (const [key, value] of Object.entries(uiComponent.expectedProperties ?? {})) {
+            if (component.props[key] !== value) {
+                const deduction = SCORING_CONFIG.NON_CRITICAL_DEDUCTIONS.MISSING_NON_CRITICAL_PROPERTY;
+                totalIssues += 1;
+                totalDeduction += deduction;
+                passed = false;
+                propertyIssues.push({
+                    componentType: uiComponent.type,
+                    property: key,
+                    expectedValue: value,
+                    actualValue: component.props[key],
+                });
+            }
+        }
+        // Check expected undefined properties
+        if (uiComponent.expectedUndefinedProperties) {
+            for (const undefinedProp of uiComponent.expectedUndefinedProperties) {
+                const propertyIsUndefined = !(undefinedProp in component.props) ||
+                    component.props[undefinedProp] === undefined;
+                if (!propertyIsUndefined) {
+                    const deduction = SCORING_CONFIG.NON_CRITICAL_DEDUCTIONS.MISSING_NON_CRITICAL_PROPERTY;
+                    totalIssues += 1;
+                    totalDeduction += deduction;
+                    passed = false;
+                    propertyIssues.push({
+                        componentType: uiComponent.type,
+                        unexpectedProperty: undefinedProp,
+                        actualValue: component.props[undefinedProp],
+                    });
+                }
+            }
+        }
+    }
+    return {
+        checkName: "hasReasonableUiComponentProperties",
+        passed,
+        scoreDelta: totalDeduction,
+        isCritical: false,
+        metadata: {
+            totalIssues,
+            propertyIssues,
+        },
+    };
+}
+export async function scoreLintIssues(context) {
+    if (!context.lintResult) {
+        throw new Error("Lint result not provided to scorer context");
+    }
+    const lintErrorDeduction = context.lintResult.errorCount *
+        SCORING_CONFIG.NON_CRITICAL_DEDUCTIONS.LINT_ERROR;
+    const lintWarningDeduction = context.lintResult.warningCount *
+        SCORING_CONFIG.NON_CRITICAL_DEDUCTIONS.LINT_WARNING;
+    const totalDeduction = lintErrorDeduction + lintWarningDeduction;
+    return {
+        checkName: "hasLintErrors",
+        passed: !context.lintResult.hasErrors,
+        scoreDelta: totalDeduction,
+        isCritical: false,
+        metadata: {
+            errorCount: context.lintResult.errorCount,
+            warningCount: context.lintResult.warningCount,
+            lintErrorDeduction,
+            lintWarningDeduction,
+            ruleBreakdown: context.lintResult.ruleBreakdown,
+        },
+    };
+}
+//# sourceMappingURL=non-critical-scorers.js.map

package/dist/ai-service/evals/helpers/non-critical-scorers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"non-critical-scorers.js","sourceRoot":"","sources":["../../../../src/ai-service/evals/helpers/non-critical-scorers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AAEzF,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD,0FAA0F;AAC1F,2BAA2B;AAE3B,MAAM,UAAU,sBAAsB,CAAC,OAAsB;IAC3D,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,qBAAqB,CAAC;IAC3D,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,IAAI,aAAa,KAAK,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1C,MAAM,GAAG,KAAK,CAAC;YACf,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,aAAa,CAAC;QACjC,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC3D,MAAM,GAAG,KAAK,CAAC;YACf,YAAY;gBACV,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG;oBACvB,CAAC,CAAC,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM;oBAC3B,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;QAClC,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,MAAM;QACvB,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,YAAY,GAAG,cAAc,CAAC,uBAAuB,CAAC,iBAAiB,CAAC;IAE5E,OAAO;QACL,SAAS,EAAE,uBAAuB;QAClC,MAAM;QACN,UAAU;QACV,UAAU,EAAE,KAAK;QACjB,QAAQ,EAAE;YACR,aAAa;YACb,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM;YAChC,YAAY;SACb;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,+BAA+B,CAAC,OAAsB;IACpE,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,MAAM,eAAe,GAAG,EAAE,CAAC;IAE3B,KAAK,MAAM,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC,oBAAoB,IAAI,EAAE,EAAE,CAAC;QACpE,MAAM,eAAe,GAAG,oBAAoB,CAC1C,OAAO,CAAC,WAAW,EACnB,WAAW,CAAC,IAAI,CACjB,CAAC;QAEF,IAAI,OAAO,WAAW,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;YAClD,MAAM,SAAS,GAAG,CAAC,CAAC;YACpB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CACnB,eAAe,CAAC,MAAM,GAAG,WAAW,CAAC,aAAa,CACnD,CAAC;YACF,IAAI,IAAI,GAAG,SAAS,EAAE,CAAC;gBACrB,MAAM,sBAAsB,GAAG,IAAI,GAAG,SAAS,CAAC;gBAChD,MAAM,SAAS,GACb,sBAAsB;oBACtB,cAAc,CAAC,uBAAuB;yBACnC,8BAA8B,CAAC;gBACpC,WAAW,IAAI,sBAAsB,CAAC;gBACtC,cAAc,IAAI,SAAS,CAAC;gBAC5B,MAAM,GAAG,KAAK,CAAC;gBACf,eAAe,CAAC,IAAI,CAAC;oBACnB,IAAI,EAAE,WAAW,CAAC,IAAI;oBACtB,QAAQ,EAAE,WAAW,CAAC,aAAa;oBACnC,MAAM,EAAE,eAAe,CAAC,MAAM;oBAC9B,UAAU,EAAE,sBAAsB;iBACnC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,WAAW,CAAC,aAAa,CAAC;YAC7C,IAAI,eAAe,CAAC,MAAM,GAAG,GAAG,IAAI,eAAe,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACjE,MAAM,sBAAsB,GAC1B,eAAe,CAAC,MAAM,GAAG,GAAG;oBAC1B,CAAC,CAAC,GAAG,GAAG,eAAe,CAAC,MAAM;oBAC9B,CAAC,CAAC,eAAe,CAAC,MAAM,GAAG,GAAG,CAAC;gBACnC,MAAM,SAAS,GACb,sBAAsB;oBACtB,cAAc,CAAC,uBAAuB;yBACnC,8BAA8B,CAAC;gBACpC,WAAW,IAAI,sBAAsB,CAAC;gBACtC,cAAc,IAAI,SAAS,CAAC;gBAC5B,MAAM,GAAG,KAAK,CAAC;gBACf,eAAe,CAAC,IAAI,CAAC;oBACnB,IAAI,EAAE,WAAW,CAAC,IAAI;oBACtB,QAAQ,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;oBACpB,MAAM,EAAE,eAAe,CAAC,MAAM;oBAC9B,UAAU,EAAE,sBAAsB;iBACnC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,SAAS,EAAE,sCAAsC;QACjD,MAAM;QACN,UAAU,EAAE,cAAc;QAC1B,UAAU,EAAE,KAAK;QACjB,QAAQ,EAAE;YACR,WAAW;YACX,eAAe;SAChB;KACF,CAAC;AACJ,CAAC;AAED,SAAS,4BAA4B,CACnC,eAAwD,EACxD,MAGC;IAED,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,OAAO,mBAAmB,CAAC,eAAe,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3D,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC;IAChC,OAAO,eAAe,CAAC,KAAK,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,kCAAkC,CAAC,OAAsB;IACvE,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,MAAM,cAAc,GAAG,EAAE,CAAC;IAE1B,KAAK,MAAM,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC,6BAA6B,IAAI,EAAE,EAAE,CAAC;QAC7E,MAAM,eAAe,GAAG,oBAAoB,CAC1C,OAAO,CAAC,WAAW,EACnB,WAAW,CAAC,IAAI,CACjB,CAAC;QACF,MAAM,SAAS,GAAG,4BAA4B,CAC5C,eAAe,EACf,WAAW,CACZ,CAAC;QAEF,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI;gBAC/B,CAAC,CAAC,YAAY,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;gBAC5C,CAAC,CAAC,YAAY,WAAW,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CACb,qBAAqB,WAAW,CAAC,IAAI,IAAI,QAAQ,mCAAmC,CACrF,CAAC;QACJ,CAAC;QAED,4BAA4B;QAC5B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CACvC,WAAW,CAAC,kBAAkB,IAAI,EAAE,CACrC,EAAE,CAAC;YACF,IAAI,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,KAAK,EAAE,CAAC;gBACnC,MAAM,SAAS,GAAG,cAAc,CAAC,uBAAuB,CAAC,6BAA6B,CAAC;gBACvF,WAAW,IAAI,CAAC,CAAC;gBACjB,cAAc,IAAI,SAAS,CAAC;gBAC5B,MAAM,GAAG,KAAK,CAAC;gBACf,cAAc,CAAC,IAAI,CAAC;oBAClB,aAAa,EAAE,WAAW,CAAC,IAAI;oBAC/B,QAAQ,EAAE,GAAG;oBACb,aAAa,EAAE,KAAK;oBACpB,WAAW,EAAE,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC;iBAClC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,sCAAsC;QACtC,IAAI,WAAW,CAAC,2BAA2B,EAAE,CAAC;YAC5C,KAAK,MAAM,aAAa,IAAI,WAAW,CAAC,2BAA2B,EAAE,CAAC;gBACpE,MAAM,mBAAmB,GACvB,CAAC,CAAC,aAAa,IAAI,SAAS,CAAC,KAAK,CAAC;oBACnC,SAAS,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,SAAS,CAAC;gBAC/C,IAAI,CAAC,mBAAmB,EAAE,CAAC;oBACzB,MAAM,SAAS,GAAG,cAAc,CAAC,uBAAuB,CAAC,6BAA6B,CAAC;oBACvF,WAAW,IAAI,CAAC,CAAC;oBACjB,cAAc,IAAI,SAAS,CAAC;oBAC5B,MAAM,GAAG,KAAK,CAAC;oBACf,cAAc,CAAC,IAAI,CAAC;wBAClB,aAAa,EAAE,WAAW,CAAC,IAAI;wBAC/B,kBAAkB,EAAE,aAAa;wBACjC,WAAW,EAAE,SAAS,CAAC,KAAK,CAAC,aAAa,CAAC;qBAC5C,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,SAAS,EAAE,oCAAoC;QAC/C,MAAM;QACN,UAAU,EAAE,cAAc;QAC1B,UAAU,EAAE,KAAK;QACjB,QAAQ,EAAE;YACR,WAAW;YACX,cAAc;SACf;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAAsB;IAC1D,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,kBAAkB,GACtB,OAAO,CAAC,UAAU,CAAC,UAAU;QAC7B,cAAc,CAAC,uBAAuB,CAAC,UAAU,CAAC;IACpD,MAAM,oBAAoB,GACxB,OAAO,CAAC,UAAU,CAAC,YAAY;QAC/B,cAAc,CAAC,uBAAuB,CAAC,YAAY,CAAC;IACtD,MAAM,cAAc,GAAG,kBAAkB,GAAG,oBAAoB,CAAC;IAEjE,OAAO;QACL,SAAS,EAAE,eAAe;QAC1B,MAAM,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS;QACrC,UAAU,EAAE,cAAc;QAC1B,UAAU,EAAE,KAAK;QACjB,QAAQ,EAAE;YACR,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,UAAU;YACzC,YAAY,EAAE,OAAO,CAAC,UAAU,CAAC,YAAY;YAC7C,kBAAkB;YAClB,oBAAoB;YACpB,aAAa,EAAE,OAAO,CAAC,UAAU,CAAC,aAAa;SAChD;KACF,CAAC;AACJ,CAAC"}

package/dist/ai-service/evals/helpers/scorer-types.d.ts

@@ -0,0 +1,39 @@
+import type { EvalScoreConfig } from "./eval-scoring.js";
+import type { MatchedArtifact, LintResult } from "./index.js";
+import type { Paths } from "../../const.js";
+export interface ScorerContext {
+    scopes: MatchedArtifact<Paths.PageScopes>[];
+    pages: MatchedArtifact<Paths.PageIndexes>[];
+    apis: MatchedArtifact<Paths.GeneratedApis>[];
+    apiNames: string[];
+    customComponents: MatchedArtifact<Paths.ComponentsFile>[];
+    components: MatchedArtifact<Paths.ComponentsFile>[];
+    config: EvalScoreConfig;
+    pageContent: string;
+    lintResult?: LintResult;
+}
+export interface ScorerResult {
+    checkName: string;
+    passed: boolean;
+    scoreDelta: number;
+    isCritical: boolean;
+    metadata?: Record<string, any>;
+}
+export type ScorerFunction = (context: ScorerContext) => ScorerResult | Promise<ScorerResult>;
+export declare const PASSING_SCORE_THRESHOLD = 80;
+export declare const SCORING_CONFIG: {
+    readonly STARTING_SCORE: 100;
+    readonly MINIMUM_SCORE: 0;
+    readonly CRITICAL_FAILURE_PENALTY: -100;
+    readonly NON_CRITICAL_DEDUCTIONS: {
+        readonly LINT_ERROR: -6;
+        readonly LINT_WARNING: -1;
+        readonly TYPE_ERROR: -6;
+        readonly API_OUTSIDE_RANGE: -10;
+        readonly MISSING_NON_CRITICAL_COMPONENT: -10;
+        readonly MISSING_NON_CRITICAL_PROPERTY: -6;
+        readonly MISSING_BIND_ENTITY: -6;
+        readonly MISSING_FRONTEND_ENTITY: -10;
+    };
+};
+//# sourceMappingURL=scorer-types.d.ts.map

package/dist/ai-service/evals/helpers/scorer-types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scorer-types.d.ts","sourceRoot":"","sources":["../../../../src/ai-service/evals/helpers/scorer-types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAC9D,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAK5C,MAAM,WAAW,aAAa;IAE5B,MAAM,EAAE,eAAe,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;IAC5C,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC;IAC5C,IAAI,EAAE,eAAe,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC;IAC7C,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,gBAAgB,EAAE,eAAe,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;IAC1D,UAAU,EAAE,eAAe,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;IAGpD,MAAM,EAAE,eAAe,CAAC;IAGxB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC;AAED,MAAM,MAAM,cAAc,GAAG,CAC3B,OAAO,EAAE,aAAa,KACnB,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;AAG1C,eAAO,MAAM,uBAAuB,KAAK,CAAC;AAG1C,eAAO,MAAM,cAAc;;;;;;;;;;;;;;CAcjB,CAAC"}

package/dist/ai-service/evals/helpers/scorer-types.js

@@ -0,0 +1,19 @@
+// Test passing threshold
+export const PASSING_SCORE_THRESHOLD = 80;
+// Scoring configuration constants
+export const SCORING_CONFIG = {
+    STARTING_SCORE: 100,
+    MINIMUM_SCORE: 0,
+    CRITICAL_FAILURE_PENALTY: -100,
+    NON_CRITICAL_DEDUCTIONS: {
+        LINT_ERROR: -6,
+        LINT_WARNING: -1,
+        TYPE_ERROR: -6,
+        API_OUTSIDE_RANGE: -10,
+        MISSING_NON_CRITICAL_COMPONENT: -10,
+        MISSING_NON_CRITICAL_PROPERTY: -6,
+        MISSING_BIND_ENTITY: -6,
+        MISSING_FRONTEND_ENTITY: -10,
+    },
+};
+//# sourceMappingURL=scorer-types.js.map

package/dist/ai-service/evals/helpers/scorer-types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scorer-types.js","sourceRoot":"","sources":["../../../../src/ai-service/evals/helpers/scorer-types.ts"],"names":[],"mappings":"AAoCA,yBAAyB;AACzB,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAE1C,kCAAkC;AAClC,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,cAAc,EAAE,GAAG;IACnB,aAAa,EAAE,CAAC;IAChB,wBAAwB,EAAE,CAAC,GAAG;IAC9B,uBAAuB,EAAE;QACvB,UAAU,EAAE,CAAC,CAAC;QACd,YAAY,EAAE,CAAC,CAAC;QAChB,UAAU,EAAE,CAAC,CAAC;QACd,iBAAiB,EAAE,CAAC,EAAE;QACtB,8BAA8B,EAAE,CAAC,EAAE;QACnC,6BAA6B,EAAE,CAAC,CAAC;QACjC,mBAAmB,EAAE,CAAC,CAAC;QACvB,uBAAuB,EAAE,CAAC,EAAE;KAC7B;CACO,CAAC"}

package/dist/ai-service/evals/llm-provider.d.ts

@@ -0,0 +1,2 @@
+export declare const llmProvider: import("../llm/types.js").LLMProvider;
+//# sourceMappingURL=llm-provider.d.ts.map

package/dist/ai-service/evals/llm-provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-provider.d.ts","sourceRoot":"","sources":["../../../src/ai-service/evals/llm-provider.ts"],"names":[],"mappings":"AAKA,eAAO,MAAM,WAAW,uCAItB,CAAC"}

package/dist/ai-service/evals/llm-provider.js

@@ -0,0 +1,9 @@
+import { createLLMProvider } from "../llm/provider.js";
+import { loadEnvVars } from "./load-env-vars.js";
+loadEnvVars();
+export const llmProvider = createLLMProvider({
+    anthropic: {
+        apiKey: process.env.SUPERBLOCKS_AI_SERVICE_EVALS_ANTHROPIC_API_KEY,
+    },
+});
+//# sourceMappingURL=llm-provider.js.map

package/dist/ai-service/evals/llm-provider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-provider.js","sourceRoot":"","sources":["../../../src/ai-service/evals/llm-provider.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEjD,WAAW,EAAE,CAAC;AAEd,MAAM,CAAC,MAAM,WAAW,GAAG,iBAAiB,CAAC;IAC3C,SAAS,EAAE;QACT,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,8CAA8C;KACnE;CACF,CAAC,CAAC"}

package/dist/ai-service/index.d.ts

@@ -1,23 +1,32 @@
 import { type ISocketClient, TracedEventEmitter } from "@superblocksteam/shared";
+import { AppContextStore } from "./context/app-context.js";
 import type { AiServiceConfig, AiServiceEvents } from "./types.ts";
 import type { SourceTracker } from "../source-tracker.js";
 import type { AiGenerateRequest, EditorClientMethods } from "@superblocksteam/library-shared/types";
 export declare class AiService extends TracedEventEmitter<AiServiceEvents> {
     private readonly config;
     private artifactProcessor;
-    private anthropicProvider;
     private templateRenderer;
     private appShell;
     private sourceTrackerInterface;
     private integrationStore;
+    private _appContext;
+    private llmProviderSettings;
     private clark;
+    get appContext(): AppContextStore;
     constructor(config: AiServiceConfig);
     private createClarkTransitionHandler;
+    private registerPromptInterpretationTask;
     handleAiGenerate(request: AiGenerateRequest, peer: ISocketClient<EditorClientMethods>): Promise<void>;
+    private submitAcceptanceEvalutation;
     handleUserAcceptedDraft(peer: ISocketClient<EditorClientMethods>): Promise<void>;
     handleUserRejectedDraft(peer: ISocketClient<EditorClientMethods>): Promise<void>;
     handleUserCanceled(): Promise<void>;
     handleBuildSystemError(error: Error): void;
+    handlePromptInterpretation(request: {
+        prompt: string;
+        jwt: string;
+    }, peer: ISocketClient<EditorClientMethods>): Promise<void>;
     isBusy(): boolean;
     setSourceTracker(sourceTracker: SourceTracker | undefined): void;
 }

package/dist/ai-service/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/ai-service/index.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,aAAa,EAClB,kBAAkB,EACnB,MAAM,yBAAyB,CAAC;AAyCjC,OAAO,KAAK,EAEV,eAAe,EACf,eAAe,EAChB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,KAAK,EACV,iBAAiB,EACjB,mBAAmB,EACpB,MAAM,uCAAuC,CAAC;AAE/C,qBAAa,SAAU,SAAQ,kBAAkB,CAAC,eAAe,CAAC;IASpD,OAAO,CAAC,QAAQ,CAAC,MAAM;IARnC,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,sBAAsB,CAAyB;IACvD,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,KAAK,CAAQ;gBAEQ,MAAM,EAAE,eAAe;IAwDpD,OAAO,CAAC,4BAA4B;IA+EvB,gBAAgB,CAC3B,OAAO,EAAE,iBAAiB,EAC1B,IAAI,EAAE,aAAa,CAAC,mBAAmB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC;IAkCH,uBAAuB,CAClC,IAAI,EAAE,aAAa,CAAC,mBAAmB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC;IAgBH,uBAAuB,CAClC,IAAI,EAAE,aAAa,CAAC,mBAAmB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC;IAgBH,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC;IAQzC,sBAAsB,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;IAgB1C,MAAM,IAAI,OAAO;IAQjB,gBAAgB,CAAC,aAAa,EAAE,aAAa,GAAG,SAAS;CAGjE"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/ai-service/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,aAAa,EAClB,kBAAkB,EACnB,MAAM,yBAAyB,CAAC;AAMjC,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAsC3D,OAAO,KAAK,EAEV,eAAe,EACf,eAAe,EAChB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,KAAK,EACV,iBAAiB,EACjB,mBAAmB,EACpB,MAAM,uCAAuC,CAAC;AAE/C,qBAAa,SAAU,SAAQ,kBAAkB,CAAC,eAAe,CAAC;IAepD,OAAO,CAAC,QAAQ,CAAC,MAAM;IAdnC,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,sBAAsB,CAAyB;IACvD,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,mBAAmB,CAAsB;IAEjD,OAAO,CAAC,KAAK,CAAQ;IAErB,IAAW,UAAU,IAAI,eAAe,CAEvC;gBAE4B,MAAM,EAAE,eAAe;IA0DpD,OAAO,CAAC,4BAA4B;YAoFtB,gCAAgC;IAwDjC,gBAAgB,CAC3B,OAAO,EAAE,iBAAiB,EAC1B,IAAI,EAAE,aAAa,CAAC,mBAAmB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC;IA4ChB,OAAO,CAAC,2BAA2B;IAmBtB,uBAAuB,CAClC,IAAI,EAAE,aAAa,CAAC,mBAAmB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC;IAkBH,uBAAuB,CAClC,IAAI,EAAE,aAAa,CAAC,mBAAmB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC;IAkBH,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC;IAQzC,sBAAsB,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;IAgBpC,0BAA0B,CACrC,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,EACxC,IAAI,EAAE,aAAa,CAAC,mBAAmB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC;IAoBT,MAAM,IAAI,OAAO;IAQjB,gBAAgB,CAAC,aAAa,EAAE,aAAa,GAAG,SAAS;CAOjE"}