langium-ai-tools 4.1.5 → 4.2.0

package/dist/testing/index.d.ts

@@ -0,0 +1,118 @@
+/**
+ * Testing API for langium-ai evaluations
+ *
+ * Provides a vitest-style API for defining evaluation test cases.
+ */
+import type { EvaluatorResult } from '../evaluator/evaluator.js';
+/**
+ * Standard data that can be expected from a case
+ */
+export type EvaluationData = {
+    /**
+     * Whether this case passed
+     */
+    passed: boolean;
+    /**
+     * Whether this case was skipped (via .skip() or .only() filtering)
+     */
+    skipped?: boolean;
+    /**
+     * When passed is false, expect an error here
+     */
+    error?: Error | string;
+};
+/**
+ * Evaluation case result from running the lai cli
+ *
+ * A modified version of the baseline evaluator result, to accomodate for additional
+ * requisite properties (certain metadat for cases + suites & a 'pass' result)
+ */
+export type EvaluationCaseResult = EvaluatorResult<EvaluationData> & {
+    metadata: {
+        /**
+         * Containing evaluation file for the associated case
+         */
+        evalFile: string;
+        /**
+         * Name of the suite that we ran under
+         */
+        suiteName: string;
+        /**
+         * Evaluation case name
+         */
+        caseName: string;
+        /**
+         * Duration of evaluation case from start to finish
+         */
+        duration: number;
+    };
+};
+export interface EvalContext {
+    systemPrompt: string;
+    project: {
+        name: string;
+    };
+}
+/**
+ * Evalutor function definition, takes some context & produce evaluation data
+ */
+export type EvaluatorFunction = (ctx: EvalContext) => Promise<EvaluationData>;
+interface EvalDefinition {
+    name: string;
+    fn: EvaluatorFunction;
+    skip?: boolean;
+    only?: boolean;
+}
+interface EvalSuite {
+    name: string;
+    evaluations: EvalDefinition[];
+    beforeAllHook?: () => void | Promise<void>;
+    afterAllHook?: () => void | Promise<void>;
+    beforeEachHook?: () => void | Promise<void>;
+    afterEachHook?: () => void | Promise<void>;
+    skip?: boolean;
+    only?: boolean;
+}
+/**
+ * Define a test suite
+ */
+export declare function describe(name: string, fn: () => void): void;
+export declare namespace describe {
+    var skip: (name: string, fn: () => void) => void;
+    var only: (name: string, fn: () => void) => void;
+}
+/**
+ * Define an evaluation case
+ */
+export declare function evaluation(name: string, fn: EvaluatorFunction): void;
+export declare namespace evaluation {
+    var skip: (name: string, fn: EvaluatorFunction) => void;
+    var only: (name: string, fn: EvaluatorFunction) => void;
+    var each: <T>(cases: T[]) => (name: string, fn: (data: T) => EvaluatorFunction) => void;
+}
+/**
+ * Define a hook that runs before each evaluation in the suite
+ */
+export declare function beforeEach(fn: () => void | Promise<void>): void;
+/**
+ * Define a hook that runs after each evaluation in the suite
+ */
+export declare function afterEach(fn: () => void | Promise<void>): void;
+/**
+ * Define a hook that runs once before all evaluations in the suite
+ */
+export declare function beforeAll(fn: () => void | Promise<void>): void;
+/**
+ * Define a hook that runs once after all evaluations in the suite
+ */
+export declare function afterAll(fn: () => void | Promise<void>): void;
+/**
+ * Export collected suites for runner
+ */
+export declare function getCollectedSuites(): EvalSuite[];
+/**
+ * Clear all collected suites
+ */
+export declare function clearSuites(): void;
+export { runEvalFile, type EvalProgressCallback } from './runner.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/testing/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/testing/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAEjE;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG;IAC3B;;OAEG;IACH,MAAM,EAAE,OAAO,CAAC;IAEhB;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;OAEG;IACH,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;CACxB,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,oBAAoB,GAAG,eAAe,CAAC,cAAc,CAAC,GAAG;IACnE,QAAQ,EAAE;QACR;;WAEG;QACH,QAAQ,EAAE,MAAM,CAAC;QAEjB;;WAEG;QACH,SAAS,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,QAAQ,EAAE,MAAM,CAAC;QAEjB;;WAEG;QACH,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAA;CACF,CAAC;AAEF,MAAM,WAAW,WAAW;IAC1B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG,CAAC,GAAG,EAAE,WAAW,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;AAG9E,UAAU,cAAc;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,iBAAiB,CAAC;IACtB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,UAAU,SAAS;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,cAAc,EAAE,CAAC;IAC9B,aAAa,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3C,YAAY,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,cAAc,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,aAAa,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3C,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAKD;;GAEG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,IAAI,GAAG,IAAI,CAM3D;yBANe,QAAQ;qBAWO,MAAM,MAAM,MAAM,IAAI,KAAG,IAAI;qBAW7B,MAAM,MAAM,MAAM,IAAI,KAAG,IAAI;;AAQ5D;;GAEG;AACH,wBAAgB,UAAU,CACxB,IAAI,EAAE,MAAM,EACZ,EAAE,EAAE,iBAAiB,GACpB,IAAI,CAKN;yBARe,UAAU;qBAclB,MAAM,MACR,iBAAiB,KACpB,IAAI;qBAWC,MAAM,MACR,iBAAiB,KACpB,IAAI;eAkBoB,CAAC,SACnB,CAAC,EAAE,KACT,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,iBAAiB,KAAK,IAAI;;AAwC7D;;GAEG;AACH,wBAAgB,UAAU,CAAC,EAAE,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAK/D;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAK9D;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAK9D;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,EAAE,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAK7D;AAED;;GAEG;AACH,wBAAgB,kBAAkB,IAAI,SAAS,EAAE,CAIhD;AAED;;GAEG;AACH,wBAAgB,WAAW,IAAI,IAAI,CAGlC;AAGD,OAAO,EAAE,WAAW,EAAE,KAAK,oBAAoB,EAAE,MAAM,aAAa,CAAC"}

package/dist/testing/index.js

@@ -0,0 +1,163 @@
+/**
+ * Testing API for langium-ai evaluations
+ *
+ * Provides a vitest-style API for defining evaluation test cases.
+ */
+const suites = [];
+let currentSuite = null;
+/**
+ * Define a test suite
+ */
+export function describe(name, fn) {
+    const parentSuite = currentSuite;
+    currentSuite = { name, evaluations: [] };
+    fn();
+    suites.push(currentSuite);
+    currentSuite = parentSuite;
+}
+/**
+ * Define a test suite that will be skipped
+ */
+describe.skip = function (name, fn) {
+    const parentSuite = currentSuite;
+    currentSuite = { name, evaluations: [], skip: true };
+    fn();
+    suites.push(currentSuite);
+    currentSuite = parentSuite;
+};
+/**
+ * Define a test suite that will run exclusively (skips all other tests)
+ */
+describe.only = function (name, fn) {
+    const parentSuite = currentSuite;
+    currentSuite = { name, evaluations: [], only: true };
+    fn();
+    suites.push(currentSuite);
+    currentSuite = parentSuite;
+};
+/**
+ * Define an evaluation case
+ */
+export function evaluation(name, fn) {
+    if (!currentSuite) {
+        throw new Error('evaluation() must be called inside describe()');
+    }
+    currentSuite.evaluations.push({ name, fn });
+}
+/**
+ * Define an evaluation case that will be skipped
+ */
+evaluation.skip = function (name, fn) {
+    if (!currentSuite) {
+        throw new Error('evaluation.skip() must be called inside describe()');
+    }
+    currentSuite.evaluations.push({ name, fn, skip: true });
+};
+/**
+ * Define an evaluation case that will run exclusively (skips all other tests)
+ */
+evaluation.only = function (name, fn) {
+    if (!currentSuite) {
+        throw new Error('evaluation.only() must be called inside describe()');
+    }
+    currentSuite.evaluations.push({ name, fn, only: true });
+};
+/**
+ * Define parametrized evaluation cases
+ *
+ * @example
+ * evaluation.each([
+ *   { input: 'test1', expected: 'result1' },
+ *   { input: 'test2', expected: 'result2' }
+ * ])('testing $input', (data) => async (ctx) => {
+ *   // test implementation using data.input and data.expected
+ * });
+ */
+evaluation.each = function (cases) {
+    return (name, fn) => {
+        if (!currentSuite) {
+            throw new Error('evaluation.each() must be called inside describe()');
+        }
+        // create an evaluation for each case
+        for (let i = 0; i < cases.length; i++) {
+            const testCase = cases[i];
+            // replace placeholders in name with case values
+            let caseName = name;
+            // support $property syntax (e.g., "$input" -> testCase.input)
+            if (typeof testCase === 'object' && testCase !== null) {
+                caseName = name.replace(/\$(\w+)/g, (match, key) => {
+                    const value = testCase[key];
+                    return value !== undefined ? String(value) : match;
+                });
+            }
+            // support %i, %s, %o, etc. for positional replacement
+            caseName = caseName
+                .replace(/%s/g, String(testCase))
+                .replace(/%i/g, String(testCase))
+                .replace(/%o/g, JSON.stringify(testCase))
+                .replace(/%j/g, JSON.stringify(testCase));
+            // if no placeholders matched, append case index
+            if (caseName === name) {
+                caseName = `${name} [${i}]`;
+            }
+            currentSuite.evaluations.push({
+                name: caseName,
+                fn: fn(testCase)
+            });
+        }
+    };
+};
+/**
+ * Define a hook that runs before each evaluation in the suite
+ */
+export function beforeEach(fn) {
+    if (!currentSuite) {
+        throw new Error('beforeEach() must be called inside describe()');
+    }
+    currentSuite.beforeEachHook = fn;
+}
+/**
+ * Define a hook that runs after each evaluation in the suite
+ */
+export function afterEach(fn) {
+    if (!currentSuite) {
+        throw new Error('afterEach() must be called inside describe()');
+    }
+    currentSuite.afterEachHook = fn;
+}
+/**
+ * Define a hook that runs once before all evaluations in the suite
+ */
+export function beforeAll(fn) {
+    if (!currentSuite) {
+        throw new Error('beforeAll() must be called inside describe()');
+    }
+    currentSuite.beforeAllHook = fn;
+}
+/**
+ * Define a hook that runs once after all evaluations in the suite
+ */
+export function afterAll(fn) {
+    if (!currentSuite) {
+        throw new Error('afterAll() must be called inside describe()');
+    }
+    currentSuite.afterAllHook = fn;
+}
+/**
+ * Export collected suites for runner
+ */
+export function getCollectedSuites() {
+    const collected = [...suites];
+    suites.length = 0;
+    return collected;
+}
+/**
+ * Clear all collected suites
+ */
+export function clearSuites() {
+    suites.length = 0;
+    currentSuite = null;
+}
+// re-export runner types and functions
+export { runEvalFile } from './runner.js';
+//# sourceMappingURL=index.js.map

package/dist/testing/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/testing/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAmFH,MAAM,MAAM,GAAgB,EAAE,CAAC;AAC/B,IAAI,YAAY,GAAqB,IAAI,CAAC;AAE1C;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAc;IACnD,MAAM,WAAW,GAAG,YAAY,CAAC;IACjC,YAAY,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;IACzC,EAAE,EAAE,CAAC;IACL,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC1B,YAAY,GAAG,WAAW,CAAC;AAC7B,CAAC;AAED;;GAEG;AACH,QAAQ,CAAC,IAAI,GAAG,UAAS,IAAY,EAAE,EAAc;IACnD,MAAM,WAAW,GAAG,YAAY,CAAC;IACjC,YAAY,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACrD,EAAE,EAAE,CAAC;IACL,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC1B,YAAY,GAAG,WAAW,CAAC;AAC7B,CAAC,CAAC;AAEF;;GAEG;AACH,QAAQ,CAAC,IAAI,GAAG,UAAS,IAAY,EAAE,EAAc;IACnD,MAAM,WAAW,GAAG,YAAY,CAAC;IACjC,YAAY,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACrD,EAAE,EAAE,CAAC;IACL,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC1B,YAAY,GAAG,WAAW,CAAC;AAC7B,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,UAAU,CACxB,IAAY,EACZ,EAAqB;IAErB,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IACnE,CAAC;IACD,YAAY,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED;;GAEG;AACH,UAAU,CAAC,IAAI,GAAG,UAChB,IAAY,EACZ,EAAqB;IAErB,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACxE,CAAC;IACD,YAAY,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;AAC1D,CAAC,CAAC;AAEF;;GAEG;AACH,UAAU,CAAC,IAAI,GAAG,UAChB,IAAY,EACZ,EAAqB;IAErB,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACxE,CAAC;IACD,YAAY,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;AAC1D,CAAC,CAAC;AAEF;;;;;;;;;;GAUG;AACH,UAAU,CAAC,IAAI,GAAG,UAChB,KAAU;IAEV,OAAO,CAAC,IAAY,EAAE,EAAkC,EAAQ,EAAE;QAChE,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;QACxE,CAAC;QAED,qCAAqC;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1B,gDAAgD;YAChD,IAAI,QAAQ,GAAG,IAAI,CAAC;YAEpB,8DAA8D;YAC9D,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;gBACtD,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;oBACjD,MAAM,KAAK,GAAI,QAAoC,CAAC,GAAG,CAAC,CAAC;oBACzD,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;gBACrD,CAAC,CAAC,CAAC;YACL,CAAC;YAED,sDAAsD;YACtD,QAAQ,GAAG,QAAQ;iBAChB,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;iBAChC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;iBAChC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;iBACxC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;YAE5C,gDAAgD;YAChD,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;gBACtB,QAAQ,GAAG,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC;YAC9B,CAAC;YAED,YAAY,CAAC,WAAW,CAAC,IAAI,CAAC;gBAC5B,IAAI,EAAE,QAAQ;gBACd,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC;aACjB,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,EAA8B;IACvD,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IACnE,CAAC;IACD,YAAY,CAAC,cAAc,GAAG,EAAE,CAAC;AACnC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,EAA8B;IACtD,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IAClE,CAAC;IACD,YAAY,CAAC,aAAa,GAAG,EAAE,CAAC;AAClC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,EAA8B;IACtD,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IAClE,CAAC;IACD,YAAY,CAAC,aAAa,GAAG,EAAE,CAAC;AAClC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,EAA8B;IACrD,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IACD,YAAY,CAAC,YAAY,GAAG,EAAE,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB;IAChC,MAAM,SAAS,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IAC9B,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAClB,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW;IACzB,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAClB,YAAY,GAAG,IAAI,CAAC;AACtB,CAAC;AAED,uCAAuC;AACvC,OAAO,EAAE,WAAW,EAA6B,MAAM,aAAa,CAAC"}

package/dist/testing/runner.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Test runner for evaluation files
+ */
+import type { EvalContext, EvaluationCaseResult } from './index.js';
+/**
+ * Progress callback for evaluation execution
+ */
+export interface EvalProgressCallback {
+    (current: number, total: number): void;
+}
+/**
+ * Result callback for evaluation execution (called as each test completes)
+ */
+export interface EvalResultCallback {
+    (result: EvaluationCaseResult, current: number, total: number): void;
+}
+/**
+ * Run a single evaluation file
+ */
+export declare function runEvalFile(filePath: string, context: EvalContext, onProgress?: EvalProgressCallback, onResult?: EvalResultCallback): Promise<EvaluationCaseResult[]>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/testing/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/testing/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAKpE;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,CAAC,MAAM,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtE;AAED;;GAEG;AACH,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,WAAW,EACpB,UAAU,CAAC,EAAE,oBAAoB,EACjC,QAAQ,CAAC,EAAE,kBAAkB,GAC5B,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAiKjC"}

package/dist/testing/runner.js

@@ -0,0 +1,159 @@
+/**
+ * Test runner for evaluation files
+ */
+import { pathToFileURL } from 'url';
+import { register } from 'node:module';
+// track if tsx loader is registered
+let tsxLoaderRegistered = false;
+/**
+ * Run a single evaluation file
+ */
+export async function runEvalFile(filePath, context, onProgress, onResult) {
+    // register tsx loader for TypeScript support (only once)
+    if (!tsxLoaderRegistered && filePath.endsWith('.ts')) {
+        try {
+            // use tsx's ESM loader (Node 20.6+ compatible)
+            register('tsx/esm', import.meta.url);
+            tsxLoaderRegistered = true;
+        }
+        catch (error) {
+            // tsx may not be available, continue anyway (silence warning since it's working)
+        }
+    }
+    const { getCollectedSuites, clearSuites } = await import('./index.js');
+    // clear any previous state
+    clearSuites();
+    // dynamically import eval file (triggers describe/test calls)
+    // add cache-busting query parameter to force fresh import (prevents module caching issues)
+    await import(pathToFileURL(filePath).href + '?run=' + Date.now());
+    // collect suites
+    const suites = getCollectedSuites();
+    // check if any suite or evaluation has .only()
+    const hasOnly = suites.some(s => s.only || s.evaluations.some(e => e.only));
+    // count total evaluations for progress tracking
+    const totalEvaluations = suites.reduce((sum, suite) => sum + suite.evaluations.length, 0);
+    let completedEvaluations = 0;
+    // execute all tests (including creating results for skipped ones)
+    const results = [];
+    for (const suite of suites) {
+        // determine if this entire suite is skipped
+        const suiteSkipped = suite.skip || (hasOnly && !suite.only && !suite.evaluations.some(e => e.only));
+        // run beforeAll hook once before all evaluations in the suite
+        if (!suiteSkipped && suite.beforeAllHook) {
+            try {
+                await suite.beforeAllHook();
+            }
+            catch (error) {
+                console.error('beforeAll hook failed:', error);
+                // continue anyway, tests will fail if setup is incomplete
+            }
+        }
+        for (const evalDef of suite.evaluations) {
+            // determine if this evaluation is skipped
+            const evalSkipped = suiteSkipped || evalDef.skip || (hasOnly && !evalDef.only && !suite.only);
+            if (evalSkipped) {
+                // add skipped result without executing the test
+                const skippedResult = {
+                    name: `${suite.name} - ${evalDef.name}`,
+                    metadata: {
+                        evalFile: filePath,
+                        suiteName: suite.name,
+                        caseName: evalDef.name,
+                        duration: 0
+                    },
+                    data: {
+                        passed: false,
+                        skipped: true
+                    }
+                };
+                results.push(skippedResult);
+                completedEvaluations++;
+                if (onProgress) {
+                    onProgress(completedEvaluations, totalEvaluations);
+                }
+                if (onResult) {
+                    onResult(skippedResult, completedEvaluations, totalEvaluations);
+                }
+                continue;
+            }
+            // execute non-skipped test
+            const start = Date.now();
+            try {
+                // run beforeEach hook if it exists
+                if (suite.beforeEachHook) {
+                    await suite.beforeEachHook();
+                }
+                // get core evaluation data
+                const evaluationData = await evalDef.fn(context);
+                // run afterEach hook if it exists
+                if (suite.afterEachHook) {
+                    await suite.afterEachHook();
+                }
+                // construct a result around it
+                const evaluatorCaseResult = {
+                    name: `${suite.name} - ${evalDef.name}`,
+                    metadata: {
+                        evalFile: filePath,
+                        suiteName: suite.name,
+                        caseName: evalDef.name,
+                        duration: Date.now() - start
+                    },
+                    data: evaluationData
+                };
+                results.push(evaluatorCaseResult);
+                completedEvaluations++;
+                if (onProgress) {
+                    onProgress(completedEvaluations, totalEvaluations);
+                }
+                if (onResult) {
+                    onResult(evaluatorCaseResult, completedEvaluations, totalEvaluations);
+                }
+            }
+            catch (error) {
+                // run afterEach hook even on error
+                if (suite.afterEachHook) {
+                    try {
+                        await suite.afterEachHook();
+                    }
+                    catch (afterEachError) {
+                        // log afterEach errors but don't fail the test twice
+                        console.error('afterEach hook failed:', afterEachError);
+                    }
+                }
+                // treat errors as failures
+                const failureResult = {
+                    name: `${suite.name} - ${evalDef.name}`,
+                    metadata: {
+                        evalFile: filePath,
+                        suiteName: suite.name,
+                        caseName: evalDef.name,
+                        duration: Date.now() - start,
+                    },
+                    data: {
+                        passed: false,
+                        error: error instanceof Error ? error.message : String(error)
+                    }
+                };
+                results.push(failureResult);
+                completedEvaluations++;
+                if (onProgress) {
+                    onProgress(completedEvaluations, totalEvaluations);
+                }
+                if (onResult) {
+                    onResult(failureResult, completedEvaluations, totalEvaluations);
+                }
+            }
+        }
+        // run afterAll hook once after all evaluations in the suite
+        if (!suiteSkipped && suite.afterAllHook) {
+            try {
+                await suite.afterAllHook();
+            }
+            catch (error) {
+                console.error('afterAll hook failed:', error);
+            }
+        }
+    }
+    return results;
+}
+//# sourceMappingURL=runner.js.map

package/dist/testing/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/testing/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAGvC,oCAAoC;AACpC,IAAI,mBAAmB,GAAG,KAAK,CAAC;AAgBhC;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAAgB,EAChB,OAAoB,EACpB,UAAiC,EACjC,QAA6B;IAE7B,yDAAyD;IACzD,IAAI,CAAC,mBAAmB,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACrD,IAAI,CAAC;YACH,+CAA+C;YAC/C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrC,mBAAmB,GAAG,IAAI,CAAC;QAC7B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,iFAAiF;QACnF,CAAC;IACH,CAAC;IAED,MAAM,EAAE,kBAAkB,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;IAEvE,2BAA2B;IAC3B,WAAW,EAAE,CAAC;IAEd,8DAA8D;IAC9D,2FAA2F;IAC3F,MAAM,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,IAAI,GAAG,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAElE,iBAAiB;IACjB,MAAM,MAAM,GAAG,kBAAkB,EAAE,CAAC;IAEpC,+CAA+C;IAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAE5E,gDAAgD;IAChD,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC1F,IAAI,oBAAoB,GAAG,CAAC,CAAC;IAE7B,kEAAkE;IAClE,MAAM,OAAO,GAA2B,EAAE,CAAC;IAE3C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,4CAA4C;QAC5C,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAEpG,8DAA8D;QAC9D,IAAI,CAAC,YAAY,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;YACzC,IAAI,CAAC;gBACH,MAAM,KAAK,CAAC,aAAa,EAAE,CAAC;YAC9B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,CAAC;gBAC/C,0DAA0D;YAC5D,CAAC;QACH,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;YACxC,0CAA0C;YAC1C,MAAM,WAAW,GAAG,YAAY,IAAI,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAE9F,IAAI,WAAW,EAAE,CAAC;gBAChB,gDAAgD;gBAChD,MAAM,aAAa,GAAyB;oBAC1C,IAAI,EAAE,GAAG,KAAK,CAAC,IAAI,MAAM,OAAO,CAAC,IAAI,EAAE;oBACvC,QAAQ,EAAE;wBACR,QAAQ,EAAE,QAAQ;wBAClB,SAAS,EAAE,KAAK,CAAC,IAAI;wBACrB,QAAQ,EAAE,OAAO,CAAC,IAAI;wBACtB,QAAQ,EAAE,CAAC;qBACZ;oBACD,IAAI,EAAE;wBACJ,MAAM,EAAE,KAAK;wBACb,OAAO,EAAE,IAAI;qBACd;iBACF,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAC5B,oBAAoB,EAAE,CAAC;gBACvB,IAAI,UAAU,EAAE,CAAC;oBACf,UAAU,CAAC,oBAAoB,EAAE,gBAAgB,CAAC,CAAC;gBACrD,CAAC;gBACD,IAAI,QAAQ,EAAE,CAAC;oBACb,QAAQ,CAAC,aAAa,EAAE,oBAAoB,EAAE,gBAAgB,CAAC,CAAC;gBAClE,CAAC;gBACD,SAAS;YACX,CAAC;YAED,2BAA2B;YAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,mCAAmC;gBACnC,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC;oBACzB,MAAM,KAAK,CAAC,cAAc,EAAE,CAAC;gBAC/B,CAAC;gBAED,2BAA2B;gBAC3B,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;gBAEjD,kCAAkC;gBAClC,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;oBACxB,MAAM,KAAK,CAAC,aAAa,EAAE,CAAC;gBAC9B,CAAC;gBAED,+BAA+B;gBAC/B,MAAM,mBAAmB,GAAyB;oBAChD,IAAI,EAAE,GAAG,KAAK,CAAC,IAAI,MAAM,OAAO,CAAC,IAAI,EAAE;oBACvC,QAAQ,EAAE;wBACR,QAAQ,EAAE,QAAQ;wBAClB,SAAS,EAAE,KAAK,CAAC,IAAI;wBACrB,QAAQ,EAAE,OAAO,CAAC,IAAI;wBACtB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;qBAC7B;oBACD,IAAI,EAAE,cAAc;iBACrB,CAAC;gBAEF,OAAO,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;gBAClC,oBAAoB,EAAE,CAAC;gBACvB,IAAI,UAAU,EAAE,CAAC;oBACf,UAAU,CAAC,oBAAoB,EAAE,gBAAgB,CAAC,CAAC;gBACrD,CAAC;gBACD,IAAI,QAAQ,EAAE,CAAC;oBACb,QAAQ,CAAC,mBAAmB,EAAE,oBAAoB,EAAE,gBAAgB,CAAC,CAAC;gBACxE,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,mCAAmC;gBACnC,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;oBACxB,IAAI,CAAC;wBACH,MAAM,KAAK,CAAC,aAAa,EAAE,CAAC;oBAC9B,CAAC;oBAAC,OAAO,cAAc,EAAE,CAAC;wBACxB,qDAAqD;wBACrD,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,cAAc,CAAC,CAAC;oBAC1D,CAAC;gBACH,CAAC;gBAED,2BAA2B;gBAC3B,MAAM,aAAa,GAAyB;oBAC1C,IAAI,EAAE,GAAG,KAAK,CAAC,IAAI,MAAM,OAAO,CAAC,IAAI,EAAE;oBACvC,QAAQ,EAAE;wBACR,QAAQ,EAAE,QAAQ;wBAClB,SAAS,EAAE,KAAK,CAAC,IAAI;wBACrB,QAAQ,EAAE,OAAO,CAAC,IAAI;wBACtB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;qBAC7B;oBACD,IAAI,EAAE;wBACJ,MAAM,EAAE,KAAK;wBACb,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;qBAC9D;iBACF,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAC5B,oBAAoB,EAAE,CAAC;gBACvB,IAAI,UAAU,EAAE,CAAC;oBACf,UAAU,CAAC,oBAAoB,EAAE,gBAAgB,CAAC,CAAC;gBACrD,CAAC;gBACD,IAAI,QAAQ,EAAE,CAAC;oBACb,QAAQ,CAAC,aAAa,EAAE,oBAAoB,EAAE,gBAAgB,CAAC,CAAC;gBAClE,CAAC;YACH,CAAC;QACH,CAAC;QAED,4DAA4D;QAC5D,IAAI,CAAC,YAAY,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;YACxC,IAAI,CAAC;gBACH,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC;YAC7B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,KAAK,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/util.d.ts

@@ -0,0 +1,9 @@
+import type { LangiumServices, LangiumSharedServices } from "langium/lsp";
+/**
+ * General service set that is produced by most Langium implementations
+ */
+export type LangiumServiceSet = {
+    shared: LangiumSharedServices;
+    grammar: LangiumServices;
+};
+//# sourceMappingURL=util.d.ts.map

package/dist/util.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAA;AAEzE;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC5B,MAAM,EAAE,qBAAqB,CAAA;IAC7B,OAAO,EAAE,eAAe,CAAA;CAC3B,CAAA"}

package/dist/util.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=util.js.map

package/dist/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":""}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langium-ai-tools",
-  "version": "4.1.5",
+  "version": "4.2.0",
   "description": "Tooling for building AI Applications that leverage Langium DSLs",
   "repository": {
     "type": "git",
@@ -50,7 +50,7 @@
   "dependencies": {
     "@protobuf-ts/runtime": "^2.11.1",
     "js-yaml": "^4.1.1",
-    "langium": "~4.1.0"
+    "langium": "~4.2.1"
   },
   "devDependencies": {
     "@protobuf-ts/plugin": "^2.11.1",