langium-ai-tools 4.1.0 → 4.1.2

package/README.md

@@ -26,10 +26,10 @@ Langium AI tools tracks tightly with Langium releases. If you're using Langium 3
 
 ```bash
 # if you're using Langium 4.1.X
-npm i --save langium@^4.1.0
+npm i --save langium-ai-tools@^4.1.0
 
 # or 3.5.X
-npm i --save langium@^3.5.0
+npm i --save langium-ai-tools@^3.5.0
 ```
 
 We don't actively support Langium 2.X or earlier.
@@ -45,7 +45,7 @@ Langium AI Tools presents various splitting utilities that are simple but flexib
 
 For example, if you have a DSL that looks like this:
 
-```dsl
+```
 // A dsl that allows writing functions...
 function foo() { ... }
 function bar() { ... }
@@ -66,7 +66,7 @@ const chunks: string[] = splitByNode(
 ```
 
 And you would get back something like this:
-```json
+```
 [
     'function foo() { ... }',
     'function bar() { ... }'
@@ -100,7 +100,7 @@ const programMap: string[] = mapper.map(myProg);
 
 Which would give you an output like so:
 
-```json
+```
 [
     'func foo',
     'func bar'

package/dist/analyzer/document-analyzer.d.ts

@@ -0,0 +1,126 @@
+/******************************************************************************
+ * Copyright 2025 TypeFox GmbH
+ * This program and the accompanying materials are made available under the
+ * terms of the MIT License, which is available in the project root.
+ *
+ * @author Dennis Hübner
+ ******************************************************************************/
+import { Grammar, GrammarAST, LangiumDocument } from "langium";
+import { LangiumServices } from "langium/lsp";
+import { EvaluationContext } from "../evaluator/document-evaluator.js";
+import { EvaluatorResult } from "../evaluator/evaluator.js";
+import { LangiumEvaluator, LangiumEvaluatorResultData } from "../evaluator/langium-evaluator.js";
+/**
+ * Extends LangiumEvaluator and adds analysis capabilities.
+ */
+export declare class LangiumDocumentAnalyzer<T extends LangiumServices> extends LangiumEvaluator<T> {
+    static readonly METADATA_KEY = "syntax_statistics";
+    private readonly analysisOptions;
+    /**
+     * Creates an instance of LangiumDocumentAnalyzer.
+     * @param services Langium services
+     * @param analysisOptions Analysis options
+     * @example
+     * ```typescript
+     * const analyzer = new LangiumDocumentAnalyzer(services, {
+     *     analysisMode: AnalysisMode.ALL,
+     *     excludeRules: ['DeprecatedRule'],
+     *     computeDiversity: false
+     * });
+     * ```
+     */
+    constructor(services: T, analysisOptions?: Partial<AnalysisOptions>);
+    evaluateDocument(doc: LangiumDocument, ctx: EvaluationContext): EvaluatorResult<LangiumEvaluatorResultData>;
+    collectSyntaxUsageStatistics(doc: LangiumDocument, grammar: Grammar): SyntaxStatistic;
+    /**
+     * Computes coverage as percentage of used rules over all available rules
+     */
+    computeCoverage(ruleUsage: Record<string, number>): number;
+    /**
+     * Computes Shannon entropy - measure of information diversity
+     * Higher values indicate more diverse usage patterns
+     */
+    computeEntropy(ruleUsage: Record<string, number>): number;
+    /**
+     * Computes Gini coefficient - measure of inequality in rule usage
+     * 0 = perfect equality, 1 = maximum inequality
+     */
+    computeGiniCoefficient(ruleUsage: Record<string, number>): number;
+    /**
+     * Computes Simpson's diversity index - probability that two randomly selected items are different
+     * Higher values indicate more diversity
+     */
+    computeSimpsonIndex(ruleUsage: Record<string, number>): number;
+    /**
+     * Extracts syntax statistics from the evaluation result.
+     * @param result The evaluation result.
+     * @returns The extracted syntax statistics or undefined if not found.
+     */
+    extractStatisticsFromResult(result: Partial<EvaluatorResult> | undefined): SyntaxStatistic | undefined;
+    protected collectAllRules(grammar: Grammar): GrammarAST.AbstractRule[];
+    protected createEmptySyntaxStatistic(): SyntaxStatistic;
+}
+/**
+ * Analysis mode for controlling what analysis operations to perform
+ */
+export declare enum AnalysisMode {
+    ALL = "ALL",
+    NO_STATISTIC = "NO_STATISTIC"
+}
+interface AnalysisOptions {
+    analysisMode: AnalysisMode;
+    /**
+     * Filter for specific rules (e.g deprecated) to exclude in the analysis.
+     * Rule WS (whitespace) is always excluded.
+     */
+    excludeRules: string[];
+    /**
+     * Whether to include rules from imported grammars. Default is true.
+     */
+    includeImportedRules: boolean;
+    /**
+     * Whether to include hidden tokens (like comments, whitespace) in the analysis. Default is false.
+     * Rule WS (whitespace) is always excluded.
+     */
+    includeHiddenRules: boolean;
+    /**
+     * Whether to compute diversity metrics for rule usage. Default is true.
+     */
+    computeDiversity: boolean;
+}
+/**
+ * Type representing syntax usage statistics.
+ */
+export type SyntaxStatistic = {
+    /** Map of rule names to their usage counts */
+    ruleUsage: Record<string, number>;
+    /** Percentage of used rules compared to all available rules */
+    coverage: number;
+    /** Diversity metrics for rule usage patterns */
+    diversity: {
+        /**
+         * Shannon entropy - information diversity measure.
+         * **Range:** 0 to log₂(n) where n = number of rules.
+         * - **Low (0-1):** dominated by few rules
+         * - **Medium (1-3):** moderate diversity
+         * - **High (>3):** high diversity
+         */
+        entropy: number;
+        /**
+         * Gini coefficient - inequality measure. Range: 0 to 1.
+         * - **Low (0-0.3):** equal distribution
+         * - **Medium (0.3-0.7):** moderate inequality
+         * - **High (0.7-1):** high inequality
+         */
+        giniCoefficient: number;
+        /**
+         * Simpson's diversity index - probability that two randomly selected items are different. **Range:** 0 to 1.
+         * - **Low (0-0.3):** low diversity
+         * - **Medium (0.3-0.7):** moderate diversity
+         * - **High (0.7-1):** high diversity
+         */
+        simpsonIndex: number;
+    };
+};
+export {};
+//# sourceMappingURL=document-analyzer.d.ts.map

package/dist/analyzer/document-analyzer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"document-analyzer.d.ts","sourceRoot":"","sources":["../../src/analyzer/document-analyzer.ts"],"names":[],"mappings":"AAAA;;;;;;gFAMgF;AAEhF,OAAO,EAAY,OAAO,EAAE,UAAU,EAAE,eAAe,EAAiB,MAAM,SAAS,CAAC;AAExF,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,oCAAoC,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,0BAA0B,EAAE,MAAM,mCAAmC,CAAC;AAEjG;;GAEG;AACH,qBAAa,uBAAuB,CAAC,CAAC,SAAS,eAAe,CAAE,SAAQ,gBAAgB,CAAC,CAAC,CAAC;IAEvF,gBAAuB,YAAY,uBAAuB;IAE1D,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAElD;;;;;;;;;;;;OAYG;gBACS,QAAQ,EAAE,CAAC,EAAE,eAAe,GAAE,OAAO,CAAC,eAAe,CAAM;IAKvE,gBAAgB,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,EAAE,iBAAiB,GAAG,eAAe,CAAC,0BAA0B,CAAC;IAU3G,4BAA4B,CAAC,GAAG,EAAE,eAAe,EAAE,OAAO,EAAE,OAAO,GAAG,eAAe;IAqDrF;;OAEG;IACH,eAAe,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM;IAK1D;;;OAGG;IACH,cAAc,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM;IAczD;;;OAGG;IACH,sBAAsB,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM;IAejE;;;OAGG;IACH,mBAAmB,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM;IAa9D;;;;OAIG;IACH,2BAA2B,CAAC,MAAM,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,SAAS,GAAG,eAAe,GAAG,SAAS;IAQtG,SAAS,CAAC,eAAe,CAAC,OAAO,EAAE,OAAO,GAAG,UAAU,CAAC,YAAY,EAAE;IAWtE,SAAS,CAAC,0BAA0B,IAAI,eAAe;CAW1D;AAED;;GAEG;AACH,oBAAY,YAAY;IACpB,GAAG,QAAQ;IACX,YAAY,iBAAiB;CAChC;AAED,UAAU,eAAe;IACrB,YAAY,EAAE,YAAY,CAAC;IAC3B;;;OAGG;IACH,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB;;OAEG;IACH,oBAAoB,EAAE,OAAO,CAAC;IAC9B;;;OAGG;IACH,kBAAkB,EAAE,OAAO,CAAC;IAC5B;;OAEG;IACH,gBAAgB,EAAE,OAAO,CAAC;CAC7B;AAUD;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG;IAC1B,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAElC,+DAA+D;IAC/D,QAAQ,EAAE,MAAM,CAAC;IAEjB,gDAAgD;IAChD,SAAS,EAAE;QAEP;;;;;;WAMG;QACH,OAAO,EAAE,MAAM,CAAC;QAEhB;;;;;WAKG;QACH,eAAe,EAAE,MAAM,CAAC;QAExB;;;;;WAKG;QACH,YAAY,EAAE,MAAM,CAAC;KACxB,CAAC;CACL,CAAA"}

package/dist/analyzer/document-analyzer.js

@@ -0,0 +1,193 @@
+/******************************************************************************
+ * Copyright 2025 TypeFox GmbH
+ * This program and the accompanying materials are made available under the
+ * terms of the MIT License, which is available in the project root.
+ *
+ * @author Dennis Hübner
+ ******************************************************************************/
+import { CstUtils, GrammarAST, isLeafCstNode } from "langium";
+import { resolveTransitiveImports, } from 'langium/grammar';
+import { LangiumEvaluator } from "../evaluator/langium-evaluator.js";
+/**
+ * Extends LangiumEvaluator and adds analysis capabilities.
+ */
+export class LangiumDocumentAnalyzer extends LangiumEvaluator {
+    /**
+     * Creates an instance of LangiumDocumentAnalyzer.
+     * @param services Langium services
+     * @param analysisOptions Analysis options
+     * @example
+     * ```typescript
+     * const analyzer = new LangiumDocumentAnalyzer(services, {
+     *     analysisMode: AnalysisMode.ALL,
+     *     excludeRules: ['DeprecatedRule'],
+     *     computeDiversity: false
+     * });
+     * ```
+     */
+    constructor(services, analysisOptions = {}) {
+        super(services);
+        this.analysisOptions = { ...DEFAULT_OPTIONS, ...analysisOptions };
+    }
+    evaluateDocument(doc, ctx) {
+        const validationResult = super.evaluateDocument(doc, ctx);
+        if (this.analysisOptions.analysisMode !== AnalysisMode.NO_STATISTIC && validationResult.data.failures === 0) {
+            // Add syntax usage statistics only if build was successful
+            const statistics = this.collectSyntaxUsageStatistics(doc, this.services.Grammar);
+            validationResult.metadata[LangiumDocumentAnalyzer.METADATA_KEY] = statistics;
+        }
+        return validationResult;
+    }
+    collectSyntaxUsageStatistics(doc, grammar) {
+        const rootCstNode = doc.parseResult.value.$cstNode;
+        if (!rootCstNode) {
+            return this.createEmptySyntaxStatistic();
+        }
+        const { includeImportedRules, excludeRules, computeDiversity, includeHiddenRules } = this.analysisOptions;
+        const excludedRules = new Set(excludeRules);
+        const isRuleExcluded = (ruleName) => ruleName === 'WS' || excludedRules.has(ruleName);
+        const allRules = includeImportedRules ? this.collectAllRules(grammar) : grammar.rules;
+        const ruleUsage = {};
+        // Initialize rule usage map, excluding rules specified in excludeRules. Also skip entry rule.
+        for (const rule of allRules) {
+            if (!isRuleExcluded(rule.name)) {
+                if ((GrammarAST.isParserRule(rule) && rule.entry)
+                    || (GrammarAST.isTerminalRule(rule) && rule.hidden && !includeHiddenRules)) {
+                    continue;
+                }
+                ruleUsage[rule.name] = 0;
+            }
+        }
+        for (const cstNode of CstUtils.streamCst(rootCstNode)) {
+            const grammarSource = cstNode.grammarSource;
+            const addIfNotExcluded = (ruleName) => {
+                if (!isRuleExcluded(ruleName)) {
+                    ruleUsage[ruleName] = (ruleUsage[ruleName] ?? 0) + 1;
+                }
+            };
+            if (grammarSource && GrammarAST.isRuleCall(grammarSource)) {
+                // For now handle only RuleCalls
+                addIfNotExcluded(grammarSource.rule.ref?.name ?? 'unknown');
+            }
+            else if (includeHiddenRules && cstNode.hidden && isLeafCstNode(cstNode)) {
+                addIfNotExcluded(cstNode.tokenType.name);
+            }
+        }
+        let diversity = { entropy: 0, giniCoefficient: 0, simpsonIndex: 0 };
+        if (computeDiversity) {
+            diversity = {
+                entropy: this.computeEntropy(ruleUsage),
+                giniCoefficient: this.computeGiniCoefficient(ruleUsage),
+                simpsonIndex: this.computeSimpsonIndex(ruleUsage)
+            };
+        }
+        const coverage = this.computeCoverage(ruleUsage);
+        return { ruleUsage, coverage, diversity };
+    }
+    /**
+     * Computes coverage as percentage of used rules over all available rules
+     */
+    computeCoverage(ruleUsage) {
+        const usedRules = Object.values(ruleUsage).filter(count => count > 0).length;
+        return usedRules > 0 ? (usedRules / Object.keys(ruleUsage).length) * 100 : 0;
+    }
+    /**
+     * Computes Shannon entropy - measure of information diversity
+     * Higher values indicate more diverse usage patterns
+     */
+    computeEntropy(ruleUsage) {
+        const totalUsage = Object.values(ruleUsage).reduce((sum, count) => sum + count, 0);
+        if (totalUsage === 0)
+            return 0;
+        let entropy = 0;
+        for (const count of Object.values(ruleUsage)) {
+            if (count > 0) {
+                const probability = count / totalUsage;
+                entropy -= probability * Math.log2(probability);
+            }
+        }
+        return entropy;
+    }
+    /**
+     * Computes Gini coefficient - measure of inequality in rule usage
+     * 0 = perfect equality, 1 = maximum inequality
+     */
+    computeGiniCoefficient(ruleUsage) {
+        const counts = Object.values(ruleUsage).sort((a, b) => a - b);
+        const n = counts.length;
+        if (n === 0)
+            return 0;
+        const sum = counts.reduce((acc, val) => acc + val, 0);
+        if (sum === 0)
+            return 0;
+        let numerator = 0;
+        for (let i = 0; i < n; i++) {
+            numerator += (2 * (i + 1) - n - 1) * counts[i];
+        }
+        return numerator / (n * sum);
+    }
+    /**
+     * Computes Simpson's diversity index - probability that two randomly selected items are different
+     * Higher values indicate more diversity
+     */
+    computeSimpsonIndex(ruleUsage) {
+        const totalUsage = Object.values(ruleUsage).reduce((sum, count) => sum + count, 0);
+        if (totalUsage === 0)
+            return 0;
+        let sum = 0;
+        for (const count of Object.values(ruleUsage)) {
+            const probability = count / totalUsage;
+            sum += probability * probability;
+        }
+        return 1 - sum; // Simpson's diversity index (1-D)
+    }
+    /**
+     * Extracts syntax statistics from the evaluation result.
+     * @param result The evaluation result.
+     * @returns The extracted syntax statistics or undefined if not found.
+     */
+    extractStatisticsFromResult(result) {
+        const metadata = result?.metadata;
+        if (metadata && metadata[LangiumDocumentAnalyzer.METADATA_KEY]) {
+            return metadata[LangiumDocumentAnalyzer.METADATA_KEY];
+        }
+        return undefined;
+    }
+    collectAllRules(grammar) {
+        try {
+            return grammar.rules.concat(resolveTransitiveImports(this.services.shared.workspace.LangiumDocuments, grammar).map(g => g.rules).flat());
+        }
+        catch (e) {
+            console.error('Error resolving imports: ', e);
+            return [];
+        }
+    }
+    createEmptySyntaxStatistic() {
+        return {
+            ruleUsage: {},
+            coverage: 0,
+            diversity: {
+                entropy: 0,
+                giniCoefficient: 0,
+                simpsonIndex: 0
+            }
+        };
+    }
+}
+LangiumDocumentAnalyzer.METADATA_KEY = 'syntax_statistics';
+/**
+ * Analysis mode for controlling what analysis operations to perform
+ */
+export var AnalysisMode;
+(function (AnalysisMode) {
+    AnalysisMode["ALL"] = "ALL";
+    AnalysisMode["NO_STATISTIC"] = "NO_STATISTIC";
+})(AnalysisMode || (AnalysisMode = {}));
+const DEFAULT_OPTIONS = {
+    analysisMode: AnalysisMode.ALL,
+    excludeRules: [],
+    includeImportedRules: true,
+    includeHiddenRules: true,
+    computeDiversity: true
+};
+//# sourceMappingURL=document-analyzer.js.map

package/dist/analyzer/document-analyzer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"document-analyzer.js","sourceRoot":"","sources":["../../src/analyzer/document-analyzer.ts"],"names":[],"mappings":"AAAA;;;;;;gFAMgF;AAEhF,OAAO,EAAE,QAAQ,EAAW,UAAU,EAAmB,aAAa,EAAE,MAAM,SAAS,CAAC;AACxF,OAAO,EAAE,wBAAwB,GAAG,MAAM,iBAAiB,CAAC;AAI5D,OAAO,EAAE,gBAAgB,EAA8B,MAAM,mCAAmC,CAAC;AAEjG;;GAEG;AACH,MAAM,OAAO,uBAAmD,SAAQ,gBAAmB;IAMvF;;;;;;;;;;;;OAYG;IACH,YAAY,QAAW,EAAE,kBAA4C,EAAE;QACnE,KAAK,CAAC,QAAQ,CAAC,CAAC;QAChB,IAAI,CAAC,eAAe,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,eAAe,EAAE,CAAC;IACtE,CAAC;IAED,gBAAgB,CAAC,GAAoB,EAAE,GAAsB;QACzD,MAAM,gBAAgB,GAAG,KAAK,CAAC,gBAAgB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAC1D,IAAI,IAAI,CAAC,eAAe,CAAC,YAAY,KAAK,YAAY,CAAC,YAAY,IAAI,gBAAgB,CAAC,IAAI,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;YAC1G,2DAA2D;YAC3D,MAAM,UAAU,GAAG,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACjF,gBAAgB,CAAC,QAAQ,CAAC,uBAAuB,CAAC,YAAY,CAAC,GAAG,UAAU,CAAC;QACjF,CAAC;QACD,OAAO,gBAAgB,CAAC;IAC5B,CAAC;IAED,4BAA4B,CAAC,GAAoB,EAAE,OAAgB;QAC/D,MAAM,WAAW,GAAG,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,QAAQ,CAAC;QACnD,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,OAAO,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAC7C,CAAC;QACD,MAAM,EAAE,oBAAoB,EAAE,YAAY,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC;QAC1G,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC;QAC5C,MAAM,cAAc,GAAG,CAAC,QAAgB,EAAE,EAAE,CAAC,QAAQ,KAAK,IAAI,IAAI,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE9F,MAAM,QAAQ,GAAG,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;QACtF,MAAM,SAAS,GAA2B,EAAE,CAAC;QAC7C,8FAA8F;QAC9F,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC1B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,IACI,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC;uBAC1C,CAAC,UAAU,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,EAC5E,CAAC;oBACC,SAAS;gBACb,CAAC;gBACD,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC7B,CAAC;QACL,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC;YACpD,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;YAE5C,MAAM,gBAAgB,GAAG,CAAC,QAAgB,EAAE,EAAE;gBAC1C,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC5B,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;gBACzD,CAAC;YACL,CAAC,CAAC;YAEF,IAAI,aAAa,IAAI,UAAU,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;gBACxD,gCAAgC;gBAChC,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,SAAS,CAAC,CAAC;YAChE,CAAC;iBAAM,IAAI,kBAAkB,IAAI,OAAO,CAAC,MAAM,IAAI,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;gBACxE,gBAAgB,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAC7C,CAAC;QACL,CAAC;QAED,IAAI,SAAS,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC;QACpE,IAAI,gBAAgB,EAAE,CAAC;YACnB,SAAS,GAAG;gBACR,OAAO,EAAE,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC;gBACvC,eAAe,EAAE,IAAI,CAAC,sBAAsB,CAAC,SAAS,CAAC;gBACvD,YAAY,EAAE,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAC;aACpD,CAAC;QACN,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;QACjD,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,SAAiC;QAC7C,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;QAC7E,OAAO,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACjF,CAAC;IAED;;;OAGG;IACH,cAAc,CAAC,SAAiC;QAC5C,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;QACnF,IAAI,UAAU,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAE/B,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;YAC3C,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACZ,MAAM,WAAW,GAAG,KAAK,GAAG,UAAU,CAAC;gBACvC,OAAO,IAAI,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACpD,CAAC;QACL,CAAC;QACD,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;OAGG;IACH,sBAAsB,CAAC,SAAiC;QACpD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC9D,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QACxB,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEtB,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC;QACtD,IAAI,GAAG,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAExB,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACzB,SAAS,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACnD,CAAC;QACD,OAAO,SAAS,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,mBAAmB,CAAC,SAAiC;QACjD,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;QACnF,IAAI,UAAU,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAE/B,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;YAC3C,MAAM,WAAW,GAAG,KAAK,GAAG,UAAU,CAAC;YACvC,GAAG,IAAI,WAAW,GAAG,WAAW,CAAC;QACrC,CAAC;QAED,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,kCAAkC;IACtD,CAAC;IAED;;;;OAIG;IACH,2BAA2B,CAAC,MAA4C;QACpE,MAAM,QAAQ,GAAG,MAAM,EAAE,QAAQ,CAAC;QAClC,IAAI,QAAQ,IAAI,QAAQ,CAAC,uBAAuB,CAAC,YAAY,CAAC,EAAE,CAAC;YAC7D,OAAO,QAAQ,CAAC,uBAAuB,CAAC,YAAY,CAAoB,CAAC;QAC7E,CAAC;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAES,eAAe,CAAC,OAAgB;QACtC,IAAI,CAAC;YACD,OAAO,OAAO,CAAC,KAAK,CAAC,MAAM,CACvB,wBAAwB,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAC9G,CAAC;QACN,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,CAAC,CAAC,CAAC;YAC9C,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAES,0BAA0B;QAChC,OAAO;YACH,SAAS,EAAE,EAAE;YACb,QAAQ,EAAE,CAAC;YACX,SAAS,EAAE;gBACP,OAAO,EAAE,CAAC;gBACV,eAAe,EAAE,CAAC;gBAClB,YAAY,EAAE,CAAC;aAClB;SACJ,CAAC;IACN,CAAC;;AArLsB,oCAAY,GAAG,mBAAmB,CAAC;AAwL9D;;GAEG;AACH,MAAM,CAAN,IAAY,YAGX;AAHD,WAAY,YAAY;IACpB,2BAAW,CAAA;IACX,6CAA6B,CAAA;AACjC,CAAC,EAHW,YAAY,KAAZ,YAAY,QAGvB;AAwBD,MAAM,eAAe,GAAoB;IACrC,YAAY,EAAE,YAAY,CAAC,GAAG;IAC9B,YAAY,EAAE,EAAE;IAChB,oBAAoB,EAAE,IAAI;IAC1B,kBAAkB,EAAE,IAAI;IACxB,gBAAgB,EAAE,IAAI;CACzB,CAAC"}

package/dist/analyzer/index.d.ts

@@ -0,0 +1,7 @@
+/******************************************************************************
+ * Copyright 2025 TypeFox GmbH
+ * This program and the accompanying materials are made available under the
+ * terms of the MIT License, which is available in the project root.
+ ******************************************************************************/
+export * from './document-analyzer.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/analyzer/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/analyzer/index.ts"],"names":[],"mappings":"AAAA;;;;gFAIgF;AAEhF,cAAc,wBAAwB,CAAC"}

package/dist/analyzer/index.js

@@ -0,0 +1,7 @@
+/******************************************************************************
+ * Copyright 2025 TypeFox GmbH
+ * This program and the accompanying materials are made available under the
+ * terms of the MIT License, which is available in the project root.
+ ******************************************************************************/
+export * from './document-analyzer.js';
+//# sourceMappingURL=index.js.map

package/dist/analyzer/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/analyzer/index.ts"],"names":[],"mappings":"AAAA;;;;gFAIgF;AAEhF,cAAc,wBAAwB,CAAC"}

package/dist/evaluator/document-evaluator.d.ts

@@ -0,0 +1,34 @@
+/******************************************************************************
+ * Copyright 2024 - 2025 TypeFox GmbH
+ * This program and the accompanying materials are made available under the
+ * terms of the MIT License, which is available in the project root.
+ ******************************************************************************/
+/**
+ * Langium Document Evaluator (evaluates on a Langium document)
+ */
+import { LangiumDocument } from "langium";
+import { LangiumServices } from "langium/lsp";
+import { Evaluator, EvaluatorResult, EvaluatorResultData } from "./evaluator.js";
+export declare abstract class AbstractDocumentEvaluator<T extends LangiumServices, RD extends FailureAwarenessData = FailureAwarenessData> extends Evaluator {
+    /**
+     * Services to use for evaluation
+     */
+    protected services: T;
+    constructor(services: T);
+    /**
+     * Validate an agent response as if it's a langium program. If we can parse it, we attempt to validate it.
+     */
+    evaluate(input: string, fileExtension?: string | undefined): Promise<Partial<EvaluatorResult<RD>>>;
+    abstract evaluateDocument(doc: LangiumDocument, ctx: EvaluationContext): Partial<EvaluatorResult<RD>>;
+    protected handleBuildError(e: unknown, ctx: EvaluationContext): Partial<EvaluatorResult<RD>>;
+}
+export type FailureAwarenessData = EvaluatorResultData & {
+    /**
+    * Number of validation failures
+    */
+    failures: number;
+};
+export type EvaluationContext = {
+    input: string;
+};
+//# sourceMappingURL=document-evaluator.d.ts.map

package/dist/evaluator/document-evaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"document-evaluator.d.ts","sourceRoot":"","sources":["../../src/evaluator/document-evaluator.ts"],"names":[],"mappings":"AAAA;;;;gFAIgF;AAEhF;;GAEG;AAEH,OAAO,EAAE,eAAe,EAAO,MAAM,SAAS,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAGjF,8BAAsB,yBAAyB,CAAC,CAAC,SAAS,eAAe,EAAE,EAAE,SAAS,oBAAoB,GAAG,oBAAoB,CAAE,SAAQ,SAAS;IAEhJ;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAEV,QAAQ,EAAE,CAAC;IAKvB;;OAEG;IACG,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,GAAE,MAAM,GAAG,SAAqB,GAAG,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,CAAC;IAoBnH,QAAQ,CAAC,gBAAgB,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;IAGrG,SAAS,CAAC,gBAAgB,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;CAS/F;AAED,MAAM,MAAM,oBAAoB,GAAG,mBAAmB,GAAG;IACrD;;MAEE;IACF,QAAQ,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC5B,KAAK,EAAE,MAAM,CAAC;CACjB,CAAC"}

package/dist/evaluator/document-evaluator.js

@@ -0,0 +1,48 @@
+/******************************************************************************
+ * Copyright 2024 - 2025 TypeFox GmbH
+ * This program and the accompanying materials are made available under the
+ * terms of the MIT License, which is available in the project root.
+ ******************************************************************************/
+/**
+ * Langium Document Evaluator (evaluates on a Langium document)
+ */
+import { URI } from "langium";
+import { Evaluator } from "./evaluator.js";
+export class AbstractDocumentEvaluator extends Evaluator {
+    constructor(services) {
+        super();
+        this.services = services;
+    }
+    /**
+     * Validate an agent response as if it's a langium program. If we can parse it, we attempt to validate it.
+     */
+    async evaluate(input, fileExtension = undefined) {
+        if (input.includes('```')) {
+            // take the first code block instead, if present (assuming it's a langium grammar)
+            const codeBlock = input.split(/```[a-z-]*/)[1];
+            input = codeBlock;
+        }
+        const fileExt = fileExtension ? fileExtension : this.services.LanguageMetaData.fileExtensions[0];
+        const doc = this.services.shared.workspace.LangiumDocumentFactory.fromString(input, URI.parse(`memory:/test.${fileExt}`));
+        const context = {
+            input: input
+        };
+        try {
+            await this.services.shared.workspace.DocumentBuilder.build([doc], { validation: true });
+            return this.evaluateDocument(doc, context);
+        }
+        catch (e) {
+            return this.handleBuildError(e, context);
+        }
+    }
+    handleBuildError(e, ctx) {
+        console.error('Error during evaluation: ', e);
+        return {
+            name: this.constructor.name,
+            data: {
+                failures: 1
+            }
+        };
+    }
+}
+//# sourceMappingURL=document-evaluator.js.map

package/dist/evaluator/document-evaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"document-evaluator.js","sourceRoot":"","sources":["../../src/evaluator/document-evaluator.ts"],"names":[],"mappings":"AAAA;;;;gFAIgF;AAEhF;;GAEG;AAEH,OAAO,EAAmB,GAAG,EAAE,MAAM,SAAS,CAAC;AAE/C,OAAO,EAAE,SAAS,EAAwC,MAAM,gBAAgB,CAAC;AAGjF,MAAM,OAAgB,yBAA6G,SAAQ,SAAS;IAOhJ,YAAY,QAAW;QACnB,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,gBAAoC,SAAS;QAEvE,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACxB,kFAAkF;YAClF,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/C,KAAK,GAAG,SAAS,CAAC;QACtB,CAAC;QACD,MAAM,OAAO,GAAG,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;QACjG,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,sBAAsB,CAAC,UAAU,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,gBAAgB,OAAO,EAAE,CAAC,CAAC,CAAC;QAC1H,MAAM,OAAO,GAAsB;YAC/B,KAAK,EAAE,KAAK;SACf,CAAC;QACF,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;YACxF,OAAO,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,OAAO,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;QAC5C,CAAC;IACL,CAAC;IAKS,gBAAgB,CAAC,CAAU,EAAE,GAAsB;QACzD,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,CAAC,CAAC,CAAC;QAC9C,OAAO;YACH,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;YAC3B,IAAI,EAAwB;gBACxB,QAAQ,EAAE,CAAC;aACG;SACrB,CAAC;IACN,CAAC;CACJ"}

package/dist/evaluator/evaluator.d.ts

@@ -9,7 +9,7 @@ export type EvaluatorResultData = Record<string, unknown> & {
 /**
  * Evaluator result type
  */
-export type EvaluatorResult = {
+export type EvaluatorResult<T = EvaluatorResultData> = {
     /**
      * Name of this evaluation
      */
@@ -21,7 +21,7 @@ export type EvaluatorResult = {
     /**
      * Data for this evaluation
      */
-    data: EvaluatorResultData;
+    data: T;
 };
 /**
  * Helper to process a set of results, averaging all runs of each runner-evaluator-case combination

package/dist/evaluator/evaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../../src/evaluator/evaluator.ts"],"names":[],"mappings":"AAAA;;;;gFAIgF;AAShF,MAAM,MAAM,mBAAmB,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG;IAC1B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAE9B;;OAEG;IACH,IAAI,EAAE,mBAAmB,CAAC;CAE7B,CAAC;AAEF;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,eAAe,EAAE,CA4ChF;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,eAAe,EAAE,CAiDlF;AAED;;GAEG;AACH,MAAM,WAAW,MAAM;IACnB,MAAM,EAAE;QACJ,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,eAAe,EAAE,CAAC;CAC9B;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAE/C;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,eAAe,EAAE,CAgC7E;AAED;;GAEG;AACH,8BAAsB,SAAS;IAC3B;;OAEG;IACH,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;CAEpG;AAED,wBAAgB,eAAe,CAAC,GAAG,UAAU,EAAE,SAAS,EAAE,GAAG,SAAS,CAGrE"}
+{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../../src/evaluator/evaluator.ts"],"names":[],"mappings":"AAAA;;;;gFAIgF;AAShF,MAAM,MAAM,mBAAmB,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,eAAe,CAAC,CAAC,GAAG,mBAAmB,IAAI;IACnD;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAE9B;;OAEG;IACH,IAAI,EAAE,CAAC,CAAC;CAEX,CAAC;AAEF;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,eAAe,EAAE,CA4ChF;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,eAAe,EAAE,CAiDlF;AAED;;GAEG;AACH,MAAM,WAAW,MAAM;IACnB,MAAM,EAAE;QACJ,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,eAAe,EAAE,CAAC;CAC9B;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAE/C;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,eAAe,EAAE,CAgC7E;AAED;;GAEG;AACH,8BAAsB,SAAS;IAC3B;;OAEG;IACH,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;CAEpG;AAED,wBAAgB,eAAe,CAAC,GAAG,UAAU,EAAE,SAAS,EAAE,GAAG,SAAS,CAGrE"}

package/dist/evaluator/langium-evaluator.d.ts

@@ -6,17 +6,15 @@
 /**
  * Base Langium DSL validator (taps into Langium's validator messages to provide better results)
  */
+import { LangiumDocument } from "langium";
 import { LangiumServices } from "langium/lsp";
 import { Diagnostic } from "vscode-languageserver-types";
-import { Evaluator, EvaluatorResult, EvaluatorResultData } from "./evaluator.js";
+import { AbstractDocumentEvaluator, EvaluationContext, FailureAwarenessData } from "./document-evaluator.js";
+import { EvaluatorResult } from "./evaluator.js";
 /**
  * Langium-specific evaluator result data
  */
-export interface LangiumEvaluatorResultData extends EvaluatorResultData {
-    /**
-     * Number of validation failures
-     */
-    failures: number;
+export interface LangiumEvaluatorResultData extends FailureAwarenessData {
     /**
      * Number of errors
      */
@@ -46,15 +44,11 @@ export interface LangiumEvaluatorResultData extends EvaluatorResultData {
      */
     diagnostics: Diagnostic[];
 }
-export declare class LangiumEvaluator<T extends LangiumServices> extends Evaluator {
-    /**
-     * Services to use for evaluation
-     */
-    protected services: T;
-    constructor(services: T);
+export declare class LangiumEvaluator<T extends LangiumServices> extends AbstractDocumentEvaluator<T, LangiumEvaluatorResultData> {
     /**
      * Validate an agent response as if it's a langium program. If we can parse it, we attempt to validate it.
      */
-    evaluate(response: string): Promise<Partial<EvaluatorResult>>;
+    evaluateDocument(doc: LangiumDocument, ctx: EvaluationContext): EvaluatorResult<LangiumEvaluatorResultData>;
+    protected createEmptyResultData(): LangiumEvaluatorResultData;
 }
 //# sourceMappingURL=langium-evaluator.d.ts.map

package/dist/evaluator/langium-evaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"langium-evaluator.d.ts","sourceRoot":"","sources":["../../src/evaluator/langium-evaluator.ts"],"names":[],"mappings":"AAAA;;;;gFAIgF;AAEhF;;GAEG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AACzD,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAGjF;;GAEG;AACH,MAAM,WAAW,0BAA2B,SAAQ,mBAAmB;IAEnE;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,WAAW,EAAE,UAAU,EAAE,CAAC;CAC7B;AAED,qBAAa,gBAAgB,CAAC,CAAC,SAAS,eAAe,CAAE,SAAQ,SAAS;IAEtE;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAEV,QAAQ,EAAE,CAAC;IAKvB;;OAEG;IACG,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;CAqEtE"}
+{"version":3,"file":"langium-evaluator.d.ts","sourceRoot":"","sources":["../../src/evaluator/langium-evaluator.ts"],"names":[],"mappings":"AAAA;;;;gFAIgF;AAEhF;;GAEG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AACzD,OAAO,EAAE,yBAAyB,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC7G,OAAO,EAAE,eAAe,EAAuB,MAAM,gBAAgB,CAAC;AAEtE;;GAEG;AACH,MAAM,WAAW,0BAA2B,SAAQ,oBAAoB;IAEpE;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,WAAW,EAAE,UAAU,EAAE,CAAC;CAC7B;AAED,qBAAa,gBAAgB,CAAC,CAAC,SAAS,eAAe,CAAE,SAAQ,yBAAyB,CAAC,CAAC,EAAE,0BAA0B,CAAC;IAGrH;;OAEG;IACH,gBAAgB,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,EAAE,iBAAiB,GAAG,eAAe,CAAC,0BAA0B,CAAC;IAwC3G,SAAS,CAAC,qBAAqB,IAAI,0BAA0B;CAYhE"}

package/dist/evaluator/langium-evaluator.js

@@ -3,78 +3,56 @@
  * This program and the accompanying materials are made available under the
  * terms of the MIT License, which is available in the project root.
  ******************************************************************************/
-import { Evaluator } from "./evaluator.js";
-import { URI } from "langium";
-export class LangiumEvaluator extends Evaluator {
-    constructor(services) {
-        super();
-        this.services = services;
-    }
+import { AbstractDocumentEvaluator } from "./document-evaluator.js";
+export class LangiumEvaluator extends AbstractDocumentEvaluator {
     /**
      * Validate an agent response as if it's a langium program. If we can parse it, we attempt to validate it.
      */
-    async evaluate(response) {
-        if (response.includes('```')) {
-            // take the first code block instead, if present (assuming it's a langium grammar)
-            const codeBlock = response.split(/```[a-z-]*/)[1];
-            response = codeBlock;
-        }
-        const doc = this.services.shared.workspace.LangiumDocumentFactory.fromString(response, URI.parse('memory://test.langium'));
-        try {
-            await this.services.shared.workspace.DocumentBuilder.build([doc], { validation: true });
-            const validationResults = doc.diagnostics ?? [];
-            // count the number of each type of diagnostic
-            let evalData = {
-                failures: 0,
-                errors: 0,
-                warnings: 0,
-                infos: 0,
-                hints: 0,
-                unassigned: 0,
-                // include length of the response for checking
-                response_length: response.length,
-                // include the diagnostics for debugging if desired
-                diagnostics: validationResults
-            };
-            for (const diagnostic of validationResults) {
-                if (diagnostic.severity) {
-                    switch (diagnostic.severity) {
-                        case 1:
-                            evalData.errors++;
-                            break;
-                        case 2:
-                            evalData.warnings++;
-                            break;
-                        case 3:
-                            evalData.infos++;
-                            break;
-                        case 4:
-                            evalData.hints++;
-                            break;
-                        default:
-                            evalData.unassigned++;
-                            break;
-                    }
+    evaluateDocument(doc, ctx) {
+        const validationResults = doc.diagnostics ?? [];
+        const evalData = this.createEmptyResultData();
+        // include length of the response for checking
+        evalData.response_length = ctx.input.length;
+        // include the diagnostics for debugging if desired
+        evalData.diagnostics = validationResults;
+        for (const diagnostic of validationResults) {
+            if (diagnostic.severity) {
+                switch (diagnostic.severity) {
+                    case 1:
+                        evalData.errors++;
+                        break;
+                    case 2:
+                        evalData.warnings++;
+                        break;
+                    case 3:
+                        evalData.infos++;
+                        break;
+                    case 4:
+                        evalData.hints++;
+                        break;
+                    default:
+                        evalData.unassigned++;
+                        break;
                 }
             }
-            return {
-                data: evalData
-            };
-        }
-        catch (e) {
-            console.error('Error during evaluation: ', e);
-            return {
-                data: {
-                    failures: 1,
-                    errors: 0,
-                    warnings: 0,
-                    infos: 0,
-                    hints: 0,
-                    unassigned: 0,
-                    response_length: response.length
-                }
-            };
         }
+        return {
+            name: this.constructor.name,
+            metadata: {},
+            data: evalData
+        };
+    }
+    createEmptyResultData() {
+        return {
+            failures: 0,
+            errors: 0,
+            warnings: 0,
+            infos: 0,
+            hints: 0,
+            unassigned: 0,
+            response_length: 0,
+            diagnostics: []
+        };
     }
 }
 //# sourceMappingURL=langium-evaluator.js.map

package/dist/evaluator/langium-evaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"langium-evaluator.js","sourceRoot":"","sources":["../../src/evaluator/langium-evaluator.ts"],"names":[],"mappings":"AAAA;;;;gFAIgF;AAQhF,OAAO,EAAE,SAAS,EAAwC,MAAM,gBAAgB,CAAC;AACjF,OAAO,EAAE,GAAG,EAAE,MAAM,SAAS,CAAC;AAgD9B,MAAM,OAAO,gBAA4C,SAAQ,SAAS;IAOtE,YAAY,QAAW;QACnB,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,QAAgB;QAE3B,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3B,kFAAkF;YAClF,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;YAClD,QAAQ,GAAG,SAAS,CAAC;QACzB,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,sBAAsB,CAAC,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC,CAAC;QAE3H,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;YACxF,MAAM,iBAAiB,GAAG,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC;YAEhD,8CAA8C;YAC9C,IAAI,QAAQ,GAA+B;gBACvC,QAAQ,EAAE,CAAC;gBACX,MAAM,EAAE,CAAC;gBACT,QAAQ,EAAE,CAAC;gBACX,KAAK,EAAE,CAAC;gBACR,KAAK,EAAE,CAAC;gBACR,UAAU,EAAE,CAAC;gBACb,8CAA8C;gBAC9C,eAAe,EAAE,QAAQ,CAAC,MAAM;gBAChC,mDAAmD;gBACnD,WAAW,EAAE,iBAAiB;aACjC,CAAC;YAEF,KAAK,MAAM,UAAU,IAAI,iBAAiB,EAAE,CAAC;gBACzC,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;oBACtB,QAAQ,UAAU,CAAC,QAAQ,EAAE,CAAC;wBAC1B,KAAK,CAAC;4BACF,QAAQ,CAAC,MAAM,EAAE,CAAC;4BAClB,MAAM;wBACV,KAAK,CAAC;4BACF,QAAQ,CAAC,QAAQ,EAAE,CAAC;4BACpB,MAAM;wBACV,KAAK,CAAC;4BACF,QAAQ,CAAC,KAAK,EAAE,CAAC;4BACjB,MAAM;wBACV,KAAK,CAAC;4BACF,QAAQ,CAAC,KAAK,EAAE,CAAC;4BACjB,MAAM;wBACV;4BACI,QAAQ,CAAC,UAAU,EAAE,CAAC;4BACtB,MAAM;oBACd,CAAC;gBACL,CAAC;YACL,CAAC;YAED,OAAO;gBACH,IAAI,EAAE,QAAQ;aACjB,CAAC;QAEN,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,CAAC,CAAC,CAAC;YAC9C,OAAO;gBACH,IAAI,EAAE;oBACF,QAAQ,EAAE,CAAC;oBACX,MAAM,EAAE,CAAC;oBACT,QAAQ,EAAE,CAAC;oBACX,KAAK,EAAE,CAAC;oBACR,KAAK,EAAE,CAAC;oBACR,UAAU,EAAE,CAAC;oBACb,eAAe,EAAE,QAAQ,CAAC,MAAM;iBACL;aAClC,CAAC;QACN,CAAC;IACL,CAAC;CACJ"}
+{"version":3,"file":"langium-evaluator.js","sourceRoot":"","sources":["../../src/evaluator/langium-evaluator.ts"],"names":[],"mappings":"AAAA;;;;gFAIgF;AAShF,OAAO,EAAE,yBAAyB,EAA2C,MAAM,yBAAyB,CAAC;AA4C7G,MAAM,OAAO,gBAA4C,SAAQ,yBAAwD;IAGrH;;OAEG;IACH,gBAAgB,CAAC,GAAoB,EAAE,GAAsB;QAEzD,MAAM,iBAAiB,GAAG,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC;QAEhD,MAAM,QAAQ,GAA+B,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC1E,8CAA8C;QAC9C,QAAQ,CAAC,eAAe,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC;QAC5C,mDAAmD;QACnD,QAAQ,CAAC,WAAW,GAAG,iBAAiB,CAAC;QAGzC,KAAK,MAAM,UAAU,IAAI,iBAAiB,EAAE,CAAC;YACzC,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;gBACtB,QAAQ,UAAU,CAAC,QAAQ,EAAE,CAAC;oBAC1B,KAAK,CAAC;wBACF,QAAQ,CAAC,MAAM,EAAE,CAAC;wBAClB,MAAM;oBACV,KAAK,CAAC;wBACF,QAAQ,CAAC,QAAQ,EAAE,CAAC;wBACpB,MAAM;oBACV,KAAK,CAAC;wBACF,QAAQ,CAAC,KAAK,EAAE,CAAC;wBACjB,MAAM;oBACV,KAAK,CAAC;wBACF,QAAQ,CAAC,KAAK,EAAE,CAAC;wBACjB,MAAM;oBACV;wBACI,QAAQ,CAAC,UAAU,EAAE,CAAC;wBACtB,MAAM;gBACd,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO;YACH,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;YAC3B,QAAQ,EAAE,EAAE;YACZ,IAAI,EAAE,QAAQ;SACjB,CAAC;IACN,CAAC;IAES,qBAAqB;QAC3B,OAAO;YACH,QAAQ,EAAE,CAAC;YACX,MAAM,EAAE,CAAC;YACT,QAAQ,EAAE,CAAC;YACX,KAAK,EAAE,CAAC;YACR,KAAK,EAAE,CAAC;YACR,UAAU,EAAE,CAAC;YACb,eAAe,EAAE,CAAC;YAClB,WAAW,EAAE,EAAE;SAClB,CAAC;IACN,CAAC;CACJ"}

package/dist/index.d.ts

@@ -4,5 +4,6 @@
  * terms of the MIT License, which is available in the project root.
  ******************************************************************************/
 export * from './evaluator/index.js';
+export * from './analyzer/index.js';
 export * from './splitter/index.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;gFAIgF;AAEhF,cAAc,sBAAsB,CAAC;AACrC,cAAc,qBAAqB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;gFAIgF;AAEhF,cAAc,sBAAsB,CAAC;AACrC,cAAc,qBAAqB,CAAC;AACpC,cAAc,qBAAqB,CAAC"}

package/dist/index.js

@@ -4,5 +4,6 @@
  * terms of the MIT License, which is available in the project root.
  ******************************************************************************/
 export * from './evaluator/index.js';
+export * from './analyzer/index.js';
 export * from './splitter/index.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;gFAIgF;AAEhF,cAAc,sBAAsB,CAAC;AACrC,cAAc,qBAAqB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;gFAIgF;AAEhF,cAAc,sBAAsB,CAAC;AACrC,cAAc,qBAAqB,CAAC;AACpC,cAAc,qBAAqB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "langium-ai-tools",
-    "version": "4.1.0",
+    "version": "4.1.2",
     "description": "Tooling for building AI Applications that leverage Langium DSLs",
     "repository": {
         "type": "git",
@@ -54,8 +54,5 @@
         "npm": "10.2.3"
     },
     "devDependencies": {
-        "rimraf": "^6.0.1",
-        "typescript": "^5.4.5",
-        "vitest": "^3.0.9"
     }
 }