@helmiq/crew 0.1.0

package/dist/evaluation/evaluation.test.js

@@ -0,0 +1,490 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { parseEvaluationResponse, computeAggregate, determinePassFail, formatFailureFeedback, } from './scorer.js';
+import { evaluate } from './evaluator.js';
+const mockModel = {
+    alias: 'claude-sonnet',
+    provider: 'anthropic',
+    concreteModel: 'claude-sonnet-4-20250514',
+    apiKeyEnv: 'ANTHROPIC_API_KEY',
+};
+const testRubric = {
+    rubric: {
+        artifact_type: 'pull-request',
+        scoring_scale: 10,
+        pass_threshold: 7,
+        criteria: [
+            {
+                name: 'Tests cover acceptance criteria',
+                weight: 'blocking',
+                description: 'Must have tests',
+            },
+            { name: 'Code follows patterns', weight: 'blocking' },
+            { name: 'PR description links to story', weight: 'important' },
+            { name: 'Readable code', weight: 'advisory' },
+        ],
+    },
+};
+function makeScoringResponse(scores) {
+    return JSON.stringify(scores);
+}
+const passingScores = makeScoringResponse([
+    { name: 'Tests cover acceptance criteria', score: 8, reasoning: 'Good coverage' },
+    { name: 'Code follows patterns', score: 9, reasoning: 'Consistent' },
+    { name: 'PR description links to story', score: 7, reasoning: 'Links present' },
+    { name: 'Readable code', score: 8, reasoning: 'Clear' },
+]);
+const failingScores = makeScoringResponse([
+    { name: 'Tests cover acceptance criteria', score: 5, reasoning: 'Missing tests for edge cases' },
+    { name: 'Code follows patterns', score: 8, reasoning: 'OK' },
+    { name: 'PR description links to story', score: 6, reasoning: 'Missing design link' },
+    { name: 'Readable code', score: 7, reasoning: 'Acceptable' },
+]);
+describe('T-01-005a: rubric loading and scoring', () => {
+    it('parses a well-formed JSON evaluation response', () => {
+        const scores = parseEvaluationResponse(passingScores, testRubric);
+        expect(scores).toHaveLength(4);
+        expect(scores[0].name).toBe('Tests cover acceptance criteria');
+        expect(scores[0].weight).toBe('blocking');
+        expect(scores[0].score).toBe(8);
+        expect(scores[0].reasoning).toBe('Good coverage');
+    });
+    it('extracts JSON array from surrounding text', () => {
+        const text = `Here is my evaluation:\n${passingScores}\nThat is my assessment.`;
+        const scores = parseEvaluationResponse(text, testRubric);
+        expect(scores).toHaveLength(4);
+        expect(scores[0].score).toBe(8);
+    });
+    it('returns zero scores for unparseable responses', () => {
+        const scores = parseEvaluationResponse('This is not JSON at all.', testRubric);
+        expect(scores).toHaveLength(4);
+        expect(scores.every((s) => s.score === 0)).toBe(true);
+    });
+    it('matches criteria case-insensitively', () => {
+        const response = makeScoringResponse([
+            { name: 'tests cover acceptance criteria', score: 9, reasoning: 'Great' },
+            { name: 'CODE FOLLOWS PATTERNS', score: 8, reasoning: 'Good' },
+            { name: 'pr description links to story', score: 7, reasoning: 'OK' },
+            { name: 'readable code', score: 6, reasoning: 'Fine' },
+        ]);
+        const scores = parseEvaluationResponse(response, testRubric);
+        expect(scores[0].score).toBe(9);
+        expect(scores[1].score).toBe(8);
+    });
+    it('computes weighted aggregate correctly', () => {
+        const scores = [
+            { name: 'A', weight: 'blocking', score: 8, reasoning: '' },
+            { name: 'B', weight: 'blocking', score: 6, reasoning: '' },
+            { name: 'C', weight: 'important', score: 7, reasoning: '' },
+            { name: 'D', weight: 'advisory', score: 9, reasoning: '' },
+        ];
+        const agg = computeAggregate(scores);
+        // (8*3 + 6*3 + 7*2 + 9*1) / (3+3+2+1) = (24+18+14+9)/9 = 65/9 = 7.22
+        expect(agg).toBeCloseTo(7.22, 1);
+    });
+    it('returns 0 aggregate for empty scores', () => {
+        expect(computeAggregate([])).toBe(0);
+    });
+});
+describe('T-01-005b: pass/fail determination', () => {
+    it('passes when all blocking criteria meet threshold', () => {
+        const scores = [
+            { name: 'A', weight: 'blocking', score: 8, reasoning: '' },
+            { name: 'B', weight: 'blocking', score: 7, reasoning: '' },
+            { name: 'C', weight: 'important', score: 5, reasoning: '' },
+        ];
+        expect(determinePassFail(scores, testRubric)).toBe(true);
+    });
+    it('fails when any blocking criterion is below threshold', () => {
+        const scores = [
+            { name: 'A', weight: 'blocking', score: 6, reasoning: '' },
+            { name: 'B', weight: 'blocking', score: 9, reasoning: '' },
+        ];
+        expect(determinePassFail(scores, testRubric)).toBe(false);
+    });
+    it('important and advisory below threshold do not cause failure', () => {
+        const scores = [
+            { name: 'A', weight: 'blocking', score: 7, reasoning: '' },
+            { name: 'B', weight: 'important', score: 3, reasoning: '' },
+            { name: 'C', weight: 'advisory', score: 2, reasoning: '' },
+        ];
+        expect(determinePassFail(scores, testRubric)).toBe(true);
+    });
+    it('checks aggregate threshold when configured', () => {
+        const rubricWithAgg = {
+            rubric: {
+                ...testRubric.rubric,
+                aggregate_threshold: 8,
+            },
+        };
+        const scores = [
+            { name: 'A', weight: 'blocking', score: 7, reasoning: '' },
+            { name: 'B', weight: 'blocking', score: 7, reasoning: '' },
+        ];
+        // aggregate = 7, threshold = 8 -> fail
+        expect(determinePassFail(scores, rubricWithAgg)).toBe(false);
+    });
+    it('passes when aggregate meets threshold', () => {
+        const rubricWithAgg = {
+            rubric: {
+                ...testRubric.rubric,
+                aggregate_threshold: 7,
+            },
+        };
+        const scores = [
+            { name: 'A', weight: 'blocking', score: 8, reasoning: '' },
+            { name: 'B', weight: 'blocking', score: 7, reasoning: '' },
+        ];
+        expect(determinePassFail(scores, rubricWithAgg)).toBe(true);
+    });
+});
+describe('T-01-005c: refinement iteration on failure', () => {
+    beforeEach(() => {
+        vi.restoreAllMocks();
+    });
+    it('calls refineCallback when evaluation fails and re-evaluates', async () => {
+        let callCount = 0;
+        const provider = {
+            generateText: vi.fn().mockImplementation(() => {
+                callCount++;
+                if (callCount === 1) {
+                    return Promise.resolve({
+                        text: failingScores,
+                        toolCalls: [],
+                        tokensIn: 10,
+                        tokensOut: 20,
+                    });
+                }
+                return Promise.resolve({ text: passingScores, toolCalls: [], tokensIn: 10, tokensOut: 20 });
+            }),
+        };
+        const refineCallback = vi.fn().mockResolvedValue('Refined artifact');
+        const result = await evaluate('Original artifact', testRubric, mockModel, provider, 3, refineCallback);
+        expect(refineCallback).toHaveBeenCalledOnce();
+        expect(result.evaluation.pass).toBe(true);
+        expect(result.finalArtifact).toBe('Refined artifact');
+    });
+    it('passes feedback with failing criterion details to refineCallback', async () => {
+        let callCount = 0;
+        const provider = {
+            generateText: vi.fn().mockImplementation(() => {
+                callCount++;
+                if (callCount === 1) {
+                    return Promise.resolve({
+                        text: failingScores,
+                        toolCalls: [],
+                        tokensIn: 10,
+                        tokensOut: 20,
+                    });
+                }
+                return Promise.resolve({ text: passingScores, toolCalls: [], tokensIn: 10, tokensOut: 20 });
+            }),
+        };
+        const refineCallback = vi.fn().mockResolvedValue('Fixed');
+        await evaluate('Artifact', testRubric, mockModel, provider, 2, refineCallback);
+        const feedback = refineCallback.mock.calls[0][0];
+        expect(feedback).toContain('Tests cover acceptance criteria');
+        expect(feedback).toContain('Missing tests');
+    });
+});
+describe('T-01-005d: publish with quality flag on exhaustion', () => {
+    it('returns pass=false when max iterations exhausted', async () => {
+        const provider = {
+            generateText: vi.fn().mockResolvedValue({
+                text: failingScores,
+                toolCalls: [],
+                tokensIn: 10,
+                tokensOut: 20,
+            }),
+        };
+        const refineCallback = vi.fn().mockResolvedValue('Still failing');
+        const result = await evaluate('Bad artifact', testRubric, mockModel, provider, 2, refineCallback);
+        expect(result.evaluation.pass).toBe(false);
+        expect(result.evaluation.maxIterations).toBe(2);
+        expect(refineCallback).toHaveBeenCalledOnce();
+    });
+    it('returns the last refined artifact even on exhaustion', async () => {
+        const provider = {
+            generateText: vi.fn().mockResolvedValue({
+                text: failingScores,
+                toolCalls: [],
+                tokensIn: 10,
+                tokensOut: 20,
+            }),
+        };
+        const refineCallback = vi.fn().mockResolvedValue('Last attempt artifact');
+        const result = await evaluate('Original', testRubric, mockModel, provider, 2, refineCallback);
+        expect(result.finalArtifact).toBe('Last attempt artifact');
+    });
+    it('works without a refineCallback (single evaluation)', async () => {
+        const provider = {
+            generateText: vi.fn().mockResolvedValue({
+                text: failingScores,
+                toolCalls: [],
+                tokensIn: 10,
+                tokensOut: 20,
+            }),
+        };
+        const result = await evaluate('Artifact', testRubric, mockModel, provider, 1);
+        expect(result.evaluation.pass).toBe(false);
+        expect(result.finalArtifact).toBe('Artifact');
+    });
+});
+describe('T-FR-03 (CREW-05): ADR self-evaluation', () => {
+    const adrRubric = {
+        rubric: {
+            artifact_type: 'adr-summary',
+            scoring_scale: 10,
+            pass_threshold: 7,
+            criteria: [
+                {
+                    name: 'ADRs follow project template',
+                    weight: 'blocking',
+                    description: 'Template compliance',
+                },
+                {
+                    name: 'Options analysis is balanced',
+                    weight: 'blocking',
+                    description: 'Balanced options',
+                },
+                {
+                    name: 'Rationale is justified by analysis',
+                    weight: 'blocking',
+                    description: 'Justified rationale',
+                },
+                {
+                    name: 'Consequences are documented',
+                    weight: 'important',
+                    description: 'Documented consequences',
+                },
+                {
+                    name: 'No contradictions with existing ADRs',
+                    weight: 'important',
+                    description: 'No contradictions',
+                },
+            ],
+        },
+    };
+    const adrPassingScores = makeScoringResponse([
+        { name: 'ADRs follow project template', score: 8, reasoning: 'All sections complete' },
+        {
+            name: 'Options analysis is balanced',
+            score: 7,
+            reasoning: 'Two viable options with substantive pros/cons',
+        },
+        {
+            name: 'Rationale is justified by analysis',
+            score: 8,
+            reasoning: 'Decision follows from drivers',
+        },
+        { name: 'Consequences are documented', score: 7, reasoning: 'Benefits and trade-offs listed' },
+        {
+            name: 'No contradictions with existing ADRs',
+            score: 9,
+            reasoning: 'Consistent with ADR register',
+        },
+    ]);
+    const adrFailingScores = makeScoringResponse([
+        { name: 'ADRs follow project template', score: 8, reasoning: 'Template OK' },
+        {
+            name: 'Options analysis is balanced',
+            score: 4,
+            reasoning: 'Only one viable option; second is a straw-man',
+        },
+        {
+            name: 'Rationale is justified by analysis',
+            score: 5,
+            reasoning: 'Decision asserted without evidence',
+        },
+        {
+            name: 'Consequences are documented',
+            score: 6,
+            reasoning: 'Only positive consequences listed',
+        },
+        { name: 'No contradictions with existing ADRs', score: 8, reasoning: 'OK' },
+    ]);
+    beforeEach(() => {
+        vi.restoreAllMocks();
+    });
+    it('T-FR-03-1: self-evaluation runs and passes for quality ADR summary', async () => {
+        const provider = {
+            generateText: vi.fn().mockResolvedValue({
+                text: adrPassingScores,
+                toolCalls: [],
+                tokensIn: 10,
+                tokensOut: 20,
+            }),
+        };
+        const result = await evaluate('# ADR Summary\n\nAll ADRs produced.', adrRubric, mockModel, provider, 2);
+        expect(result.evaluation.pass).toBe(true);
+        expect(result.evaluation.scores).toHaveLength(5);
+        expect(result.evaluation.aggregate).toBeGreaterThan(7);
+    });
+    it('T-FR-03-1: self-evaluation fails when blocking criteria are below threshold', async () => {
+        const provider = {
+            generateText: vi.fn().mockResolvedValue({
+                text: adrFailingScores,
+                toolCalls: [],
+                tokensIn: 10,
+                tokensOut: 20,
+            }),
+        };
+        const result = await evaluate('# Weak ADR Summary', adrRubric, mockModel, provider, 1);
+        expect(result.evaluation.pass).toBe(false);
+        const failingCriteria = result.evaluation.scores.filter((s) => s.weight === 'blocking' && s.score < 7);
+        expect(failingCriteria.length).toBeGreaterThan(0);
+    });
+    it('T-FR-03-2: refinement is triggered when ADR evaluation fails', async () => {
+        let callCount = 0;
+        const provider = {
+            generateText: vi.fn().mockImplementation(() => {
+                callCount++;
+                if (callCount === 1) {
+                    return Promise.resolve({
+                        text: adrFailingScores,
+                        toolCalls: [],
+                        tokensIn: 10,
+                        tokensOut: 20,
+                    });
+                }
+                return Promise.resolve({
+                    text: adrPassingScores,
+                    toolCalls: [],
+                    tokensIn: 10,
+                    tokensOut: 20,
+                });
+            }),
+        };
+        const refineCallback = vi
+            .fn()
+            .mockResolvedValue('# Revised ADR Summary\n\nBalanced options added.');
+        const result = await evaluate('# Weak ADR', adrRubric, mockModel, provider, 3, refineCallback);
+        expect(refineCallback).toHaveBeenCalledOnce();
+        expect(result.evaluation.pass).toBe(true);
+        expect(result.finalArtifact).toContain('Revised ADR Summary');
+    });
+    it('T-FR-03-2: refinement feedback includes failing ADR criteria', async () => {
+        let callCount = 0;
+        const provider = {
+            generateText: vi.fn().mockImplementation(() => {
+                callCount++;
+                if (callCount === 1) {
+                    return Promise.resolve({
+                        text: adrFailingScores,
+                        toolCalls: [],
+                        tokensIn: 10,
+                        tokensOut: 20,
+                    });
+                }
+                return Promise.resolve({
+                    text: adrPassingScores,
+                    toolCalls: [],
+                    tokensIn: 10,
+                    tokensOut: 20,
+                });
+            }),
+        };
+        const refineCallback = vi.fn().mockResolvedValue('Refined');
+        await evaluate('ADR', adrRubric, mockModel, provider, 2, refineCallback);
+        const feedback = refineCallback.mock.calls[0][0];
+        expect(feedback).toContain('Options analysis is balanced');
+        expect(feedback).toContain('straw-man');
+        expect(feedback).toContain('Rationale is justified by analysis');
+    });
+    it('T-FR-03-3: quality flag set when self-evaluation is exhausted', async () => {
+        const provider = {
+            generateText: vi.fn().mockResolvedValue({
+                text: adrFailingScores,
+                toolCalls: [],
+                tokensIn: 10,
+                tokensOut: 20,
+            }),
+        };
+        const refineCallback = vi.fn().mockResolvedValue('Still failing');
+        const result = await evaluate('Bad ADR', adrRubric, mockModel, provider, 2, refineCallback);
+        expect(result.evaluation.pass).toBe(false);
+        expect(result.evaluation.maxIterations).toBe(2);
+        expect(refineCallback).toHaveBeenCalledOnce();
+    });
+    it('T-FR-03-4: evaluation scores are structured for provenance recording', async () => {
+        const provider = {
+            generateText: vi.fn().mockResolvedValue({
+                text: adrPassingScores,
+                toolCalls: [],
+                tokensIn: 10,
+                tokensOut: 20,
+            }),
+        };
+        const result = await evaluate('ADR Summary', adrRubric, mockModel, provider, 1);
+        expect(result.evaluation.scores).toHaveLength(5);
+        for (const score of result.evaluation.scores) {
+            expect(score).toHaveProperty('name');
+            expect(score).toHaveProperty('weight');
+            expect(score).toHaveProperty('score');
+            expect(score).toHaveProperty('reasoning');
+        }
+        expect(typeof result.evaluation.aggregate).toBe('number');
+        expect(typeof result.evaluation.iteration).toBe('number');
+        expect(typeof result.evaluation.maxIterations).toBe('number');
+    });
+});
+describe('T-01-005e: scores in provenance', () => {
+    it('returns per-criterion scores with name, weight, score, reasoning', async () => {
+        const provider = {
+            generateText: vi.fn().mockResolvedValue({
+                text: passingScores,
+                toolCalls: [],
+                tokensIn: 10,
+                tokensOut: 20,
+            }),
+        };
+        const result = await evaluate('Good artifact', testRubric, mockModel, provider, 1);
+        expect(result.evaluation.scores).toHaveLength(4);
+        for (const score of result.evaluation.scores) {
+            expect(score).toHaveProperty('name');
+            expect(score).toHaveProperty('weight');
+            expect(score).toHaveProperty('score');
+            expect(score).toHaveProperty('reasoning');
+            expect(typeof score.score).toBe('number');
+        }
+    });
+    it('returns aggregate score for provenance recording', async () => {
+        const provider = {
+            generateText: vi.fn().mockResolvedValue({
+                text: passingScores,
+                toolCalls: [],
+                tokensIn: 10,
+                tokensOut: 20,
+            }),
+        };
+        const result = await evaluate('Artifact', testRubric, mockModel, provider, 1);
+        expect(result.evaluation.aggregate).toBeGreaterThan(0);
+        expect(typeof result.evaluation.aggregate).toBe('number');
+    });
+    it('includes iteration count and maxIterations', async () => {
+        const provider = {
+            generateText: vi.fn().mockResolvedValue({
+                text: passingScores,
+                toolCalls: [],
+                tokensIn: 10,
+                tokensOut: 20,
+            }),
+        };
+        const result = await evaluate('Artifact', testRubric, mockModel, provider, 3);
+        expect(result.evaluation.iteration).toBe(0);
+        expect(result.evaluation.maxIterations).toBe(3);
+    });
+    it('formats failure feedback with criterion details', () => {
+        const scores = [
+            { name: 'Tests', weight: 'blocking', score: 5, reasoning: 'Missing edge cases' },
+            { name: 'Patterns', weight: 'blocking', score: 8, reasoning: 'OK' },
+            { name: 'Docs', weight: 'important', score: 4, reasoning: 'No links' },
+        ];
+        const feedback = formatFailureFeedback(scores);
+        expect(feedback).toContain('Tests');
+        expect(feedback).toContain('Missing edge cases');
+        expect(feedback).toContain('blocking');
+        expect(feedback).toContain('Docs');
+        expect(feedback).toContain('No links');
+        expect(feedback).not.toContain('Patterns');
+    });
+});

package/dist/evaluation/evaluator.d.ts

@@ -0,0 +1,19 @@
+import type { EvaluationRubric, LlmProvider } from '../types/index.js';
+import type { ResolvedModel } from '../config/model-resolver.js';
+import type { EvaluationResult } from './scorer.js';
+/**
+ * Evaluate an artifact against a quality rubric using an LLM.
+ *
+ * Implements the self-evaluation loop per ADR-0004:
+ * 1. Score each criterion numerically (LLM output)
+ * 2. Check blocking criteria against pass_threshold
+ * 3. If failed, invoke refineCallback with feedback and re-evaluate
+ * 4. On exhaustion, return the last evaluation with pass=false
+ *
+ * The caller (pipeline) decides whether to publish with a quality flag.
+ */
+export declare function evaluate(artifact: string, rubric: EvaluationRubric, model: ResolvedModel, provider: LlmProvider, maxIterations?: number, refineCallback?: (feedback: string) => Promise<string>): Promise<{
+    finalArtifact: string;
+    evaluation: EvaluationResult;
+}>;
+//# sourceMappingURL=evaluator.d.ts.map

package/dist/evaluation/evaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../../src/evaluation/evaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACvE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAQjE,OAAO,KAAK,EAAkB,gBAAgB,EAAE,MAAM,aAAa,CAAC;AA4BpE;;;;;;;;;;GAUG;AACH,wBAAsB,QAAQ,CAC5B,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,gBAAgB,EACxB,KAAK,EAAE,aAAa,EACpB,QAAQ,EAAE,WAAW,EACrB,aAAa,CAAC,EAAE,MAAM,EACtB,cAAc,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,GACrD,OAAO,CAAC;IAAE,aAAa,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,gBAAgB,CAAA;CAAE,CAAC,CAkDlE"}

package/dist/evaluation/evaluator.js

@@ -0,0 +1,78 @@
+import { callLlm } from '../engine/llm-client.js';
+import { parseEvaluationResponse, computeAggregate, determinePassFail, formatFailureFeedback, } from './scorer.js';
+const DEFAULT_MAX_ITERATIONS = 2;
+function getMaxIterations() {
+    const env = process.env['CREW_MAX_REFINEMENT_ITERATIONS'];
+    if (env) {
+        const parsed = parseInt(env, 10);
+        if (!Number.isNaN(parsed) && parsed > 0)
+            return parsed;
+    }
+    return DEFAULT_MAX_ITERATIONS;
+}
+function buildEvaluationPrompt(artifact, rubric) {
+    const scale = rubric.rubric.scoring_scale ?? 10;
+    const criteriaList = rubric.rubric.criteria
+        .map((c) => `- "${c.name}" (weight: ${c.weight})${c.description ? `: ${c.description}` : ''}`)
+        .join('\n');
+    return (`Evaluate the following artifact against each criterion on a scale of 1-${scale}.\n\n` +
+        `Criteria:\n${criteriaList}\n\n` +
+        `Respond with ONLY a JSON array of objects, one per criterion:\n` +
+        `[{ "name": "criterion name", "score": <number 1-${scale}>, "reasoning": "brief explanation" }]\n\n` +
+        `--- Artifact to evaluate ---\n${artifact}\n--- End artifact ---`);
+}
+/**
+ * Evaluate an artifact against a quality rubric using an LLM.
+ *
+ * Implements the self-evaluation loop per ADR-0004:
+ * 1. Score each criterion numerically (LLM output)
+ * 2. Check blocking criteria against pass_threshold
+ * 3. If failed, invoke refineCallback with feedback and re-evaluate
+ * 4. On exhaustion, return the last evaluation with pass=false
+ *
+ * The caller (pipeline) decides whether to publish with a quality flag.
+ */
+export async function evaluate(artifact, rubric, model, provider, maxIterations, refineCallback) {
+    const maxIter = maxIterations ?? getMaxIterations();
+    let currentArtifact = artifact;
+    let lastScores = [];
+    let lastAggregate = 0;
+    for (let iteration = 0; iteration < maxIter; iteration++) {
+        const prompt = buildEvaluationPrompt(currentArtifact, rubric);
+        const result = await callLlm({
+            provider,
+            model,
+            system: 'You are a quality evaluator. Score the artifact against the provided criteria. Return ONLY a JSON array.',
+            messages: [{ role: 'user', content: prompt }],
+            maxRetries: 1,
+        });
+        lastScores = parseEvaluationResponse(result.text, rubric);
+        lastAggregate = computeAggregate(lastScores);
+        if (determinePassFail(lastScores, rubric)) {
+            return {
+                finalArtifact: currentArtifact,
+                evaluation: {
+                    scores: lastScores,
+                    aggregate: lastAggregate,
+                    pass: true,
+                    iteration,
+                    maxIterations: maxIter,
+                },
+            };
+        }
+        if (iteration < maxIter - 1 && refineCallback) {
+            const feedback = formatFailureFeedback(lastScores);
+            currentArtifact = await refineCallback(feedback);
+        }
+    }
+    return {
+        finalArtifact: currentArtifact,
+        evaluation: {
+            scores: lastScores,
+            aggregate: lastAggregate,
+            pass: false,
+            iteration: maxIter - 1,
+            maxIterations: maxIter,
+        },
+    };
+}

package/dist/evaluation/index.d.ts

@@ -0,0 +1,4 @@
+export { evaluate } from './evaluator.js';
+export { computeAggregate, determinePassFail, parseEvaluationResponse, formatFailureFeedback, } from './scorer.js';
+export type { CriterionScore, EvaluationResult } from './scorer.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/evaluation/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/evaluation/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EACL,gBAAgB,EAChB,iBAAiB,EACjB,uBAAuB,EACvB,qBAAqB,GACtB,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC"}

package/dist/evaluation/index.js

@@ -0,0 +1,2 @@
+export { evaluate } from './evaluator.js';
+export { computeAggregate, determinePassFail, parseEvaluationResponse, formatFailureFeedback, } from './scorer.js';

package/dist/evaluation/scorer.d.ts

@@ -0,0 +1,38 @@
+import type { CriterionWeight, EvaluationRubric } from '../types/index.js';
+export interface CriterionScore {
+    name: string;
+    weight: CriterionWeight;
+    score: number;
+    reasoning: string;
+}
+export interface EvaluationResult {
+    scores: CriterionScore[];
+    aggregate: number;
+    pass: boolean;
+    iteration: number;
+    maxIterations: number;
+}
+/**
+ * Compute a weighted aggregate score from per-criterion scores.
+ * blocking=3, important=2, advisory=1
+ */
+export declare function computeAggregate(scores: CriterionScore[]): number;
+/**
+ * Determine pass/fail from per-criterion scores and rubric thresholds.
+ *
+ * Pass requires:
+ * 1. All blocking criteria score >= pass_threshold
+ * 2. If aggregate_threshold is set, the weighted aggregate must meet it
+ */
+export declare function determinePassFail(scores: CriterionScore[], rubric: EvaluationRubric): boolean;
+/**
+ * Parse the LLM's structured evaluation response into typed scores.
+ * Expects a JSON array of { name, score, reasoning } objects.
+ * Merges the weight from the rubric criteria definitions.
+ */
+export declare function parseEvaluationResponse(text: string, rubric: EvaluationRubric): CriterionScore[];
+/**
+ * Format failure feedback for the refinement callback.
+ */
+export declare function formatFailureFeedback(scores: CriterionScore[]): string;
+//# sourceMappingURL=scorer.d.ts.map

package/dist/evaluation/scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scorer.d.ts","sourceRoot":"","sources":["../../src/evaluation/scorer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAE3E,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,eAAe,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,cAAc,EAAE,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;CACvB;AAQD;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,CAcjE;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,cAAc,EAAE,EAAE,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAY7F;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,cAAc,EAAE,CAuChG;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,CAUtE"}