@cat-factory/executor-harness 1.31.0 → 1.31.6

package/dist/tester.js

@@ -1,191 +0,0 @@
-import { execFile } from 'node:child_process';
-import { promisify } from 'node:util';
-import { cloneRepo } from './git.js';
-import { extractJsonObject } from './blueprint.js';
-import { agentNeverActed, agentOutputTail, NEVER_ACTED_CAUSE, runAgentInWorkspace, withWorkspace, } from './pi-workspace.js';
-import { diagnosticsSuffix, resolveStructuredOutput, } from './structured-output.js';
-import { log } from './logger.js';
-const exec = promisify(execFile);
-// Async job execution for the Tester. The engine dispatches this to run the project's
-// tests: clone the PR HEAD branch, stand its dependencies up (local docker-compose
-// infra, or test against an ephemeral env), run Pi to exercise the change + regress
-// related behaviour, and return ONLY a structured JSON report. The Tester makes NO
-// commits — on a withheld greenlight the engine loops the `fixer` and re-tests.
-/** Compact description of the report shape, fed to the JSON repair call. */
-const REPORT_SHAPE_HINT = 'Expected a test report: {"greenlight": boolean, "summary": string, "tested": string[], ' +
-    '"outcomes": [{"name": string, "status": "passed"|"failed"|"skipped", "detail"?: string}], ' +
-    '"concerns": [{"title": string, "detail": string, "severity": "low"|"medium"|"high"|"critical"}]}.';
-const SEVERITIES = new Set(['low', 'medium', 'high', 'critical']);
-const STATUSES = new Set(['passed', 'failed', 'skipped']);
-/** Coerce the agent's JSON into a well-formed report, defaulting conservatively. */
-function coerceReport(raw, summary, env) {
-    const o = (typeof raw === 'object' && raw !== null ? raw : {});
-    const outcomes = Array.isArray(o.outcomes)
-        ? o.outcomes
-            .filter((x) => typeof x === 'object' && x !== null)
-            .map((x) => ({
-            name: typeof x.name === 'string' ? x.name : '(unnamed)',
-            status: (STATUSES.has(x.status)
-                ? x.status
-                : 'skipped'),
-            ...(typeof x.detail === 'string' && x.detail ? { detail: x.detail } : {}),
-        }))
-        : [];
-    const concerns = Array.isArray(o.concerns)
-        ? o.concerns
-            .filter((x) => typeof x === 'object' && x !== null)
-            .map((x) => ({
-            title: typeof x.title === 'string' ? x.title : '(concern)',
-            detail: typeof x.detail === 'string' ? x.detail : '',
-            severity: (SEVERITIES.has(x.severity)
-                ? x.severity
-                : 'medium'),
-        }))
-        : [];
-    // A greenlight is only honoured when no BLOCKING (high/critical) concern was
-    // raised — never auto-pass a run with an open blocker, even if the model set
-    // greenlight:true by mistake. Low/medium concerns are advisory: they're reported
-    // but don't, on their own, withhold the greenlight (which would otherwise burn the
-    // whole fixer budget looping on a trivial nit). The engine re-applies this rule.
-    const blocking = concerns.some((c) => c.severity === 'high' || c.severity === 'critical');
-    const greenlight = o.greenlight === true && !blocking;
-    return {
-        greenlight,
-        summary: typeof o.summary === 'string' && o.summary ? o.summary : summary.slice(0, 2000),
-        tested: Array.isArray(o.tested)
-            ? o.tested.filter((t) => typeof t === 'string')
-            : [],
-        outcomes,
-        concerns,
-        environment: env,
-    };
-}
-/** Build the tester task prompt: how to bring the deps up + what to test. */
-function buildUserPrompt(job) {
-    const lines = [job.userPrompt, ''];
-    if (job.test.environment === 'ephemeral') {
-        lines.push('Run mode: ephemeral environment.', job.test.environmentUrl
-            ? `Test against the deployed environment at ${job.test.environmentUrl}. Do not start the service locally.`
-            : 'Test against the provided ephemeral environment URL from your context. Do not start the service locally.');
-    }
-    else if (job.test.noInfraDependencies) {
-        lines.push('Run mode: local, no infra dependencies — just install, build and run the test suite directly.');
-    }
-    else {
-        lines.push("Run mode: local. The service's infra dependencies from its docker-compose file have been started and are reachable on localhost. Read the README to learn how to configure the service against them, run any migrations, start the service and exercise it.");
-    }
-    lines.push('', 'Respond with ONLY the JSON test report described in your instructions.');
-    return lines.join('\n');
-}
-/**
- * Bring the service's docker-compose dependencies up (local mode only). Best-effort:
- * runs `docker compose -f <path> up -d --wait` in the checkout. A missing Docker
- * daemon or a compose failure is logged and surfaced to the agent rather than failing
- * the whole job — the agent can still run unit-level tests and report what it could.
- */
-async function standUpInfra(dir, test, signal, trace) {
-    if (test.environment !== 'local' || test.noInfraDependencies || !test.composePath) {
-        return { started: false };
-    }
-    try {
-        log.info('test: standing up infra', { ...trace, composePath: test.composePath });
-        await exec('docker', ['compose', '-f', test.composePath, 'up', '-d', '--wait'], {
-            cwd: dir,
-            signal,
-            timeout: 5 * 60_000,
-        });
-        return { started: true };
-    }
-    catch (err) {
-        const note = err instanceof Error ? err.message : String(err);
-        log.warn('test: infra stand-up failed', { ...trace, error: note });
-        return { started: false, note };
-    }
-}
-/** Tear the docker-compose dependencies down (best-effort). */
-async function tearDownInfra(dir, test) {
-    if (test.environment !== 'local' || test.noInfraDependencies || !test.composePath)
-        return;
-    try {
-        await exec('docker', ['compose', '-f', test.composePath, 'down', '-v'], {
-            cwd: dir,
-            timeout: 2 * 60_000,
-        });
-    }
-    catch {
-        // The container is ephemeral and torn down with the run anyway — ignore.
-    }
-}
-/** Run one Tester job end to end: clone branch → stand up infra → Pi tests → report. */
-export async function handleTester(job, opts = {}) {
-    const trace = { jobId: job.jobId, repo: `${job.repo.owner}/${job.repo.name}`, branch: job.branch };
-    return withWorkspace('test', async (dir) => {
-        log.info('test: cloning PR branch', trace);
-        await cloneRepo({
-            repo: { ...job.repo, baseBranch: job.branch },
-            ghToken: job.ghToken,
-            dir,
-            signal: opts.signal,
-        });
-        const infra = await standUpInfra(dir, job.test, opts.signal, trace);
-        try {
-            log.info('test: running agent', { ...trace, environment: job.test.environment });
-            let userPrompt = buildUserPrompt(job);
-            if (infra.note) {
-                userPrompt += `\n\nNote: standing the infra up reported a problem (${infra.note}). Test what you can and flag any dependency-related gaps as concerns.`;
-            }
-            const { summary, stats, stderrTail, usage } = await runAgentInWorkspace({
-                dir,
-                systemPrompt: job.systemPrompt,
-                userPrompt,
-                model: job.model,
-                harness: job.harness,
-                subscriptionToken: job.subscriptionToken,
-                subscriptionBaseUrl: job.subscriptionBaseUrl,
-                proxyBaseUrl: job.proxyBaseUrl,
-                sessionToken: job.sessionToken,
-                // The tester only assesses (it commits nothing), so the no-edit guard must
-                // not fire on its legitimately edit-free run.
-                expectsEdits: false,
-            }, opts);
-            const { value: report, diagnostics } = await resolveStructuredOutput({
-                label: 'tester',
-                shapeHint: REPORT_SHAPE_HINT,
-                parse: (text) => coerceReport(extractJsonObject(text), text, job.test.environment),
-            }, summary, {
-                harness: job.harness,
-                subscriptionToken: job.subscriptionToken,
-                subscriptionBaseUrl: job.subscriptionBaseUrl,
-                proxyBaseUrl: job.proxyBaseUrl,
-                sessionToken: job.sessionToken,
-                model: job.model,
-                jobId: job.jobId,
-                signal: opts.signal,
-            });
-            if (!report) {
-                return {
-                    summary,
-                    stats,
-                    error: noReportReason(stats, stderrTail, diagnostics),
-                    ...(usage ? { usage } : {}),
-                };
-            }
-            log.info('test: reported', {
-                ...trace,
-                greenlight: report.greenlight,
-                concerns: report.concerns.length,
-            });
-            return { report, summary, stats, ...(usage ? { usage } : {}) };
-        }
-        finally {
-            await tearDownInfra(dir, job.test);
-        }
-    });
-}
-/** Human-readable reason a tester run produced no usable report. */
-function noReportReason(stats, stderrTail, diagnostics) {
-    const cause = agentNeverActed(stats)
-        ? NEVER_ACTED_CAUSE
-        : ' The agent did not return a parseable JSON test report.';
-    return `Tester produced no report.${cause}${diagnostics ? diagnosticsSuffix(diagnostics) : ''}${agentOutputTail(stderrTail)}`;
-}