@kylecheng3146/agent-ops 0.0.1

package/dist/runtime/src/verify/service.js

@@ -0,0 +1,217 @@
+import { AgentOpsError, resolveContainedPath } from "../fs/paths.js";
+import { validateTaskAgainstConfig } from "../schema/validate.js";
+import { collectChangeSurface } from "./change-surface.js";
+import { buildVerificationEvidence } from "./evidence.js";
+import { createFailureFingerprint } from "./fingerprint.js";
+import { selectVerificationScope } from "./scope.js";
+import { runVerificationCommand } from "./spawn.js";
+import { evaluateTestCount, parseTestCount } from "./test-count.js";
+function verificationError(code, message) {
+    return new AgentOpsError(code, message);
+}
+function classifyTestCountCode(code) {
+    const classes = {
+        TEST_COUNT_BELOW_MINIMUM: "test-count-below-minimum",
+        TEST_COUNT_INVALID: "test-count-invalid",
+        TEST_COUNT_OK: "none",
+        TEST_COUNT_REQUIREMENT_INVALID: "test-count-requirement-invalid",
+        TEST_COUNT_UNPARSEABLE: "test-count-unparseable",
+        ZERO_TESTS: "zero-tests"
+    };
+    return classes[code];
+}
+function classifyResult(command, spawned) {
+    if (command.evidence.kind === "file") {
+        return {
+            status: spawned.status === "PASS" ? "UNKNOWN" : spawned.status,
+            failureClass: spawned.status === "PASS"
+                ? "file-evidence-unsupported"
+                : spawned.failureClass,
+            testCount: null
+        };
+    }
+    if (command.evidence.kind !== "test-count") {
+        return {
+            status: spawned.status,
+            failureClass: spawned.failureClass,
+            testCount: null
+        };
+    }
+    const testCount = parseTestCount(`${spawned.stdout}\n${spawned.stderr}`);
+    if (spawned.status !== "PASS") {
+        return {
+            status: spawned.status,
+            failureClass: spawned.failureClass,
+            testCount
+        };
+    }
+    const evaluation = evaluateTestCount(testCount, command.evidence.minimum);
+    return {
+        status: evaluation.status,
+        failureClass: classifyTestCountCode(evaluation.code),
+        testCount: evaluation.testCount
+    };
+}
+function exitCategory(result) {
+    if (result.timedOut) {
+        return "timeout";
+    }
+    if (result.failureClass === "signal-exit") {
+        return "signal-exit";
+    }
+    if (result.exitCode === null) {
+        return "no-exit";
+    }
+    return result.exitCode === 0 ? "exit-zero" : "nonzero-exit";
+}
+function overallStatus(results) {
+    const required = results.filter((result) => result.required);
+    const gating = required.length > 0 ? required : results;
+    if (gating.some((result) => result.status === "FAIL")) {
+        return "FAIL";
+    }
+    if (gating.some((result) => result.status === "UNKNOWN")) {
+        return "UNKNOWN";
+    }
+    return "PASS";
+}
+function relevantCriteria(task, commandId) {
+    return task.criteria.filter((criterion) => criterion.verifierIds.includes(commandId));
+}
+function commandById(config, commandId) {
+    const command = config.verification.commands.find((candidate) => candidate.id === commandId);
+    if (command === undefined) {
+        throw verificationError("VERIFICATION_COMMAND_NOT_FOUND", `Verification command not found: ${commandId}`);
+    }
+    return command;
+}
+function diagnostics(spawned) {
+    return spawned.stderr || spawned.stdout || spawned.failureClass;
+}
+export class VerificationService {
+    #options;
+    constructor(options) {
+        this.#options = options;
+    }
+    async #commandCwd(command) {
+        if (command.cwd === ".") {
+            return this.#options.root;
+        }
+        return await resolveContainedPath(this.#options.root, command.cwd);
+    }
+    async #persistEvidence(task, command, startedAt, finishedAt, result, exitCode) {
+        const references = [];
+        for (const criterion of relevantCriteria(task, command.id)) {
+            const evidence = buildVerificationEvidence({
+                taskId: task.id,
+                criterionId: criterion.id,
+                command,
+                scope: this.#options.scope,
+                startedAt,
+                finishedAt,
+                exitCode,
+                testCount: result.testCount,
+                toolVersions: this.#options.toolVersions ?? {},
+                config: this.#options.config
+            });
+            references.push(await this.#options.evidenceStore.save(evidence));
+        }
+        return references;
+    }
+    async #runCommand(task, command) {
+        const startedAt = (this.#options.now ?? (() => new Date().toISOString()))();
+        const spawned = this.#options.trusted
+            ? await runVerificationCommand(command, {
+                cwd: await this.#commandCwd(command),
+                runner: this.#options.processRunner
+            })
+            : {
+                commandId: command.id,
+                status: "UNKNOWN",
+                failureClass: "repository-untrusted",
+                exitCode: null,
+                signal: null,
+                timedOut: false,
+                durationMs: 0,
+                stdout: "",
+                stderr: "",
+                stdoutTruncated: false,
+                stderrTruncated: false
+            };
+        const classified = classifyResult(command, spawned);
+        const fingerprint = classified.status === "PASS"
+            ? null
+            : createFailureFingerprint({
+                commandId: command.id,
+                failureClass: classified.failureClass,
+                exitCategory: spawned.timedOut
+                    ? "timeout"
+                    : spawned.signal !== null
+                        ? "signal-exit"
+                        : spawned.exitCode === null
+                            ? "no-exit"
+                            : spawned.exitCode === 0
+                                ? "exit-zero"
+                                : "nonzero-exit",
+                diagnostics: diagnostics(spawned)
+            });
+        const finishedAt = (this.#options.now ?? (() => new Date().toISOString()))();
+        const evidenceReferences = await this.#persistEvidence(task, command, startedAt, finishedAt, classified, spawned.exitCode);
+        return {
+            commandId: command.id,
+            required: command.required,
+            status: classified.status,
+            failureClass: classified.failureClass,
+            exitCode: spawned.exitCode,
+            timedOut: spawned.timedOut,
+            testCount: classified.testCount,
+            diagnostic: fingerprint?.diagnostics ?? "",
+            evidenceReferences
+        };
+    }
+    async verify(taskId) {
+        const stored = await this.#options.taskService.status({ taskId });
+        if (stored.status === "archived") {
+            throw verificationError("TASK_NOT_ACTIVE", "An archived task cannot be verified.");
+        }
+        const validation = validateTaskAgainstConfig(stored.task, this.#options.config);
+        if (!validation.ok) {
+            throw verificationError("VERIFICATION_INPUT_INVALID", validation.errors[0]?.message ??
+                "Task and verification configuration are incompatible.");
+        }
+        const surface = await collectChangeSurface(this.#options.gitRunner);
+        const selection = selectVerificationScope(surface.paths, this.#options.config);
+        const results = [];
+        for (const commandId of selection.verifierIds) {
+            results.push(await this.#runCommand(validation.value, commandById(this.#options.config, commandId)));
+        }
+        const status = overallStatus(results);
+        let signal = null;
+        if (status === "PASS") {
+            await this.#options.taskService.clearFailure(taskId);
+        }
+        else {
+            const required = results.filter((result) => result.required);
+            const gating = required.length > 0 ? required : results;
+            const failed = gating.find((result) => result.status !== "PASS");
+            if (failed === undefined) {
+                throw verificationError("VERIFICATION_RESULT_INVALID", "Verification failed without a gating result.");
+            }
+            const advanced = await this.#options.taskService.recordFailure(taskId, createFailureFingerprint({
+                commandId: failed.commandId,
+                failureClass: failed.failureClass,
+                exitCategory: exitCategory(failed),
+                diagnostics: failed.diagnostic
+            }));
+            signal = advanced.signal;
+        }
+        return {
+            taskId,
+            status,
+            surface,
+            selection,
+            results,
+            signal
+        };
+    }
+}

package/dist/runtime/src/verify/spawn.js

@@ -0,0 +1,326 @@
+import { execFile as execFileCallback, spawn } from "node:child_process";
+import { promisify } from "node:util";
+const DEFAULT_TIMEOUT_MS = 5 * 60 * 1000;
+const DEFAULT_OUTPUT_LIMIT_BYTES = 64 * 1024;
+const MAX_OUTPUT_LIMIT_BYTES = 1024 * 1024;
+const DEFAULT_TERMINATION_GRACE_MS = 500;
+const MAX_TERMINATION_GRACE_MS = 5_000;
+const execFile = promisify(execFileCallback);
+function boundedPositiveInteger(value, maximum, name) {
+    if (!Number.isSafeInteger(value) ||
+        value <= 0 ||
+        value > maximum) {
+        throw new RangeError(`${name} must be a positive integer no greater than ${maximum}.`);
+    }
+    return value;
+}
+function errorCode(error) {
+    if (typeof error === "object" &&
+        error !== null &&
+        "code" in error &&
+        typeof error.code === "string") {
+        return error.code;
+    }
+    return "UNKNOWN";
+}
+async function* readableBytes(stream) {
+    if (stream === null) {
+        return;
+    }
+    for await (const value of stream) {
+        if (typeof value === "string") {
+            yield Buffer.from(value);
+            continue;
+        }
+        if (value instanceof Uint8Array) {
+            yield value;
+            continue;
+        }
+        throw new TypeError("Process output contained an unsupported chunk.");
+    }
+}
+async function captureOutput(stream, limit) {
+    const chunks = [];
+    let storedBytes = 0;
+    let truncated = false;
+    try {
+        for await (const value of stream) {
+            const chunk = Buffer.from(value);
+            const remaining = limit - storedBytes;
+            if (remaining > 0) {
+                const retained = chunk.subarray(0, remaining);
+                chunks.push(retained);
+                storedBytes += retained.length;
+            }
+            if (chunk.length > remaining) {
+                truncated = true;
+            }
+        }
+    }
+    catch {
+        return {
+            text: Buffer.concat(chunks, storedBytes).toString("utf8"),
+            truncated,
+            failed: true
+        };
+    }
+    return {
+        text: Buffer.concat(chunks, storedBytes).toString("utf8"),
+        truncated,
+        failed: false
+    };
+}
+function settleCapturedOutput(capture, graceMs) {
+    let timer;
+    const timeout = new Promise((resolve) => {
+        timer = setTimeout(() => resolve({
+            text: "",
+            truncated: true,
+            failed: true
+        }), graceMs);
+    });
+    return Promise.race([capture, timeout]).finally(() => {
+        if (timer !== undefined) {
+            clearTimeout(timer);
+        }
+    });
+}
+function isMissingProcess(errorCodeValue) {
+    return errorCodeValue === "ENOENT";
+}
+function waitForCompletion(completion, timeoutMs) {
+    let timer;
+    const timeout = new Promise((resolve) => {
+        timer = setTimeout(() => resolve(null), timeoutMs);
+    });
+    return Promise.race([completion, timeout]).finally(() => {
+        if (timer !== undefined) {
+            clearTimeout(timer);
+        }
+    });
+}
+async function terminatePosixTree(processId, completion, graceMs) {
+    try {
+        process.kill(-processId, "SIGTERM");
+    }
+    catch (error) {
+        if (errorCode(error) === "ESRCH") {
+            return;
+        }
+        throw error;
+    }
+    await waitForCompletion(completion, graceMs);
+    try {
+        process.kill(-processId, "SIGKILL");
+    }
+    catch (error) {
+        if (errorCode(error) !== "ESRCH") {
+            throw error;
+        }
+    }
+}
+async function defaultTerminateWindowsTree(processId, graceMs) {
+    await execFile("taskkill", ["/pid", String(processId), "/T", "/F"], {
+        encoding: "utf8",
+        timeout: graceMs,
+        windowsHide: true
+    });
+}
+export class NodeVerificationProcessRunner {
+    #platform;
+    #terminateWindowsTree;
+    constructor(options = {}) {
+        this.#platform = options.platform ?? process.platform;
+        this.#terminateWindowsTree =
+            options.terminateWindowsTree ?? defaultTerminateWindowsTree;
+    }
+    start(request) {
+        const child = spawn(request.command, [...request.args], {
+            cwd: request.cwd,
+            detached: this.#platform !== "win32",
+            shell: request.shell,
+            stdio: ["ignore", "pipe", "pipe"],
+            windowsHide: true
+        });
+        let settled = false;
+        const completion = new Promise((resolve) => {
+            child.once("error", (error) => {
+                if (!settled) {
+                    settled = true;
+                    resolve({
+                        exitCode: null,
+                        signal: null,
+                        errorCode: error.code ?? "UNKNOWN"
+                    });
+                }
+            });
+            child.once("close", (exitCode, signal) => {
+                if (!settled) {
+                    settled = true;
+                    resolve({
+                        exitCode,
+                        signal
+                    });
+                }
+            });
+        });
+        const processId = child.pid ?? null;
+        return {
+            pid: processId,
+            stdout: readableBytes(child.stdout),
+            stderr: readableBytes(child.stderr),
+            completion,
+            terminateTree: async (graceMs) => {
+                if (processId === null) {
+                    throw new Error("Cannot terminate a process without an ID.");
+                }
+                if (this.#platform === "win32") {
+                    await this.#terminateWindowsTree(processId, graceMs);
+                    return;
+                }
+                await terminatePosixTree(processId, completion, graceMs);
+            }
+        };
+    }
+}
+function emptyResult(commandId, failureClass, durationMs) {
+    return {
+        commandId,
+        status: "UNKNOWN",
+        failureClass,
+        exitCode: null,
+        signal: null,
+        timedOut: false,
+        durationMs,
+        stdout: "",
+        stderr: "",
+        stdoutTruncated: false,
+        stderrTruncated: false
+    };
+}
+function classifyCompletion(completion, outputFailed) {
+    if (outputFailed) {
+        return {
+            status: "UNKNOWN",
+            failureClass: "output-read-failed"
+        };
+    }
+    if (isMissingProcess(completion.errorCode)) {
+        return {
+            status: "UNKNOWN",
+            failureClass: "missing-executable"
+        };
+    }
+    if (completion.errorCode !== undefined) {
+        return {
+            status: "UNKNOWN",
+            failureClass: "spawn-failed"
+        };
+    }
+    if (completion.signal !== null) {
+        return {
+            status: "FAIL",
+            failureClass: "signal-exit"
+        };
+    }
+    if (completion.exitCode === 0) {
+        return {
+            status: "PASS",
+            failureClass: "none"
+        };
+    }
+    return {
+        status: "FAIL",
+        failureClass: "nonzero-exit"
+    };
+}
+function elapsedMilliseconds(startedAt, finishedAt) {
+    const elapsed = Math.max(0, finishedAt - startedAt);
+    return Number.isFinite(elapsed) ? Math.round(elapsed) : 0;
+}
+function hasAcknowledgedShell(command) {
+    const value = command;
+    return value.shell !== true || value.acknowledgeRisk === true;
+}
+export async function runVerificationCommand(command, options) {
+    const now = options.now ?? Date.now;
+    const startedAt = now();
+    if (!hasAcknowledgedShell(command)) {
+        return emptyResult(command.id, "shell-risk-unacknowledged", elapsedMilliseconds(startedAt, now()));
+    }
+    const outputLimit = boundedPositiveInteger(options.outputLimitBytes ?? DEFAULT_OUTPUT_LIMIT_BYTES, MAX_OUTPUT_LIMIT_BYTES, "outputLimitBytes");
+    const terminationGrace = boundedPositiveInteger(options.terminationGraceMs ?? DEFAULT_TERMINATION_GRACE_MS, MAX_TERMINATION_GRACE_MS, "terminationGraceMs");
+    const timeoutMs = boundedPositiveInteger(command.timeoutMs ?? DEFAULT_TIMEOUT_MS, 2_147_483_647, "timeoutMs");
+    const runner = options.runner ?? new NodeVerificationProcessRunner();
+    let running;
+    try {
+        running = runner.start({
+            command: command.command,
+            args: [...command.args],
+            cwd: options.cwd,
+            shell: command.shell === true
+        });
+    }
+    catch {
+        return emptyResult(command.id, "spawn-failed", elapsedMilliseconds(startedAt, now()));
+    }
+    const stdout = captureOutput(running.stdout, outputLimit);
+    const stderr = captureOutput(running.stderr, outputLimit);
+    let timer;
+    const timeout = new Promise((resolve) => {
+        timer = setTimeout(() => resolve({ kind: "timeout" }), timeoutMs);
+    });
+    const outcome = await Promise.race([
+        running.completion.then((completion) => ({
+            kind: "completion",
+            completion
+        })),
+        timeout
+    ]);
+    if (timer !== undefined) {
+        clearTimeout(timer);
+    }
+    let completion;
+    let timedOut = false;
+    let terminationFailed = false;
+    if (outcome.kind === "timeout") {
+        timedOut = true;
+        try {
+            await running.terminateTree(terminationGrace);
+        }
+        catch {
+            terminationFailed = true;
+        }
+        completion =
+            await waitForCompletion(running.completion, terminationGrace) ??
+                { exitCode: null, signal: null };
+    }
+    else {
+        completion = outcome.completion;
+    }
+    const [capturedStdout, capturedStderr] = await Promise.all([
+        settleCapturedOutput(stdout, terminationGrace),
+        settleCapturedOutput(stderr, terminationGrace)
+    ]);
+    const classified = timedOut
+        ? {
+            status: terminationFailed ? "UNKNOWN" : "FAIL",
+            failureClass: terminationFailed
+                ? "termination-failed"
+                : "timeout"
+        }
+        : classifyCompletion(completion, capturedStdout.failed || capturedStderr.failed);
+    return {
+        commandId: command.id,
+        status: classified.status,
+        failureClass: classified.failureClass,
+        exitCode: completion.exitCode,
+        signal: completion.signal,
+        timedOut,
+        durationMs: elapsedMilliseconds(startedAt, now()),
+        stdout: capturedStdout.text,
+        stderr: capturedStderr.text,
+        stdoutTruncated: capturedStdout.truncated,
+        stderrTruncated: capturedStderr.truncated
+    };
+}

package/dist/runtime/src/verify/test-count.js

@@ -0,0 +1,148 @@
+const MAX_SUMMARY_BYTES = 1024 * 1024;
+const NUMBER_SOURCE = String.raw `\d{1,16}`;
+function safeCount(value) {
+    const parsed = Number(value);
+    return Number.isSafeInteger(parsed) && parsed >= 0
+        ? parsed
+        : null;
+}
+function addCandidate(candidates, source) {
+    const parsed = safeCount(source);
+    if (parsed !== null) {
+        candidates.push(parsed);
+    }
+}
+function parsePytestSummary(line) {
+    if (!/\bin \d+(?:\.\d+)?s(?:\s|$)/u.test(line)) {
+        return null;
+    }
+    const statusPattern = new RegExp(`(${NUMBER_SOURCE}) (passed|failed|skipped|error|errors|xfailed|xpassed|deselected)`, "gu");
+    let total = 0;
+    let matched = false;
+    for (const match of line.matchAll(statusPattern)) {
+        const source = match[1];
+        const status = match[2];
+        if (source === undefined || status === undefined) {
+            return null;
+        }
+        const count = safeCount(source);
+        if (count === null ||
+            (status !== "deselected" &&
+                total > Number.MAX_SAFE_INTEGER - count)) {
+            return null;
+        }
+        if (status !== "deselected") {
+            total += count;
+        }
+        matched = true;
+    }
+    return matched ? total : null;
+}
+function parseRustSummary(line) {
+    const match = new RegExp(`^test result: (?:ok|FAILED)\\. (${NUMBER_SOURCE}) passed; (${NUMBER_SOURCE}) failed; ${NUMBER_SOURCE} ignored; ${NUMBER_SOURCE} measured; ${NUMBER_SOURCE} filtered out$`, "u").exec(line);
+    if (match === null) {
+        return null;
+    }
+    const passedSource = match[1];
+    const failedSource = match[2];
+    if (passedSource === undefined || failedSource === undefined) {
+        return null;
+    }
+    const passed = safeCount(passedSource);
+    const failed = safeCount(failedSource);
+    if (passed === null ||
+        failed === null ||
+        passed > Number.MAX_SAFE_INTEGER - failed) {
+        return null;
+    }
+    return passed + failed;
+}
+export function parseTestCount(output) {
+    if (output.includes("\0") ||
+        Buffer.byteLength(output, "utf8") > MAX_SUMMARY_BYTES) {
+        return null;
+    }
+    const candidates = [];
+    const nodePattern = new RegExp(`^# tests (${NUMBER_SOURCE})$`, "u");
+    const collectedPattern = new RegExp(`^collected (${NUMBER_SOURCE}) items?$`, "u");
+    const jestPattern = new RegExp(`^Tests:\\s+.*\\b(${NUMBER_SOURCE}) total(?:\\s|$)`, "u");
+    const vitestPattern = new RegExp(`^Tests\\s+.*\\((${NUMBER_SOURCE})\\)\\s*$`, "u");
+    for (const rawLine of output.split(/\r?\n/u)) {
+        const line = rawLine.trim();
+        if (line.length === 0) {
+            continue;
+        }
+        const node = nodePattern.exec(line);
+        if (node?.[1] !== undefined) {
+            addCandidate(candidates, node[1]);
+        }
+        const collected = collectedPattern.exec(line);
+        if (collected?.[1] !== undefined) {
+            addCandidate(candidates, collected[1]);
+        }
+        const jest = jestPattern.exec(line);
+        if (jest?.[1] !== undefined) {
+            addCandidate(candidates, jest[1]);
+        }
+        const vitest = vitestPattern.exec(line);
+        if (vitest?.[1] !== undefined) {
+            addCandidate(candidates, vitest[1]);
+        }
+        const pytest = parsePytestSummary(line);
+        if (pytest !== null) {
+            candidates.push(pytest);
+        }
+        const rust = parseRustSummary(line);
+        if (rust !== null) {
+            candidates.push(rust);
+        }
+    }
+    const first = candidates[0];
+    if (first === undefined ||
+        candidates.some((candidate) => candidate !== first)) {
+        return null;
+    }
+    return first;
+}
+export function evaluateTestCount(testCount, minimum = 1) {
+    if (!Number.isSafeInteger(minimum) || minimum < 0) {
+        return {
+            status: "UNKNOWN",
+            code: "TEST_COUNT_REQUIREMENT_INVALID",
+            testCount
+        };
+    }
+    if (testCount === null) {
+        return {
+            status: "UNKNOWN",
+            code: "TEST_COUNT_UNPARSEABLE",
+            testCount: null
+        };
+    }
+    if (!Number.isSafeInteger(testCount) || testCount < 0) {
+        return {
+            status: "UNKNOWN",
+            code: "TEST_COUNT_INVALID",
+            testCount: null
+        };
+    }
+    if (testCount === 0) {
+        return {
+            status: "FAIL",
+            code: "ZERO_TESTS",
+            testCount
+        };
+    }
+    if (testCount < minimum) {
+        return {
+            status: "FAIL",
+            code: "TEST_COUNT_BELOW_MINIMUM",
+            testCount
+        };
+    }
+    return {
+        status: "PASS",
+        code: "TEST_COUNT_OK",
+        testCount
+    };
+}

package/docs/en/spec/README.md

@@ -0,0 +1,13 @@
+# Loop Engineering Specification
+
+This is the normative English specification for bounded, evidence-driven work.
+
+- [Loop engineering](./loop-engineering.md)
+- [Acceptance and evidence](./acceptance-and-evidence.md)
+- [Judgment](./judgment.md)
+- [Delegation](./delegation.md)
+- [Review](./review.md)
+- [Troubleshooting](./troubleshooting.md)
+- [Guardrails](./guardrails.md)
+- [Maintenance](./maintenance.md)
+- [Harness adapters](./harness-adapters.md)