@tea-agent/loop-agent 0.4.0 → 0.6.0

package/dist/worker/run-task/run-task.js

@@ -2,6 +2,7 @@ import { createHash } from "node:crypto";
 import { mkdir, readFile, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { formatDuration, noopProgressReporter } from "../progress-reporter.js";
+import { redactForPreview, truncatePreview } from "../observability/events.js";
 import { materializeTaskSpec, } from "../materialize/harness-task-materializer.js";
 import { preflightTargetRepo } from "../preflight.js";
 export const DEFAULT_RUN_DAG_TIMEOUT_MS = 1_800_000;
@@ -10,110 +11,139 @@ export async function runTaskSpec(options) {
     const now = options.now ?? new Date();
     const workerRunId = options.workerRunId ?? buildWorkerRunId(options.taskSpec.id, now);
     const commands = [];
-    const client = new RecordingRunTaskClient(options.client, commands);
     const progress = options.progress ?? noopProgressReporter;
+    const taskId = options.taskSpec.id;
+    const eventCtx = { progress, workerRunId, taskId };
+    const recordingClient = new RecordingRunTaskClient(options.client, commands);
+    const client = new ObservedRunTaskClient(recordingClient, eventCtx);
     if (options.preflight !== false) {
-        progress.step("preflight: version + inspect + docs-audit + git-status" + (options.preflight && typeof options.preflight === "object" && options.preflight.runCheckRepo ? " + check-repo" : ""));
+        const preflightStep = "preflight";
         const preflightStartedAt = Date.now();
-        const preflight = await preflightTargetRepo({
-            repoRoot: options.repoRoot,
-            client,
-            ...(typeof options.preflight === "object" ? options.preflight : {}),
+        await runObservedStep(eventCtx, preflightStep, async () => {
+            progress.step("preflight: version + inspect + docs-audit + git-status" + (options.preflight && typeof options.preflight === "object" && options.preflight.runCheckRepo ? " + check-repo" : ""));
+            const preflight = await preflightTargetRepo({
+                repoRoot: options.repoRoot,
+                client,
+                ...(typeof options.preflight === "object" ? options.preflight : {}),
+            });
+            if (!preflight.ok) {
+                progress.note(`preflight FAILED: ${preflight.code}`);
+                throw new Error(`target repo preflight failed: ${preflight.code}: ${preflight.message}`);
+            }
         });
-        if (!preflight.ok) {
-            progress.note(`preflight FAILED: ${preflight.code}`);
-            throw new Error(`target repo preflight failed: ${preflight.code}: ${preflight.message}`);
-        }
         progress.step(`preflight ok in ${formatDuration(Date.now() - preflightStartedAt)}`);
     }
-    progress.step("materialize harness task");
+    const materializeStep = "materialize";
     const materializeStartedAt = Date.now();
-    const materializeManifest = await materializeTaskSpec({
-        repoRoot: options.repoRoot,
-        taskSpec: options.taskSpec,
-        taskSpecPath: options.taskSpecPath,
-        client,
-        now,
+    const materializeManifest = await runObservedStep(eventCtx, materializeStep, async () => {
+        progress.step("materialize harness task");
+        return materializeTaskSpec({
+            repoRoot: options.repoRoot,
+            taskSpec: options.taskSpec,
+            taskSpecPath: options.taskSpecPath,
+            client,
+            now,
+        });
     });
     progress.step(`materialized ${materializeManifest.harnessTaskId} in ${formatDuration(Date.now() - materializeStartedAt)}`);
     const taskArtifactsDir = path.join(options.repoRoot, ".harness", "tasks", materializeManifest.harnessTaskId, "artifacts");
     await mkdir(taskArtifactsDir, { recursive: true });
     const dagPath = path.join(taskArtifactsDir, `${workerRunId}-dag.json`);
     const runRecordPath = path.join(taskArtifactsDir, "worker-run-record.json");
-    progress.step("dag run-task: generate DAG");
+    const dagGenStep = "dag-run-task";
     const dagGenStartedAt = Date.now();
-    await runRequiredCommand(options.repoRoot, client, "dag-run-task", [
-        "dag",
-        "run-task",
-        materializeManifest.harnessTaskId,
-        "--profile",
-        materializeManifest.loopAgentProfile,
-        "--strict-models",
-        "--output",
-        dagPath,
-        "--cwd",
-        options.repoRoot,
-        ...noCursorArgs(options.taskSpec),
-    ]);
+    await runObservedStep(eventCtx, dagGenStep, async () => {
+        progress.step("dag run-task: generate DAG");
+        await runRequiredCommand(options.repoRoot, client, "dag-run-task", [
+            "dag",
+            "run-task",
+            materializeManifest.harnessTaskId,
+            "--profile",
+            materializeManifest.loopAgentProfile,
+            "--strict-models",
+            "--output",
+            dagPath,
+            "--cwd",
+            options.repoRoot,
+            ...noCursorArgs(options.taskSpec),
+        ], true, undefined, eventCtx);
+    });
     progress.step(`dag generated in ${formatDuration(Date.now() - dagGenStartedAt)}`);
     if (options.piModel) {
-        progress.step(`pi-model override: rewriting executorModels.pi → ${options.piModel}`);
-        await applyPiModelOverride(dagPath, options.piModel);
+        const piModelStep = "pi-model-override";
+        await runObservedStep(eventCtx, piModelStep, async () => {
+            progress.step(`pi-model override: rewriting executorModels.pi → ${options.piModel}`);
+            await applyPiModelOverride(dagPath, options.piModel);
+        });
     }
-    progress.step("dag validate");
+    const validateStep = "dag-validate";
     const validateStartedAt = Date.now();
-    await runRequiredCommand(options.repoRoot, client, "dag-validate", [
-        "dag",
-        "validate",
-        "--dag",
-        dagPath,
-        ...strictModelsArgs(options.piModel),
-        "--strict-governance",
-        "--spine-task",
-        materializeManifest.harnessTaskId,
-        ...forbidCursorArgs(options.taskSpec),
-    ]);
+    await runObservedStep(eventCtx, validateStep, async () => {
+        progress.step("dag validate");
+        await runRequiredCommand(options.repoRoot, client, "dag-validate", [
+            "dag",
+            "validate",
+            "--dag",
+            dagPath,
+            ...strictModelsArgs(options.piModel),
+            "--strict-governance",
+            "--spine-task",
+            materializeManifest.harnessTaskId,
+            ...forbidCursorArgs(options.taskSpec),
+        ], true, undefined, eventCtx);
+    });
     progress.step(`dag validated in ${formatDuration(Date.now() - validateStartedAt)}`);
-    progress.step("run-dag: executing DAG nodes (this is the long step)");
+    const runDagStep = "run-dag";
     const runDagStartedAt = Date.now();
-    await runRequiredCommand(options.repoRoot, client, "run-dag", [
-        "run-dag",
-        "--dag",
-        dagPath,
-        "--cwd",
-        options.repoRoot,
-        "--run-id",
-        workerRunId,
-        ...maxConcurrentArgs(options.taskSpec),
-        ...noCursorArgs(options.taskSpec),
-    ], true, resolveRunDagTimeoutMs(options.taskSpec));
-    progress.step(`run-dag finished in ${formatDuration(Date.now() - runDagStartedAt)}`);
-    const reportJson = await client.run([
-        "dag",
-        "report",
-        "--run-id",
-        workerRunId,
-        "--lifecycle",
-        "all",
-        "--json",
-    ], {
-        cwd: options.repoRoot,
-        artifactName: "dag-report-json",
-        expectJson: true,
-    });
-    const reportDecision = decideFromReport(workerRunId, reportJson);
-    await client.run([
-        "dag",
-        "report",
-        "--run-id",
-        workerRunId,
-        "--lifecycle",
-        "all",
-        "--markdown",
-    ], {
-        cwd: options.repoRoot,
-        artifactName: "dag-report-markdown",
+    const dagEventsPath = path.join(options.repoRoot, ".task-pool", "observability", "runs", workerRunId, "dag-events.jsonl");
+    await mkdir(path.dirname(dagEventsPath), { recursive: true }).catch(() => { });
+    await runObservedStep(eventCtx, runDagStep, async () => {
+        progress.step("run-dag: executing DAG nodes (this is the long step)");
+        await runDagWithEventsFallback(options.repoRoot, client, [
+            "run-dag",
+            "--dag",
+            dagPath,
+            "--cwd",
+            options.repoRoot,
+            "--run-id",
+            workerRunId,
+            "--events-jsonl",
+            dagEventsPath,
+            ...maxConcurrentArgs(options.taskSpec),
+            ...noCursorArgs(options.taskSpec),
+        ], resolveRunDagTimeoutMs(options.taskSpec), eventCtx);
     });
+    progress.step(`run-dag finished in ${formatDuration(Date.now() - runDagStartedAt)}`);
+    const reportStep = "report-decision";
+    const reportDecision = await runObservedStep(eventCtx, reportStep, async () => {
+        const reportJson = await client.run([
+            "dag",
+            "report",
+            "--run-id",
+            workerRunId,
+            "--lifecycle",
+            "all",
+            "--json",
+        ], {
+            cwd: options.repoRoot,
+            artifactName: "dag-report-json",
+            expectJson: true,
+        });
+        const decision = decideFromReport(workerRunId, reportJson);
+        await client.run([
+            "dag",
+            "report",
+            "--run-id",
+            workerRunId,
+            "--lifecycle",
+            "all",
+            "--markdown",
+        ], {
+            cwd: options.repoRoot,
+            artifactName: "dag-report-markdown",
+        });
+        return decision;
+    }, { statusForResult: (decision) => decision.succeeded ? "succeeded" : "failed" });
     const status = reportDecision.succeeded ? "succeeded" : "failed";
     progress.step(`report decision: ${status} (${reportDecision.reason}${reportDecision.runStatus ? `, status=${reportDecision.runStatus}` : ""})`);
     const failureArtifacts = reportDecision.succeeded
@@ -123,11 +153,17 @@ export async function runTaskSpec(options) {
             repoRoot: options.repoRoot,
             workerRunId,
             taskArtifactsDir,
+            eventCtx,
         });
     if (reportDecision.succeeded) {
-        progress.step("promote + closeout");
-        await runRequiredCommand(options.repoRoot, client, "promote-run", ["promote-run", materializeManifest.harnessTaskId, "--run-id", workerRunId], true);
-        await runRequiredCommand(options.repoRoot, client, "closeout-task", ["closeout", "task", materializeManifest.harnessTaskId], true);
+        await runObservedStep(eventCtx, "promote-run", async () => {
+            progress.step("promote run");
+            await runRequiredCommand(options.repoRoot, client, "promote-run", ["promote-run", materializeManifest.harnessTaskId, "--run-id", workerRunId], true, undefined, eventCtx);
+        });
+        await runObservedStep(eventCtx, "closeout-task", async () => {
+            progress.step("closeout task");
+            await runRequiredCommand(options.repoRoot, client, "closeout-task", ["closeout", "task", materializeManifest.harnessTaskId], true, undefined, eventCtx);
+        });
     }
     const record = {
         schemaVersion: 1,
@@ -145,6 +181,24 @@ export async function runTaskSpec(options) {
         ...(failureArtifacts ? { failureArtifacts } : {}),
     };
     await writeFile(runRecordPath, `${JSON.stringify(record, null, 2)}\n`, "utf-8");
+    emit(progress, {
+        type: "artifact.written",
+        source: "artifact",
+        label: "worker-run-record",
+        workerRunId,
+        taskId,
+        artifactRefs: {
+            runRecordPath,
+            dagPath,
+            ...(failureArtifacts
+                ? {
+                    reportMarkdown: failureArtifacts.reportMarkdownArtifactPath,
+                    doctorMarkdown: failureArtifacts.doctorMarkdownArtifactPath,
+                    closeoutDraft: failureArtifacts.closeoutDraftPath,
+                }
+                : {}),
+        },
+    });
     return {
         status,
         workerRunId,
@@ -164,7 +218,7 @@ export function buildWorkerRunId(businessId, now) {
         .slice(0, 6);
     return `wr-${date}-${businessId}-${hash}`;
 }
-async function runRequiredCommand(repoRoot, client, artifactName, args, expectJson = true, timeoutMs) {
+async function runRequiredCommand(repoRoot, client, artifactName, args, expectJson = true, timeoutMs, eventCtx) {
     const result = await client.run(args, {
         cwd: repoRoot,
         artifactName,
@@ -176,16 +230,99 @@ async function runRequiredCommand(repoRoot, client, artifactName, args, expectJs
     }
     return result;
 }
+/**
+ * `--events-jsonl` is additive observability. Older published controllers do
+ * not know the flag; their parser rejects before any DAG node can execute, so
+ * retrying the exact command without it is safe and preserves the same run id.
+ */
+async function runDagWithEventsFallback(repoRoot, client, args, timeoutMs, eventCtx) {
+    const result = await client.run(args, {
+        cwd: repoRoot,
+        artifactName: "run-dag",
+        expectJson: true,
+        timeoutMs,
+    });
+    if (result.ok)
+        return result;
+    if (!rejectsEventsJsonl(result) || !args.includes("--events-jsonl")) {
+        throw new Error(`loop-agent command failed: ${args.join(" ")}`);
+    }
+    const fallbackArgs = withoutFlagAndValue(args, "--events-jsonl");
+    eventCtx.progress.note("run-dag controller does not support --events-jsonl; retrying without DAG event stream");
+    const fallback = await client.run(fallbackArgs, {
+        cwd: repoRoot,
+        artifactName: "run-dag-events-fallback",
+        expectJson: true,
+        timeoutMs,
+    });
+    if (!fallback.ok) {
+        throw new Error(`loop-agent command failed: ${fallbackArgs.join(" ")}`);
+    }
+    return fallback;
+}
+function rejectsEventsJsonl(result) {
+    return /unknown run-dag flag:\s*--events-jsonl/i.test(`${result.stderr}\n${result.stdout}`);
+}
+function withoutFlagAndValue(args, flag) {
+    const index = args.indexOf(flag);
+    if (index < 0)
+        return args;
+    return [...args.slice(0, index), ...args.slice(index + 2)];
+}
+function emit(progress, input) {
+    const structured = progress;
+    if (typeof structured.event !== "function")
+        return;
+    void structured.event(input).catch(() => { });
+}
+function emitStepStarted(ctx, stepName) {
+    emit(ctx.progress, {
+        type: "step.started",
+        source: "worker",
+        label: stepName,
+        spanId: `step:${ctx.workerRunId}:${stepName}`,
+        parentSpanId: ctx.workerRunId,
+        workerRunId: ctx.workerRunId,
+        taskId: ctx.taskId,
+        status: "running",
+    });
+}
+function emitStepFinished(ctx, stepName, status, durationMs) {
+    emit(ctx.progress, {
+        type: "step.finished",
+        source: "worker",
+        label: stepName,
+        spanId: `step:${ctx.workerRunId}:${stepName}`,
+        parentSpanId: ctx.workerRunId,
+        workerRunId: ctx.workerRunId,
+        taskId: ctx.taskId,
+        status,
+        durationMs,
+    });
+}
+async function runObservedStep(ctx, stepName, run, options) {
+    const startedAt = Date.now();
+    emitStepStarted(ctx, stepName);
+    try {
+        const result = await run();
+        emitStepFinished(ctx, stepName, options?.statusForResult?.(result) ?? "succeeded", Date.now() - startedAt);
+        return result;
+    }
+    catch (error) {
+        emitStepFinished(ctx, stepName, "failed", Date.now() - startedAt);
+        throw error;
+    }
+}
 function resolveRunDagTimeoutMs(taskSpec) {
     return Math.min(taskSpec.worker.timeout_ms ?? DEFAULT_RUN_DAG_TIMEOUT_MS, MAX_WORKER_TIMEOUT_MS);
 }
 async function collectFailureArtifacts(input) {
-    const doctor = await input.client.run(["dag", "doctor", "--run-id", input.workerRunId, "--markdown"], {
+    const doctor = await runObservedStep(input.eventCtx, "dag-doctor", () => input.client.run(["dag", "doctor", "--run-id", input.workerRunId, "--markdown"], {
         cwd: input.repoRoot,
         artifactName: "dag-doctor-markdown",
-    });
+    }), { statusForResult: (result) => (result.ok ? "succeeded" : "failed") });
     const closeoutDraftPath = path.join(input.repoRoot, ".task-pool", "failure-handoffs", `${input.workerRunId}-failure-closeout-draft.md`);
-    await input.client.run([
+    await runObservedStep(input.eventCtx, "closeout-draft", () => input.client.run([
         "dag",
         "closeout-draft",
         "--run-id",
@@ -195,8 +332,9 @@ async function collectFailureArtifacts(input) {
     ], {
         cwd: input.repoRoot,
         artifactName: "dag-closeout-draft",
-    });
-    const reportMarkdown = input.client instanceof RecordingRunTaskClient
+    }), { statusForResult: (result) => (result.ok ? "succeeded" : "failed") });
+    const reportMarkdown = input.client instanceof ObservedRunTaskClient ||
+        input.client instanceof RecordingRunTaskClient
         ? input.client.findArtifact("dag-report-markdown")
         : undefined;
     return {
@@ -298,6 +436,100 @@ class RecordingRunTaskClient {
         return this.records.find((record) => record.artifactName === artifactName);
     }
 }
+class ObservedRunTaskClient {
+    delegate;
+    eventCtx;
+    constructor(delegate, eventCtx) {
+        this.delegate = delegate;
+        this.eventCtx = eventCtx;
+    }
+    run(args, options) {
+        return this.runObserved(args, options, (observedOptions) => this.delegate.run(args, observedOptions));
+    }
+    runExternal(command, args, options) {
+        return this.runObserved([command, ...args], options, (observedOptions) => this.delegate.runExternal(command, args, observedOptions));
+    }
+    findArtifact(artifactName) {
+        return this.delegate.findArtifact(artifactName);
+    }
+    async runObserved(command, options, run) {
+        const observedOptions = {
+            ...options,
+            onSpawn: (info) => {
+                options.onSpawn?.(info);
+                emit(this.eventCtx.progress, {
+                    type: "command.started",
+                    source: "loop-agent-command",
+                    label: options.artifactName,
+                    workerRunId: this.eventCtx.workerRunId,
+                    taskId: this.eventCtx.taskId,
+                    pid: info.pid,
+                    timeoutMs: options.timeoutMs,
+                    command,
+                    artifactRefs: info.artifactRefs,
+                    status: "running",
+                });
+            },
+            onStdout: (chunk) => {
+                options.onStdout?.(chunk);
+                this.emitOutput(options.artifactName, chunk);
+            },
+            onStderr: (chunk) => {
+                options.onStderr?.(chunk);
+                this.emitOutput(options.artifactName, chunk);
+            },
+            onHeartbeat: (info) => {
+                options.onHeartbeat?.(info);
+                emit(this.eventCtx.progress, {
+                    type: "command.heartbeat",
+                    source: "loop-agent-command",
+                    label: options.artifactName,
+                    workerRunId: this.eventCtx.workerRunId,
+                    taskId: this.eventCtx.taskId,
+                    message: `elapsed ${info.elapsedMs}ms`,
+                });
+            },
+        };
+        try {
+            const result = await run(observedOptions);
+            emit(this.eventCtx.progress, {
+                type: "command.finished",
+                source: "loop-agent-command",
+                label: options.artifactName,
+                workerRunId: this.eventCtx.workerRunId,
+                taskId: this.eventCtx.taskId,
+                exitCode: result.exitCode,
+                timedOut: result.timedOut,
+                durationMs: result.durationMs,
+                artifactRefs: result.artifacts,
+                status: result.ok ? "succeeded" : "failed",
+            });
+            return result;
+        }
+        catch (error) {
+            emit(this.eventCtx.progress, {
+                type: "command.finished",
+                source: "loop-agent-command",
+                label: options.artifactName,
+                workerRunId: this.eventCtx.workerRunId,
+                taskId: this.eventCtx.taskId,
+                status: "failed",
+                message: errorMessage(error),
+            });
+            throw error;
+        }
+    }
+    emitOutput(label, chunk) {
+        emit(this.eventCtx.progress, {
+            type: "command.output",
+            source: "loop-agent-command",
+            label,
+            workerRunId: this.eventCtx.workerRunId,
+            taskId: this.eventCtx.taskId,
+            outputPreview: redactForPreview(truncatePreview(chunk)),
+        });
+    }
+}
 function commandRecordName(args, artifactName) {
     if (args[0] === "new-task")
         return "new-task";
@@ -357,3 +589,6 @@ function readProperty(value, key) {
         return undefined;
     return value[key];
 }
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}