@engineeros/connector 0.14.0 → 0.14.4

package/src/runner.mjs

@@ -28,10 +28,13 @@ const MAX_ARCHIVE_BYTES = 25_000_000;
 const MAX_EVIDENCE_BYTES = 24_000_000;
 const MAX_SHAREABLE_FILE_BYTES = 5 * 1024 * 1024;
 const MAX_EVIDENCE_FILES = 5_000;
-const MAX_INVENTORY_FILES = 100_000;
-const MAX_COMPRESSED_INVENTORY_BYTES = 10_000_000;
-const EVIDENCE_POLICY_VERSION = "workspace-evidence-v1";
-export const ASSESSMENT_INACTIVITY_TIMEOUT_MS = 10 * 60 * 1_000;
+const MAX_INVENTORY_FILES = 100_000;
+const MAX_COMPRESSED_INVENTORY_BYTES = 10_000_000;
+const EVIDENCE_POLICY_VERSION = "workspace-evidence-v1";
+export const ASSESSMENT_INACTIVITY_TIMEOUT_MS = 10 * 60 * 1_000;
+export const ASSESSMENT_OUTPUT_FLUSH_INTERVAL_MS = 15_000;
+export const ASSESSMENT_RESULT_TIMEOUT_MS = 120_000;
+export const MAX_ASSESSMENT_WORKERS = 3;
 const EXCLUDED_DIRECTORIES = new Set([
   ".agents",
   ".claude",
@@ -395,94 +398,94 @@ export async function executeWorkspaceAssessment(
     execution.profile,
   );
   callbacks.onProcess?.(controller.child);
-  let completed = await controller.completed;
-  const recovered = await recoverAssessmentStageOutput({
-    completed,
-    prompt,
-    requiredOutputHeading: execution.requiredOutputHeading,
-    retry: async (correctionPrompt, previousSessionId) => {
-      callbacks.onEvent?.({
-        type: "assessment.output_correction",
-        message: "The Agent is completing the required stage report structure",
-      });
-      const correction = launchAgentProcess(
-        config.workspace,
-        correctionPrompt,
-        execution.sandboxMode,
-        config,
-        callbacks,
-        execution.profile,
-        previousSessionId,
-      );
-      callbacks.onProcess?.(correction.child);
-      return correction.completed;
-    },
-  });
-  completed = recovered.completed;
-  const buildResult = async (turn, report) => {
-    const endingRevision = await run(
-      "git",
-      ["rev-parse", "HEAD"],
-      config.workspace,
-      { allowFailure: true },
-    );
-    const endingHead =
-      endingRevision.code === 0
-        ? endingRevision.stdout.trim().slice(0, 128)
-        : null;
-    if (startingHead !== endingHead) {
-      throw new Error(
-        "The Git commit changed during assessment. Refresh the workspace inventory and assess the new commit.",
-      );
-    }
-    return {
-      stage: assignment.stage,
-      report_markdown: report,
-      observed_head_revision: endingHead,
-      changed_files: changeImpact.changedFiles,
-      change_impact_markdown: changeImpact.markdown,
-      model: turn.model ?? config.agent_protocol ?? "coding-agent",
-      usage: turn.usage ?? null,
-    };
-  };
-  const result = await buildResult(completed, recovered.report);
-  if (!recovered.correctionUsed) {
-    Object.defineProperty(result, "correctAfterRejection", {
-      enumerable: false,
-      value: async (validationMessage) => {
-        callbacks.onEvent?.({
-          type: "assessment.output_correction",
-          message: "The Agent is correcting the rejected stage report",
-        });
-        const correction = launchAgentProcess(
-          config.workspace,
-          assessmentRejectionCorrectionPrompt(
-            validationMessage,
-            execution.requiredOutputHeading,
-          ),
-          execution.sandboxMode,
-          config,
-          callbacks,
-          execution.profile,
-          completed.sessionId,
-        );
-        callbacks.onProcess?.(correction.child);
-        const corrected = await correction.completed;
-        const correctedReport = normalizeAgentStructuredOutput(
-          corrected.finalMessage,
-          execution.requiredOutputHeading,
-        );
-        return buildResult(
-          {
-            ...corrected,
-            usage: mergeTokenUsage(completed.usage, corrected.usage),
-          },
-          correctedReport,
-        );
-      },
-    });
-  }
-  return result;
+  let completed = await controller.completed;
+  const recovered = await recoverAssessmentStageOutput({
+    completed,
+    prompt,
+    requiredOutputHeading: execution.requiredOutputHeading,
+    retry: async (correctionPrompt, previousSessionId) => {
+      callbacks.onEvent?.({
+        type: "assessment.output_correction",
+        message: "The Agent is completing the required stage report structure",
+      });
+      const correction = launchAgentProcess(
+        config.workspace,
+        correctionPrompt,
+        execution.sandboxMode,
+        config,
+        callbacks,
+        execution.profile,
+        previousSessionId,
+      );
+      callbacks.onProcess?.(correction.child);
+      return correction.completed;
+    },
+  });
+  completed = recovered.completed;
+  const buildResult = async (turn, report) => {
+    const endingRevision = await run(
+      "git",
+      ["rev-parse", "HEAD"],
+      config.workspace,
+      { allowFailure: true },
+    );
+    const endingHead =
+      endingRevision.code === 0
+        ? endingRevision.stdout.trim().slice(0, 128)
+        : null;
+    if (startingHead !== endingHead) {
+      throw new Error(
+        "The Git commit changed during assessment. Refresh the workspace inventory and assess the new commit.",
+      );
+    }
+    return {
+      stage: assignment.stage,
+      report_markdown: report,
+      observed_head_revision: endingHead,
+      changed_files: changeImpact.changedFiles,
+      change_impact_markdown: changeImpact.markdown,
+      model: turn.model ?? config.agent_protocol ?? "coding-agent",
+      usage: turn.usage ?? null,
+    };
+  };
+  const result = await buildResult(completed, recovered.report);
+  if (!recovered.correctionUsed) {
+    Object.defineProperty(result, "correctAfterRejection", {
+      enumerable: false,
+      value: async (validationMessage) => {
+        callbacks.onEvent?.({
+          type: "assessment.output_correction",
+          message: "The Agent is correcting the rejected stage report",
+        });
+        const correction = launchAgentProcess(
+          config.workspace,
+          assessmentRejectionCorrectionPrompt(
+            validationMessage,
+            execution.requiredOutputHeading,
+          ),
+          execution.sandboxMode,
+          config,
+          callbacks,
+          execution.profile,
+          completed.sessionId,
+        );
+        callbacks.onProcess?.(correction.child);
+        const corrected = await correction.completed;
+        const correctedReport = normalizeAgentStructuredOutput(
+          corrected.finalMessage,
+          execution.requiredOutputHeading,
+        );
+        return buildResult(
+          {
+            ...corrected,
+            usage: mergeTokenUsage(completed.usage, corrected.usage),
+          },
+          correctedReport,
+        );
+      },
+    });
+  }
+  return result;
 }
 
 export async function workspaceChangeImpact(
@@ -695,23 +698,23 @@ export function codexFailureMessage(output, code) {
   return `Codex exited with code ${code}. ${output.slice(-1_000)}`;
 }
 
-export function assessmentProgressMessage(event) {
-  if (!event || typeof event !== "object") return null;
-  if (event.type === "assessment.output_correction")
-    return "Completing the required stage report structure";
-  if (event.type === "agent.connected")
-    return "Connected agent is ready to inspect repository evidence";
-  if (event.type === "acp.plan")
-    return "Organizing the repository assessment plan";
-  if (event.type === "acp.agent_thought_chunk")
-    return "Reasoning through repository evidence";
-  if (event.type === "acp.agent_message_chunk")
-    return "Drafting the evidence-backed stage report";
-  if (event.type === "acp.tool_call" || event.type === "acp.tool_call_update") {
-    return assessmentCommandMilestone(event.update?.title);
-  }
-  if (event.type === "turn.started")
-    return "Reviewing repository structure and current Git state";
+export function assessmentProgressMessage(event) {
+  if (!event || typeof event !== "object") return null;
+  if (event.type === "assessment.output_correction")
+    return "Completing the required stage report structure";
+  if (event.type === "agent.connected")
+    return "Connected agent is ready to inspect the workspace";
+  if (event.type === "acp.plan")
+    return "Organizing the repository assessment plan";
+  if (event.type === "acp.agent_thought_chunk")
+    return "Reasoning through the current implementation";
+  if (event.type === "acp.agent_message_chunk")
+    return "Drafting the stage report";
+  if (event.type === "acp.tool_call" || event.type === "acp.tool_call_update") {
+    return assessmentCommandMilestone(event.update?.title);
+  }
+  if (event.type === "turn.started")
+    return "Reviewing repository structure and current Git state";
   if (event.type === "item.started") {
     if (event.item?.type === "command_execution") {
       return assessmentCommandMilestone(event.item.command);
@@ -725,16 +728,16 @@ export function assessmentProgressMessage(event) {
   if (event.type === "item.completed" && event.item?.type === "agent_message") {
     return "Synthesizing findings and highest-return actions";
   }
-  return null;
-}
-
-export function assessmentInactivityFailure(stage, inactiveMs) {
-  if (inactiveMs < ASSESSMENT_INACTIVITY_TIMEOUT_MS) return null;
-  return (
-    `The connected agent produced no activity for 10 minutes during ${stage}. ` +
-    "The stage was stopped instead of waiting indefinitely. Retry it after checking the agent terminal."
-  );
-}
+  return null;
+}
+
+export function assessmentInactivityFailure(stage, inactiveMs) {
+  if (inactiveMs < ASSESSMENT_INACTIVITY_TIMEOUT_MS) return null;
+  return (
+    `The connected agent produced no activity for 10 minutes during ${stage}. ` +
+    "The stage was stopped instead of waiting indefinitely. Retry it after checking the agent terminal."
+  );
+}
 
 export function promptProgressMessage(event) {
   if (!event || typeof event !== "object") return null;
@@ -824,7 +827,7 @@ function assessmentCommandMilestone(command) {
     ? command.join(" ")
     : String(command || "");
   const normalized = value.replace(/\s+/g, " ").trim().toLowerCase();
-  if (!normalized) return "Inspecting repository evidence";
+  if (!normalized) return "Inspecting workspace source";
   if (/\bgit\s+(status|log|diff|show|rev-parse)\b/.test(normalized)) {
     return "Comparing Git history and workspace changes";
   }
@@ -843,190 +846,257 @@ function assessmentCommandMilestone(command) {
   return "Tracing architecture and code relationships";
 }
 
-export function workspaceAssessmentExecution(assignment) {
-  const stage = assignment?.stage;
-  const requiredOutputHeading = String(
-    assignment?.required_output_heading || "",
-  ).trim();
-  if (!stage || !requiredOutputHeading.startsWith("# Assessment Stage: ")) {
-    throw new Error("EngineerOS assessment assignment has an unsupported stage.");
-  }
+export function workspaceAssessmentExecution(assignment) {
+  const stage = assignment?.stage;
+  const requiredOutputHeading = String(
+    assignment?.required_output_heading || "",
+  ).trim();
+  if (!stage || !requiredOutputHeading.startsWith("# Assessment Stage: ")) {
+    throw new Error("EngineerOS assessment assignment has an unsupported stage.");
+  }
   return {
     ...connectorExecution(assignment, { requiredOutputHeading }),
     requiredOutputHeading,
-  };
-}
-
-export async function recoverAssessmentStageOutput({
-  completed,
-  prompt,
-  requiredOutputHeading,
-  retry,
-}) {
-  try {
-    return {
-      completed,
-      correctionUsed: false,
-      report: normalizeAgentStructuredOutput(
-        completed.finalMessage,
-        requiredOutputHeading,
-      ),
-    };
-  } catch (error) {
-    if (!recoverableStructuredOutputFailure(error)) throw error;
-  }
-
-  const corrected = await retry(
-    assessmentStageCorrectionPrompt(prompt, requiredOutputHeading),
-    completed.sessionId,
-  );
-  try {
-    return {
-      completed: {
-        ...corrected,
-        usage: mergeTokenUsage(completed.usage, corrected.usage),
-      },
-      correctionUsed: true,
-      report: normalizeAgentStructuredOutput(
-        corrected.finalMessage,
-        requiredOutputHeading,
-      ),
-    };
-  } catch (error) {
-    throw new Error(
-      `Agent did not return the required stage report after one automatic correction attempt. ${error.message}`,
-      { cause: error },
-    );
-  }
-}
-
-export function assessmentRejectionCorrectionPrompt(
-  validationMessage,
-  requiredOutputHeading,
-) {
-  return [
-    "## Rejected Stage Output Recovery",
-    "",
-    "EngineerOS rejected the previous stage report because it was incomplete or violated the structured Markdown contract:",
-    "",
-    String(validationMessage || "The stage report was rejected.").trim(),
-    "",
-    "Treat the previous stage report as the authoritative draft.",
-    "Copy every valid section and block unchanged; repair only the incomplete or invalid entries identified by EngineerOS validation.",
-    "Do not re-inspect the repository or replace valid evidence unless the validation error requires it.",
-    "Return the entire corrected structured Markdown stage report so EngineerOS can validate it atomically, not only the repaired fragment or missing tail.",
-    `The first non-whitespace line must be exactly: ${requiredOutputHeading}`,
-    "Do not return progress commentary, an explanation of the correction, or a code fence.",
-  ].join("\n");
-}
-
-export function assessmentStageCorrectionPrompt(
-  originalPrompt,
-  requiredOutputHeading,
-) {
-  return [
-    String(originalPrompt || "").trim(),
-    "",
-    "## Incomplete Stage Output Recovery",
-    "",
-    "The previous turn ended without a complete stage deliverable. Return the entire structured Markdown stage report now.",
-    "Use repository evidence already inspected in the previous turn when it is available; inspect only what remains necessary.",
-    `The first non-whitespace line of the final answer must be exactly: ${requiredOutputHeading}`,
-    "Follow every section, evidence, and completeness rule in the Assignment.",
-    "Do not return progress commentary, an explanation of the correction, or a code fence.",
-  ].join("\n");
-}
-
-function recoverableStructuredOutputFailure(error) {
-  const message = error instanceof Error ? error.message : "";
-  return (
-    message.startsWith("Agent completed") ||
-    message.startsWith("Agent response")
-  );
-}
-
-function mergeTokenUsage(first, second) {
-  const usages = [first, second].filter(
-    (usage) => usage && typeof usage === "object",
-  );
-  if (!usages.length) return null;
-  return Object.fromEntries(
-    [
-      "input_tokens",
-      "output_tokens",
-      "cache_read_tokens",
-      "cache_write_tokens",
-      "reasoning_tokens",
-      "total_tokens",
-    ].map((field) => [
-      field,
-      usages.reduce(
-        (total, usage) =>
-          total + (Number.isFinite(usage[field]) ? usage[field] : 0),
-        0,
-      ),
-    ]),
-  );
-}
-
-export function enqueueWorkspaceAssessment(
-  queue,
-  activeAssessment,
-  assignment,
-  { front = false } = {},
-) {
-  const matches = (candidate) =>
-    candidate?.assessment_id === assignment?.assessment_id &&
-    candidate?.stage === assignment?.stage;
-  if (
-    activeAssessment?.kind === "assessment" &&
-    activeAssessment.runId === assignment?.assessment_id &&
-    activeAssessment.stage === assignment?.stage
-  ) {
-    activeAssessment.resumeAssignment = assignment;
-    return "deferred";
-  }
-  if (queue.some(matches)) return "duplicate";
-  if (front) queue.unshift(assignment);
-  else queue.push(assignment);
-  return "queued";
-}
-
-export function takeWorkspaceAssessmentWave(queue, activeCount, limit = 3) {
-  const available = Math.max(0, limit - activeCount);
-  if (!available || !queue.length) return [];
-  if (queue[0]?.parallelizable !== true) {
-    return activeCount === 0 ? queue.splice(0, 1) : [];
-  }
-  const wave = [];
-  while (
-    wave.length < available &&
-    queue[0]?.parallelizable === true
-  ) {
-    wave.push(queue.shift());
-  }
-  return wave;
-}
-
-export function requeueInterruptedAssessment(
-  queue,
-  assessmentState,
-  { accepted, failureReported },
-) {
-  if (accepted || failureReported || !assessmentState?.resumeAssignment) {
-    return false;
-  }
-  return (
-    enqueueWorkspaceAssessment(
-      queue,
-      null,
-      assessmentState.resumeAssignment,
-      { front: true },
-    ) === "queued"
-  );
-}
-
-export function connectorExecution(assignment, options = {}) {
+  };
+}
+
+export async function recoverAssessmentStageOutput({
+  completed,
+  prompt,
+  requiredOutputHeading,
+  retry,
+}) {
+  try {
+    return {
+      completed,
+      correctionUsed: false,
+      report: normalizeAgentStructuredOutput(
+        completed.finalMessage,
+        requiredOutputHeading,
+      ),
+    };
+  } catch (error) {
+    if (!recoverableStructuredOutputFailure(error)) throw error;
+  }
+
+  const corrected = await retry(
+    assessmentStageCorrectionPrompt(prompt, requiredOutputHeading),
+    completed.sessionId,
+  );
+  try {
+    return {
+      completed: {
+        ...corrected,
+        usage: mergeTokenUsage(completed.usage, corrected.usage),
+      },
+      correctionUsed: true,
+      report: normalizeAgentStructuredOutput(
+        corrected.finalMessage,
+        requiredOutputHeading,
+      ),
+    };
+  } catch (error) {
+    throw new Error(
+      `Agent did not return the required stage report after one automatic correction attempt. ${error.message}`,
+      { cause: error },
+    );
+  }
+}
+
+export function assessmentRejectionCorrectionPrompt(
+  validationMessage,
+  requiredOutputHeading,
+) {
+  return [
+    "## Rejected Stage Output Recovery",
+    "",
+    "EngineerOS rejected the previous stage report because it was incomplete or violated the structured Markdown contract:",
+    "",
+    String(validationMessage || "The stage report was rejected.").trim(),
+    "",
+    "Treat the previous stage report as the authoritative draft.",
+    "Copy every valid section and block unchanged; repair only the incomplete or invalid entries identified by EngineerOS validation.",
+    "Do not re-inspect the repository or replace valid evidence unless the validation error requires it.",
+    "Return the entire corrected structured Markdown stage report so EngineerOS can validate it atomically, not only the repaired fragment or missing tail.",
+    `The first non-whitespace line must be exactly: ${requiredOutputHeading}`,
+    "Do not return progress commentary, an explanation of the correction, or a code fence.",
+  ].join("\n");
+}
+
+export function assessmentStageCorrectionPrompt(
+  originalPrompt,
+  requiredOutputHeading,
+) {
+  return [
+    String(originalPrompt || "").trim(),
+    "",
+    "## Incomplete Stage Output Recovery",
+    "",
+    "The previous turn ended without a complete stage deliverable. Return the entire structured Markdown stage report now.",
+    "Use repository context already inspected in the previous turn when it is available; inspect only what remains necessary.",
+    `The first non-whitespace line of the final answer must be exactly: ${requiredOutputHeading}`,
+    "Follow every section, inspection, and completeness rule in the Assignment.",
+    "Do not return progress commentary, an explanation of the correction, or a code fence.",
+  ].join("\n");
+}
+
+function recoverableStructuredOutputFailure(error) {
+  const message = error instanceof Error ? error.message : "";
+  return (
+    message.startsWith("Agent completed") ||
+    message.startsWith("Agent response")
+  );
+}
+
+function mergeTokenUsage(first, second) {
+  const usages = [first, second].filter(
+    (usage) => usage && typeof usage === "object",
+  );
+  if (!usages.length) return null;
+  return Object.fromEntries(
+    [
+      "input_tokens",
+      "output_tokens",
+      "cache_read_tokens",
+      "cache_write_tokens",
+      "reasoning_tokens",
+      "total_tokens",
+    ].map((field) => [
+      field,
+      usages.reduce(
+        (total, usage) =>
+          total + (Number.isFinite(usage[field]) ? usage[field] : 0),
+        0,
+      ),
+    ]),
+  );
+}
+
+export function enqueueWorkspaceAssessment(
+  queue,
+  activeAssessment,
+  assignment,
+  { front = false } = {},
+) {
+  const matches = (candidate) =>
+    candidate?.assessment_id === assignment?.assessment_id &&
+    candidate?.stage === assignment?.stage;
+  if (
+    activeAssessment?.kind === "assessment" &&
+    activeAssessment.runId === assignment?.assessment_id &&
+    activeAssessment.stage === assignment?.stage
+  ) {
+    activeAssessment.resumeAssignment = assignment;
+    return "deferred";
+  }
+  if (queue.some(matches)) return "duplicate";
+  if (front) queue.unshift(assignment);
+  else queue.push(assignment);
+  return "queued";
+}
+
+export function takeWorkspaceAssessmentWave(queue, activeCount, limit = 3) {
+  const available = Math.max(0, limit - activeCount);
+  if (!available || !queue.length) return [];
+  if (queue[0]?.parallelizable !== true) {
+    return activeCount === 0 ? queue.splice(0, 1) : [];
+  }
+  const wave = [];
+  while (
+    wave.length < available &&
+    queue[0]?.parallelizable === true
+  ) {
+    wave.push(queue.shift());
+  }
+  return wave;
+}
+
+export function workspaceAssessmentWorkerLimit() {
+  return MAX_ASSESSMENT_WORKERS;
+}
+
+export function assessmentWorkerSnapshots(activeAssessments) {
+  return [...activeAssessments.values()]
+    .slice(0, MAX_ASSESSMENT_WORKERS)
+    .map((state) => ({
+      assessment_id: String(state.runId),
+      stage: String(state.stage).slice(0, 96),
+      phase: state.phase,
+      progress_percent: Math.max(
+        0,
+        Math.min(95, Number(state.progressPercent) || 0),
+      ),
+      message: String(state.lastMessage || "Assessment worker is starting").slice(
+        0,
+        500,
+      ),
+      started_at: new Date(state.startedAt).toISOString(),
+      last_activity_at: new Date(state.lastActivityAt).toISOString(),
+      event_count: Math.max(0, Number(state.eventCount) || 0),
+    }));
+}
+
+export function requeueInterruptedAssessment(
+  queue,
+  assessmentState,
+  { accepted, failureReported },
+) {
+  if (accepted || failureReported || !assessmentState?.resumeAssignment) {
+    return false;
+  }
+  return (
+    enqueueWorkspaceAssessment(
+      queue,
+      null,
+      assessmentState.resumeAssignment,
+      { front: true },
+    ) === "queued"
+  );
+}
+
+export async function submitAssessmentResultWithRetry(
+  payload,
+  submit,
+  {
+    isActive = () => true,
+    onRetry = () => {},
+    wait = (milliseconds) =>
+      new Promise((resolve) => setTimeout(resolve, milliseconds)),
+    initialDelayMs = 1_000,
+    maxDelayMs = 30_000,
+  } = {},
+) {
+  let retryDelayMs = initialDelayMs;
+  while (isActive()) {
+    let retryFailure;
+    try {
+      const response = await submit(payload);
+      if (!isTransientAssessmentResultResponse(response)) return response;
+      await response.body?.cancel?.();
+      retryFailure = new Error(
+        `EngineerOS temporarily rejected the completed assessment result (${response.status}).`,
+      );
+    } catch (error) {
+      retryFailure = error;
+    }
+    if (!isActive()) break;
+    onRetry(retryFailure, retryDelayMs);
+    await wait(retryDelayMs);
+    retryDelayMs = Math.min(maxDelayMs, retryDelayMs * 2);
+  }
+  throw new Error("Assessment result delivery stopped before EngineerOS accepted it.");
+}
+
+function isTransientAssessmentResultResponse(response) {
+  return (
+    response.status === 408 ||
+    response.status === 425 ||
+    response.status === 429 ||
+    response.status >= 500
+  );
+}
+
+export function connectorExecution(assignment, options = {}) {
   const rawPrompt = assignment?.prompt_markdown;
   if (typeof rawPrompt !== "string" || !rawPrompt.trim()) {
     throw new Error("EngineerOS assignment is missing prompt_markdown.");
@@ -1076,13 +1146,13 @@ export function connectorExecution(assignment, options = {}) {
   };
 }
 
-export async function stopProcess(child) {
-  if (!child || child.exitCode !== null) return;
-  if (typeof child.engineerOsCancel === "function") {
-    await child.engineerOsCancel();
-    return;
-  }
-  if (process.platform === "win32") {
+export async function stopProcess(child) {
+  if (!child || child.exitCode !== null) return;
+  if (typeof child.engineerOsCancel === "function") {
+    await child.engineerOsCancel();
+    return;
+  }
+  if (process.platform === "win32") {
     await run(
       "taskkill",
       ["/pid", String(child.pid), "/t", "/f"],