@the-open-engine/zeroshot 6.34.0 → 6.34.2

package/task-lib/watcher-output-runtime.js

@@ -1,4 +1,6 @@
 import { spawn } from 'child_process';
+import { createHash } from 'crypto';
+import { StringDecoder } from 'string_decoder';
 import {
   detectProviderFatalError,
   detectProviderStreamingModeError,
@@ -9,6 +11,9 @@ import { terminateProcess } from './process-termination.js';
 
 export const COMMAND_CLEANUP_UNINITIALIZED = Symbol('command-cleanup-uninitialized');
 
+const MAX_CODEX_CONTROL_RECORD_BYTES = 64 * 1024;
+const MAX_WATCHER_CONTROL_RECORD_BYTES = 1024 * 1024;
+
 export function spawnWatcherProvider(command, finalArgs, options) {
   return spawn(command, finalArgs, {
     ...options,
@@ -32,10 +37,162 @@ export async function terminateWatcherProvider(providerProcess, options = {}) {
   return result.terminated;
 }
 
-function splitBufferLines(buffer, chunk) {
-  const nextBuffer = buffer + chunk;
-  const lines = nextBuffer.split('\n');
-  return { lines: lines.slice(0, -1), remaining: lines.at(-1) || '' };
+function createCodexOutputPassthrough({ log, captureProviderSession }) {
+  const decoder = new StringDecoder('utf8');
+  let atLineStart = true;
+  let inspectable = true;
+  let inspectionBytes = 0;
+  let inspectionParts = [];
+
+  function inspectPart(part) {
+    if (!inspectable || !part) return;
+    inspectionBytes += Buffer.byteLength(part);
+    if (inspectionBytes > MAX_CODEX_CONTROL_RECORD_BYTES) {
+      inspectable = false;
+      inspectionParts = [];
+      return;
+    }
+    inspectionParts.push(part);
+  }
+
+  function finishLine() {
+    if (inspectable) captureProviderSession(inspectionParts.join(''));
+    atLineStart = true;
+    inspectable = true;
+    inspectionBytes = 0;
+    inspectionParts = [];
+  }
+
+  function writeText(text, timestamp) {
+    if (!text) return;
+    const logged = [];
+    let offset = 0;
+    while (offset < text.length) {
+      if (atLineStart) {
+        logged.push(`[${timestamp}]`);
+        atLineStart = false;
+      }
+      const newline = text.indexOf('\n', offset);
+      if (newline === -1) {
+        const part = text.slice(offset);
+        inspectPart(part);
+        logged.push(part);
+        break;
+      }
+      const part = text.slice(offset, newline);
+      inspectPart(part);
+      logged.push(part, '\n');
+      finishLine();
+      offset = newline + 1;
+    }
+    log(logged.join(''));
+  }
+
+  return {
+    consume(chunk) {
+      const text = typeof chunk === 'string' ? chunk : decoder.write(chunk);
+      writeText(text, Date.now());
+    },
+    flush() {
+      writeText(decoder.end(), Date.now());
+      if (!atLineStart) {
+        finishLine();
+        log('\n');
+      }
+    },
+  };
+}
+
+function createBoundedLinePassthrough({ log, handleLine, deferRawUntilOverflow = false }) {
+  const decoder = new StringDecoder('utf8');
+  let atLineStart = true;
+  let lineTimestamp = null;
+  let byteLength = 0;
+  let inspectable = true;
+  let inspectionParts = [];
+  let digest = createHash('sha256');
+  let rawOverflowStreaming = false;
+
+  function inspectPart(part) {
+    if (!part) return;
+    byteLength += Buffer.byteLength(part);
+    digest.update(part);
+    if (!inspectable) {
+      if (deferRawUntilOverflow && log) log(part);
+      return;
+    }
+    if (byteLength > MAX_WATCHER_CONTROL_RECORD_BYTES) {
+      if (deferRawUntilOverflow && log) {
+        log(`[${lineTimestamp}]${inspectionParts.join('')}${part}`);
+        rawOverflowStreaming = true;
+      }
+      inspectable = false;
+      inspectionParts = [];
+      return;
+    }
+    inspectionParts.push(part);
+  }
+
+  function finishLine() {
+    const oversized = !inspectable;
+    const line = inspectable
+      ? inspectionParts.join('')
+      : `[ZEROSHOT] Provider output record retained in task log but omitted from watcher inspection ` +
+        `(byte_length=${byteLength}, sha256=${digest.digest('hex')})`;
+    handleLine(line, lineTimestamp || Date.now(), { oversized });
+    atLineStart = true;
+    lineTimestamp = null;
+    byteLength = 0;
+    inspectable = true;
+    inspectionParts = [];
+    digest = createHash('sha256');
+    rawOverflowStreaming = false;
+  }
+
+  function appendRaw(logged, ...parts) {
+    if (log && !deferRawUntilOverflow) logged.push(...parts);
+  }
+
+  function writeText(text) {
+    if (!text) return;
+    const logged = [];
+    let offset = 0;
+    while (offset < text.length) {
+      if (atLineStart) {
+        lineTimestamp = Date.now();
+        appendRaw(logged, `[${lineTimestamp}]`);
+        atLineStart = false;
+      }
+      const newline = text.indexOf('\n', offset);
+      if (newline === -1) {
+        const part = text.slice(offset);
+        inspectPart(part);
+        appendRaw(logged, part);
+        break;
+      }
+      const part = text.slice(offset, newline);
+      inspectPart(part);
+      appendRaw(logged, part, '\n');
+      if (deferRawUntilOverflow && rawOverflowStreaming && log) log('\n');
+      finishLine();
+      offset = newline + 1;
+    }
+    if (log && logged.length > 0) log(logged.join(''));
+  }
+
+  return {
+    consume(chunk) {
+      writeText(typeof chunk === 'string' ? chunk : decoder.write(chunk));
+    },
+    flush() {
+      writeText(decoder.end());
+      if (!atLineStart) {
+        if (deferRawUntilOverflow && rawOverflowStreaming && log) log('\n');
+        finishLine();
+        if (log && !deferRawUntilOverflow) log('\n');
+      }
+    },
+  };
 }
 
 export function resolveWatcherCommand(config, commandSpec, fallbackArgs, normalizeProviderName) {
@@ -189,6 +346,23 @@ export function createWatcherOutputRuntime({
   let streamingModeError = null;
   let fatalError = null;
   const captureProviderSession = providerSessionCapture?.captureLine || (() => {});
+  const codexOutputPassthrough =
+    providerName === 'codex' ? createCodexOutputPassthrough({ log, captureProviderSession }) : null;
+  const outputPassthrough = codexOutputPassthrough
+    ? null
+    : createBoundedLinePassthrough({
+        log,
+        deferRawUntilOverflow: silentJsonMode,
+        handleLine: (line, timestamp, { oversized }) =>
+          handleOutputLine(line, timestamp, {
+            alreadyLogged: !silentJsonMode,
+            oversized,
+          }),
+      });
+  const stderrPassthrough = createBoundedLinePassthrough({
+    log,
+    handleLine: (line, timestamp) => maybeHandleFatalError(line, timestamp),
+  });
 
   function maybeHandleFatalError(line, timestamp) {
     if (fatalError) return false;
@@ -217,52 +391,38 @@ export function createWatcherOutputRuntime({
     }
   }
 
-  function handleOutputLine(line, timestamp) {
+  function handleOutputLine(line, timestamp, { alreadyLogged = false, oversized = false } = {}) {
+    if (silentJsonMode && oversized) {
+      fatalError =
+        `Provider structured output exceeded the ${MAX_WATCHER_CONTROL_RECORD_BYTES}-byte ` +
+        'watcher inspection limit; complete output remains in the task log';
+      log(`[${timestamp}][FATAL] ${fatalError}\n`);
+      stopProvider(timestamp);
+      return;
+    }
     captureProviderSession(line);
     if (silentJsonMode && !line.trim()) return;
     maybeHandleFatalError(line, timestamp);
     if (captureStreamingError(line, timestamp)) return;
     if (silentJsonMode) {
       maybeCaptureStructuredOutput(line);
-    } else {
+    } else if (!alreadyLogged) {
       log(`[${timestamp}]${line}\n`);
     }
   }
 
-  function consumeOutput(buffer, chunk) {
-    const timestamp = Date.now();
-    const { lines, remaining } = splitBufferLines(buffer, chunk.toString());
-    for (const line of lines) handleOutputLine(line, timestamp);
-    return remaining;
-  }
-
-  function consumeStderr(buffer, chunk) {
-    const timestamp = Date.now();
-    const { lines, remaining } = splitBufferLines(buffer, chunk.toString());
-    for (const line of lines) log(`[${timestamp}]${line}\n`);
-    return remaining;
-  }
-
-  function flushOutput(buffer, timestamp) {
-    if (!buffer.trim()) return;
-    captureProviderSession(buffer);
-    if (!enableRecovery) {
-      if (!silentJsonMode) log(`[${timestamp}]${buffer}\n`);
-      return;
-    }
-    maybeHandleFatalError(buffer, timestamp);
-    if (captureStreamingError(buffer, timestamp)) return;
-    if (silentJsonMode) {
-      maybeCaptureStructuredOutput(buffer);
+  function consumeOutput(_buffer, chunk) {
+    if (codexOutputPassthrough) {
+      codexOutputPassthrough.consume(chunk);
     } else {
-      log(`[${timestamp}]${buffer}\n`);
+      outputPassthrough.consume(chunk);
     }
+    return '';
   }
 
-  function flushStderr(buffer, timestamp) {
-    if (!buffer.trim()) return;
-    maybeHandleFatalError(buffer, timestamp);
-    log(`[${timestamp}]${buffer}\n`);
+  function consumeStderr(_buffer, chunk) {
+    stderrPassthrough.consume(chunk);
+    return '';
   }
 
   function attemptRecovery(code, timestamp) {
@@ -282,10 +442,14 @@ export function createWatcherOutputRuntime({
     return recovered;
   }
 
-  function complete({ code, signal, outputBuffer, stderrBuffer = null }) {
+  function complete({ code, signal, stderrBuffer = null }) {
     const timestamp = Date.now();
-    flushOutput(outputBuffer, timestamp);
-    if (stderrBuffer !== null) flushStderr(stderrBuffer, timestamp);
+    if (codexOutputPassthrough) {
+      codexOutputPassthrough.flush();
+    } else {
+      outputPassthrough.flush();
+    }
+    if (stderrBuffer !== null) stderrPassthrough.flush();
     const recovered = attemptRecovery(code, timestamp);
     const sessionIdentityError = providerSessionCapture?.getCompletionError() || null;
     if (silentJsonMode && finalResultJson) log(`${finalResultJson}\n`);