@the-open-engine/zeroshot 6.39.2 → 6.41.0

package/cli/trace-export.ts

@@ -0,0 +1,164 @@
+import path = require('path');
+import {
+  compareText,
+  createExclusiveDestination,
+  createRecordWriter,
+  nullableString,
+  type ExportStream,
+  type RecordWriter,
+} from './export-stream';
+import {
+  collectTaskCauses,
+  expectedLogPath,
+  forEachLedgerMessage,
+  hasTerminalTaskStatus,
+  logicalTaskRef,
+  readTraceTask,
+  type ClusterLedger,
+  type TaskCause,
+  type TraceTask,
+} from './trace-evidence';
+import { streamTaskOutput, TRACE_OUTPUT_CHUNK_BYTES, writeUnavailableOutput } from './trace-output';
+
+const TRACE_SCHEMA_VERSION = 'zeroshot.trace.v1';
+const TRACE_MEDIA_TYPE = 'application/x-zeroshot-trace+jsonl';
+
+interface TraceExportOptions {
+  ledger: ClusterLedger;
+  clusterId: string;
+  readTask(taskId: string): TraceTask | null;
+  allowedLogRoot: string;
+  outputPath?: string | null;
+  stdout?: ExportStream;
+}
+
+interface StreamTaskOptions {
+  writer: RecordWriter;
+  taskId: string;
+  cause: TaskCause;
+  readTask(taskId: string): TraceTask | null;
+  allowedLogRoot: string;
+  issues: string[];
+}
+
+function nullableInteger(value: unknown): number | null {
+  return typeof value === 'number' && Number.isInteger(value) ? value : null;
+}
+
+function assertOutputDoesNotReplaceTaskLog(
+  outputPath: string | null | undefined,
+  allowedLogRoot: string,
+  taskIds: Iterable<string>
+): void {
+  if (!outputPath) return;
+  const resolvedOutput = path.resolve(outputPath);
+  for (const taskId of taskIds) {
+    if (expectedLogPath(allowedLogRoot, taskId) === resolvedOutput) {
+      throw new Error('Trace export output cannot replace a source task log');
+    }
+  }
+}
+
+function taskPrompt(task: TraceTask): string | null {
+  return nullableString(task.fullPrompt) ?? nullableString(task.prompt);
+}
+
+function streamTask(options: StreamTaskOptions): { bytes: number } {
+  const { writer, taskId, cause, readTask, allowedLogRoot, issues } = options;
+  const agentIds = [...cause.agentIds].sort(compareText);
+  if (agentIds.length > 1) issues.push(`task:${taskId}:ambiguous_agent`);
+  const taskRead = readTraceTask(taskId, readTask);
+  const task = taskRead.task;
+  if (taskRead.issue) issues.push(taskRead.issue);
+  const taskTerminal = task !== null && hasTerminalTaskStatus(task);
+  if (task && !taskTerminal) issues.push(`task:${taskId}:task_not_terminal`);
+  const promptRef = logicalTaskRef(taskId, 'prompt');
+  const rawOutputRef = logicalTaskRef(taskId, 'output');
+  writer.write({
+    record_type: 'task',
+    task_id: taskId,
+    agent_id: agentIds.length === 1 ? agentIds[0] : null,
+    provider: nullableString(task?.provider),
+    model: nullableString(task?.model),
+    status: nullableString(task?.status),
+    created_at: nullableString(task?.createdAt),
+    updated_at: nullableString(task?.updatedAt),
+    exit_code: nullableInteger(task?.exitCode),
+    prompt_ref: promptRef,
+    prompt: task ? taskPrompt(task) : null,
+    raw_output_ref: rawOutputRef,
+  });
+  if (!task) {
+    writeUnavailableOutput(writer, taskId, rawOutputRef);
+    return { bytes: 0 };
+  }
+  return streamTaskOutput({
+    writer,
+    taskId,
+    task,
+    allowedLogRoot,
+    rawOutputRef,
+    taskTerminal,
+    issues,
+  });
+}
+
+function streamClusterTraceExport(options: TraceExportOptions): void {
+  const { ledger, clusterId, readTask, allowedLogRoot, outputPath = null } = options;
+  const causes = collectTaskCauses(ledger, clusterId);
+  assertOutputDoesNotReplaceTaskLog(outputPath, allowedLogRoot, causes.keys());
+  const destination = createExclusiveDestination(
+    outputPath,
+    options.stdout ?? process.stdout,
+    'Trace'
+  );
+  const writer = createRecordWriter(destination);
+  let ledgerMessages = 0;
+  let taskOutputBytes = 0;
+  const issues: string[] = [];
+  try {
+    writer.write({
+      record_type: 'header',
+      schema_version: TRACE_SCHEMA_VERSION,
+      media_type: TRACE_MEDIA_TYPE,
+      cluster_id: clusterId,
+      chunk_bytes: TRACE_OUTPUT_CHUNK_BYTES,
+    });
+    forEachLedgerMessage(ledger, clusterId, (message) => {
+      writer.write({ record_type: 'ledger_message', message });
+      ledgerMessages += 1;
+    });
+    const ordered = [...causes.entries()].sort(([left], [right]) => compareText(left, right));
+    for (const [taskId, cause] of ordered) {
+      taskOutputBytes += streamTask({
+        writer,
+        taskId,
+        cause,
+        readTask,
+        allowedLogRoot,
+        issues,
+      }).bytes;
+    }
+    issues.sort(compareText);
+    writer.finish({
+      record_type: 'footer',
+      complete: issues.length === 0,
+      ledger_messages: ledgerMessages,
+      tasks: causes.size,
+      task_output_bytes: taskOutputBytes,
+      issues,
+    });
+  } finally {
+    destination.close();
+  }
+}
+
+export = {
+  TRACE_OUTPUT_CHUNK_BYTES,
+  collectTaskCauses,
+  expectedLogPath,
+  hasTerminalTaskStatus,
+  logicalTaskRef,
+  readTraceTask,
+  streamClusterTraceExport,
+};

package/cli/trace-output-record.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.writeUnavailableOutput = writeUnavailableOutput;
+function writeUnavailableOutput(writer, taskId, rawOutputRef) {
+    writer.write({
+        record_type: 'task_output_end',
+        task_id: taskId,
+        raw_output_ref: rawOutputRef,
+        available: false,
+        complete: false,
+        byte_length: null,
+        chunks: 0,
+        sha256: null,
+    });
+}

package/cli/trace-output-record.ts

@@ -0,0 +1,18 @@
+import type { RecordWriter } from './export-stream';
+
+export function writeUnavailableOutput(
+  writer: RecordWriter,
+  taskId: string,
+  rawOutputRef: string
+): void {
+  writer.write({
+    record_type: 'task_output_end',
+    task_id: taskId,
+    raw_output_ref: rawOutputRef,
+    available: false,
+    complete: false,
+    byte_length: null,
+    chunks: 0,
+    sha256: null,
+  });
+}

package/cli/trace-output.js

@@ -0,0 +1,98 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.writeUnavailableOutput = exports.TRACE_OUTPUT_CHUNK_BYTES = void 0;
+exports.streamTaskOutput = streamTaskOutput;
+const crypto = require("crypto");
+const fs = require("fs");
+const path = require("path");
+const export_stream_1 = require("./export-stream");
+const trace_evidence_1 = require("./trace-evidence");
+const trace_output_record_1 = require("./trace-output-record");
+Object.defineProperty(exports, "writeUnavailableOutput", { enumerable: true, get: function () { return trace_output_record_1.writeUnavailableOutput; } });
+exports.TRACE_OUTPUT_CHUNK_BYTES = 48 * 1024;
+function openTaskLog(expected, taskId, issues) {
+    try {
+        return fs.openSync(expected, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0));
+    }
+    catch (error) {
+        issues.push(`task:${taskId}:${issueForOpenFailure(error)}`);
+        return null;
+    }
+}
+function captureOpenedTaskLog(fd, options) {
+    const { writer, taskId, rawOutputRef, issues } = options;
+    let bytes = 0;
+    let chunks = 0;
+    const digest = crypto.createHash('sha256');
+    try {
+        const before = fs.fstatSync(fd, { bigint: true });
+        if (!before.isFile() || before.size > BigInt(Number.MAX_SAFE_INTEGER)) {
+            issues.push(`task:${taskId}:log_not_regular`);
+            return { available: false, complete: false, bytes, chunks, sha256: null };
+        }
+        const targetBytes = Number(before.size);
+        while (bytes < targetBytes) {
+            const buffer = Buffer.allocUnsafe(Math.min(exports.TRACE_OUTPUT_CHUNK_BYTES, targetBytes - bytes));
+            const read = fs.readSync(fd, buffer, 0, buffer.length, bytes);
+            if (read === 0)
+                break;
+            const chunk = read === buffer.length ? buffer : buffer.subarray(0, read);
+            digest.update(chunk);
+            writer.write({
+                record_type: 'task_output_chunk',
+                task_id: taskId,
+                raw_output_ref: rawOutputRef,
+                chunk_index: chunks,
+                encoding: 'base64',
+                data_base64: chunk.toString('base64'),
+            });
+            bytes += read;
+            chunks += 1;
+        }
+        const after = fs.fstatSync(fd, { bigint: true });
+        const complete = bytes === targetBytes && (0, export_stream_1.sameFileSnapshot)(before, after);
+        if (!complete)
+            issues.push(`task:${taskId}:log_changed_during_export`);
+        return { available: true, complete, bytes, chunks, sha256: digest.digest('hex') };
+    }
+    catch {
+        issues.push(`task:${taskId}:log_read_failed`);
+        return { available: true, complete: false, bytes, chunks, sha256: digest.digest('hex') };
+    }
+}
+function streamTaskOutput(options) {
+    const { writer, taskId, task, allowedLogRoot, rawOutputRef, taskTerminal, issues } = options;
+    const expected = (0, trace_evidence_1.expectedLogPath)(allowedLogRoot, taskId);
+    const recorded = (0, export_stream_1.nullableString)(task.logFile);
+    if (!expected || !recorded || path.resolve(recorded) !== expected) {
+        issues.push(`task:${taskId}:log_reference_invalid`);
+        (0, trace_output_record_1.writeUnavailableOutput)(writer, taskId, rawOutputRef);
+        return { bytes: 0 };
+    }
+    const fd = openTaskLog(expected, taskId, issues);
+    if (fd === null) {
+        (0, trace_output_record_1.writeUnavailableOutput)(writer, taskId, rawOutputRef);
+        return { bytes: 0 };
+    }
+    let captured;
+    try {
+        captured = captureOpenedTaskLog(fd, { writer, taskId, rawOutputRef, issues });
+    }
+    finally {
+        fs.closeSync(fd);
+    }
+    writer.write({
+        record_type: 'task_output_end',
+        task_id: taskId,
+        raw_output_ref: rawOutputRef,
+        available: captured.available,
+        complete: captured.complete && taskTerminal,
+        byte_length: captured.available ? captured.bytes : null,
+        chunks: captured.chunks,
+        sha256: captured.sha256,
+    });
+    return { bytes: captured.bytes };
+}
+function issueForOpenFailure(error) {
+    return (0, export_stream_1.isRecord)(error) && error.code === 'ENOENT' ? 'log_missing' : 'log_unreadable';
+}

package/cli/trace-output.ts

@@ -0,0 +1,119 @@
+import crypto = require('crypto');
+import fs = require('fs');
+import path = require('path');
+import { isRecord, nullableString, type RecordWriter, sameFileSnapshot } from './export-stream';
+import { expectedLogPath, type TraceTask } from './trace-evidence';
+import { writeUnavailableOutput } from './trace-output-record';
+
+export const TRACE_OUTPUT_CHUNK_BYTES = 48 * 1024;
+
+interface OutputCapture {
+  bytes: number;
+}
+
+interface CapturedTaskOutput extends OutputCapture {
+  available: boolean;
+  complete: boolean;
+  chunks: number;
+  sha256: string | null;
+}
+
+interface StreamTaskOutputOptions {
+  writer: RecordWriter;
+  taskId: string;
+  task: TraceTask;
+  allowedLogRoot: string;
+  rawOutputRef: string;
+  taskTerminal: boolean;
+  issues: string[];
+}
+
+function openTaskLog(expected: string, taskId: string, issues: string[]): number | null {
+  try {
+    return fs.openSync(expected, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0));
+  } catch (error) {
+    issues.push(`task:${taskId}:${issueForOpenFailure(error)}`);
+    return null;
+  }
+}
+
+function captureOpenedTaskLog(
+  fd: number,
+  options: Pick<StreamTaskOutputOptions, 'writer' | 'taskId' | 'rawOutputRef' | 'issues'>
+): CapturedTaskOutput {
+  const { writer, taskId, rawOutputRef, issues } = options;
+  let bytes = 0;
+  let chunks = 0;
+  const digest = crypto.createHash('sha256');
+  try {
+    const before = fs.fstatSync(fd, { bigint: true });
+    if (!before.isFile() || before.size > BigInt(Number.MAX_SAFE_INTEGER)) {
+      issues.push(`task:${taskId}:log_not_regular`);
+      return { available: false, complete: false, bytes, chunks, sha256: null };
+    }
+    const targetBytes = Number(before.size);
+    while (bytes < targetBytes) {
+      const buffer = Buffer.allocUnsafe(Math.min(TRACE_OUTPUT_CHUNK_BYTES, targetBytes - bytes));
+      const read = fs.readSync(fd, buffer, 0, buffer.length, bytes);
+      if (read === 0) break;
+      const chunk = read === buffer.length ? buffer : buffer.subarray(0, read);
+      digest.update(chunk);
+      writer.write({
+        record_type: 'task_output_chunk',
+        task_id: taskId,
+        raw_output_ref: rawOutputRef,
+        chunk_index: chunks,
+        encoding: 'base64',
+        data_base64: chunk.toString('base64'),
+      });
+      bytes += read;
+      chunks += 1;
+    }
+    const after = fs.fstatSync(fd, { bigint: true });
+    const complete = bytes === targetBytes && sameFileSnapshot(before, after);
+    if (!complete) issues.push(`task:${taskId}:log_changed_during_export`);
+    return { available: true, complete, bytes, chunks, sha256: digest.digest('hex') };
+  } catch {
+    issues.push(`task:${taskId}:log_read_failed`);
+    return { available: true, complete: false, bytes, chunks, sha256: digest.digest('hex') };
+  }
+}
+
+export function streamTaskOutput(options: StreamTaskOutputOptions): OutputCapture {
+  const { writer, taskId, task, allowedLogRoot, rawOutputRef, taskTerminal, issues } = options;
+  const expected = expectedLogPath(allowedLogRoot, taskId);
+  const recorded = nullableString(task.logFile);
+  if (!expected || !recorded || path.resolve(recorded) !== expected) {
+    issues.push(`task:${taskId}:log_reference_invalid`);
+    writeUnavailableOutput(writer, taskId, rawOutputRef);
+    return { bytes: 0 };
+  }
+  const fd = openTaskLog(expected, taskId, issues);
+  if (fd === null) {
+    writeUnavailableOutput(writer, taskId, rawOutputRef);
+    return { bytes: 0 };
+  }
+  let captured: CapturedTaskOutput;
+  try {
+    captured = captureOpenedTaskLog(fd, { writer, taskId, rawOutputRef, issues });
+  } finally {
+    fs.closeSync(fd);
+  }
+  writer.write({
+    record_type: 'task_output_end',
+    task_id: taskId,
+    raw_output_ref: rawOutputRef,
+    available: captured.available,
+    complete: captured.complete && taskTerminal,
+    byte_length: captured.available ? captured.bytes : null,
+    chunks: captured.chunks,
+    sha256: captured.sha256,
+  });
+  return { bytes: captured.bytes };
+}
+
+function issueForOpenFailure(error: unknown): 'log_missing' | 'log_unreadable' {
+  return isRecord(error) && error.code === 'ENOENT' ? 'log_missing' : 'log_unreadable';
+}
+
+export { writeUnavailableOutput };

package/lib/agent-cli-provider/log-prefix.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"log-prefix.d.ts","sourceRoot":"","sources":["../../src/agent-cli-provider/log-prefix.ts"],"names":[],"mappings":"AAAA,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAmBzD"}
+{"version":3,"file":"log-prefix.d.ts","sourceRoot":"","sources":["../../src/agent-cli-provider/log-prefix.ts"],"names":[],"mappings":"AASA,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAezD"}

package/lib/agent-cli-provider/log-prefix.js

@@ -1,15 +1,17 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.stripTimestampPrefix = stripTimestampPrefix;
+// The canonical decoder is maintained JavaScript and remains at the same source path after build.
+// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+const taskLogDecoder = require('../../src/task-log-line');
+const { decodeTaskLogLine } = taskLogDecoder;
 function stripTimestampPrefix(line) {
-    let trimmed = line.trim().replace(/\r$/, '');
+    const decoded = decodeTaskLogLine(line.trim());
+    if (!decoded.providerOutput)
+        return '';
+    let trimmed = decoded.content;
     if (!trimmed)
         return '';
-    const timestampMatch = /^\[(\d{13})\](.*)$/.exec(trimmed);
-    const timestampRemainder = timestampMatch?.[2];
-    if (typeof timestampRemainder === 'string') {
-        trimmed = timestampRemainder.trimStart();
-    }
     if (!trimmed.startsWith('{') && !trimmed.startsWith('[')) {
         const pipeMatch = /^[^|]{1,40}\|\s*(.*)$/.exec(trimmed);
         const afterPipe = pipeMatch?.[1]?.trimStart();

package/lib/agent-cli-provider/log-prefix.js.map

@@ -1 +1 @@
-{"version":3,"file":"log-prefix.js","sourceRoot":"","sources":["../../src/agent-cli-provider/log-prefix.ts"],"names":[],"mappings":";;AAAA,oDAmBC;AAnBD,SAAgB,oBAAoB,CAAC,IAAY;IAC/C,IAAI,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC7C,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IAExB,MAAM,cAAc,GAAG,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC1D,MAAM,kBAAkB,GAAG,cAAc,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/C,IAAI,OAAO,kBAAkB,KAAK,QAAQ,EAAE,CAAC;QAC3C,OAAO,GAAG,kBAAkB,CAAC,SAAS,EAAE,CAAC;IAC3C,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACzD,MAAM,SAAS,GAAG,uBAAuB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC;QAC9C,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YAC9F,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
+{"version":3,"file":"log-prefix.js","sourceRoot":"","sources":["../../src/agent-cli-provider/log-prefix.ts"],"names":[],"mappings":";;AASA,oDAeC;AApBD,kGAAkG;AAClG,mEAAmE;AACnE,MAAM,cAAc,GAAmB,OAAO,CAAC,yBAAyB,CAAC,CAAC;AAC1E,MAAM,EAAE,iBAAiB,EAAE,GAAG,cAAc,CAAC;AAE7C,SAAgB,oBAAoB,CAAC,IAAY;IAC/C,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IAC/C,IAAI,CAAC,OAAO,CAAC,cAAc;QAAE,OAAO,EAAE,CAAC;IACvC,IAAI,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAC9B,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IAExB,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACzD,MAAM,SAAS,GAAG,uBAAuB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC;QAC9C,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YAC9F,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/lib/cluster/connection-types.d.cts

@@ -1,11 +1,11 @@
 import type { FrameRecord } from './frames.js';
 import type { BoundedQueue } from './queue.js';
-import type { ClusterMethod } from './generated/protocol.js';
+import type { ClusterMethod, SubscriptionMethod } from './generated/protocol.js';
 export interface CallOptions {
     readonly signal?: AbortSignal;
     readonly requestTimeoutMs?: number;
 }
-export type SubscriptionKind = 'watch' | 'logs' | 'agent/attach';
+export type SubscriptionKind = SubscriptionMethod;
 export type SubscriptionRegistration = {
     readonly id: string;
     readonly kind: SubscriptionKind;

package/lib/cluster/connection-types.d.mts

@@ -1,11 +1,11 @@
 import type { FrameRecord } from './frames.js';
 import type { BoundedQueue } from './queue.js';
-import type { ClusterMethod } from './generated/protocol.js';
+import type { ClusterMethod, SubscriptionMethod } from './generated/protocol.js';
 export interface CallOptions {
     readonly signal?: AbortSignal;
     readonly requestTimeoutMs?: number;
 }
-export type SubscriptionKind = 'watch' | 'logs' | 'agent/attach';
+export type SubscriptionKind = SubscriptionMethod;
 export type SubscriptionRegistration = {
     readonly id: string;
     readonly kind: SubscriptionKind;

package/lib/cluster/connection-types.d.ts

@@ -1,11 +1,11 @@
 import type { FrameRecord } from './frames.js';
 import type { BoundedQueue } from './queue.js';
-import type { ClusterMethod } from './generated/protocol.js';
+import type { ClusterMethod, SubscriptionMethod } from './generated/protocol.js';
 export interface CallOptions {
     readonly signal?: AbortSignal;
     readonly requestTimeoutMs?: number;
 }
-export type SubscriptionKind = 'watch' | 'logs' | 'agent/attach';
+export type SubscriptionKind = SubscriptionMethod;
 export type SubscriptionRegistration = {
     readonly id: string;
     readonly kind: SubscriptionKind;