@the-open-engine/zeroshot 6.39.2 → 6.40.0

package/README.md

@@ -146,6 +146,8 @@ zeroshot logs <id> -f            # stream logs
 zeroshot resume <id> [prompt]    # resume a stopped or failed run
 zeroshot stop <id>               # graceful stop
 zeroshot kill <id>               # force stop
+zeroshot export <id> --format trace --output run.trace.jsonl
+zeroshot export <id> --format semantic --output run.semantic.jsonl
 
 zeroshot providers               # provider availability and defaults
 zeroshot settings                # effective settings
@@ -158,6 +160,16 @@ template supports `{{issue_number}}`, `{{issue_title}}`, and `{{issue_reference}
 expand to empty text for tasks without an issue, so manual runs never emit `Closes #unknown`.
 The unrendered template is retained for detached and resumed runs.
 
+The `trace` export is a deterministic, provider-neutral research bundle. It preserves the ordered
+cluster ledger, exact selected prompts, and exact raw task-log bytes without interpreting a Claude,
+Codex, Pi, or other provider protocol. Missing evidence is recorded explicitly in its footer. File
+exports are create-only: choose a new output path rather than replacing an existing bundle. Live
+tasks are exported only as explicitly incomplete snapshots.
+The separate `semantic` export runs those task bytes through Zeroshot's existing stateful provider
+adapters and emits bounded `text`, `thinking`, `tool_call`, `tool_result`, and `result` events.
+Zeroshot-owned wrapper and stderr records remain native-only. Parser diagnostics affect only
+semantic completeness; they do not alter the native trace or run.
+
 </details>
 
 <details>

package/cli/agent-provider-boundary.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.stripTimestampPrefix = exports.getProviderAdapter = void 0;
+function isRuntime(value, method) {
+    return (value !== null && typeof value === 'object' && typeof Reflect.get(value, method) === 'function');
+}
+function isAdapterRuntime(value) {
+    return isRuntime(value, 'getProviderAdapter');
+}
+function isPrefixRuntime(value) {
+    return isRuntime(value, 'stripTimestampPrefix');
+}
+const adapters = require('../lib/agent-cli-provider/adapters');
+const prefixes = require('../lib/agent-cli-provider/log-prefix');
+if (!isAdapterRuntime(adapters) || !isPrefixRuntime(prefixes)) {
+    throw new Error('Built agent-provider runtime is unavailable');
+}
+exports.getProviderAdapter = adapters.getProviderAdapter;
+exports.stripTimestampPrefix = prefixes.stripTimestampPrefix;

package/cli/agent-provider-boundary.ts

@@ -0,0 +1,79 @@
+interface TextEvent {
+  readonly type: 'text' | 'thinking';
+  readonly text: string;
+}
+
+interface ToolCallEvent {
+  readonly type: 'tool_call';
+  readonly toolName: string | null | undefined;
+  readonly toolId: string | null | undefined;
+  readonly input: unknown;
+}
+
+interface ToolResultEvent {
+  readonly type: 'tool_result';
+  readonly toolId: string | null | undefined;
+  readonly content: unknown;
+  readonly isError: unknown;
+}
+
+interface ResultEvent {
+  readonly type: 'result';
+  readonly success: boolean;
+  readonly result?: unknown;
+  readonly error?: unknown;
+  readonly cost?: unknown;
+  readonly duration?: unknown;
+  readonly inputTokens?: number;
+  readonly outputTokens?: number;
+  readonly cacheReadInputTokens?: number;
+  readonly cacheCreationInputTokens?: number;
+  readonly modelUsage?: unknown;
+  readonly requests?: number;
+  readonly usageSource?: unknown;
+  readonly usageCompleteness?: unknown;
+  readonly invocation?: unknown;
+  readonly ompSdk?: unknown;
+}
+
+export type OutputEvent = TextEvent | ToolCallEvent | ToolResultEvent | ResultEvent;
+export type ProviderParseResult = OutputEvent | readonly OutputEvent[] | null;
+
+export interface ProviderAdapter {
+  readonly id: string;
+  readonly adapterVersion: string;
+  createParserState(): object;
+  parseEvent(line: string, state: object): ProviderParseResult;
+  finishParsing?(state: object): ProviderParseResult;
+}
+
+interface AdapterRuntime {
+  getProviderAdapter(provider: string): ProviderAdapter;
+}
+
+interface PrefixRuntime {
+  stripTimestampPrefix(line: string): string;
+}
+
+function isRuntime(value: unknown, method: string): boolean {
+  return (
+    value !== null && typeof value === 'object' && typeof Reflect.get(value, method) === 'function'
+  );
+}
+
+function isAdapterRuntime(value: unknown): value is AdapterRuntime {
+  return isRuntime(value, 'getProviderAdapter');
+}
+
+function isPrefixRuntime(value: unknown): value is PrefixRuntime {
+  return isRuntime(value, 'stripTimestampPrefix');
+}
+
+const adapters: unknown = require('../lib/agent-cli-provider/adapters');
+const prefixes: unknown = require('../lib/agent-cli-provider/log-prefix');
+if (!isAdapterRuntime(adapters) || !isPrefixRuntime(prefixes)) {
+  throw new Error('Built agent-provider runtime is unavailable');
+}
+
+export const getProviderAdapter = adapters.getProviderAdapter;
+export const stripTimestampPrefix = prefixes.stripTimestampPrefix;

package/cli/export-stream.js

@@ -0,0 +1,94 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.nullableString = nullableString;
+exports.isRecord = isRecord;
+exports.compareText = compareText;
+exports.sameFileSnapshot = sameFileSnapshot;
+exports.createExclusiveDestination = createExclusiveDestination;
+exports.createReplacingDestination = createReplacingDestination;
+exports.createRecordWriter = createRecordWriter;
+const crypto = require("crypto");
+const fs = require("fs");
+function nullableString(value) {
+    return typeof value === 'string' ? value : null;
+}
+function isRecord(value) {
+    return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+function compareText(left, right) {
+    if (left < right)
+        return -1;
+    return left > right ? 1 : 0;
+}
+function sameFileSnapshot(before, after) {
+    return (before.dev === after.dev &&
+        before.ino === after.ino &&
+        before.size === after.size &&
+        before.mtimeNs === after.mtimeNs &&
+        before.ctimeNs === after.ctimeNs);
+}
+function writeAll(fd, value, label) {
+    const bytes = Buffer.from(value);
+    let offset = 0;
+    while (offset < bytes.length) {
+        const written = fs.writeSync(fd, bytes, offset, bytes.length - offset);
+        if (!Number.isInteger(written) || written <= 0) {
+            throw new Error(`${label} export destination stopped accepting bytes`);
+        }
+        offset += written;
+    }
+}
+function streamDestination(stdout, label) {
+    if (typeof stdout.fd === 'number' && Number.isInteger(stdout.fd)) {
+        const fd = stdout.fd;
+        return { close() { }, write: (value) => writeAll(fd, value, label) };
+    }
+    return {
+        close() { },
+        write(value) {
+            stdout.write(value);
+        },
+    };
+}
+function createExclusiveDestination(outputPath, stdout, label) {
+    if (!outputPath)
+        return streamDestination(stdout, label);
+    const flags = fs.constants.O_WRONLY |
+        fs.constants.O_CREAT |
+        fs.constants.O_EXCL |
+        (fs.constants.O_NOFOLLOW || 0);
+    const fd = fs.openSync(outputPath, flags, 0o600);
+    try {
+        fs.fchmodSync(fd, 0o600);
+    }
+    catch (error) {
+        fs.closeSync(fd);
+        throw error;
+    }
+    return { close: () => fs.closeSync(fd), write: (value) => writeAll(fd, value, label) };
+}
+function createReplacingDestination(outputPath, stdout) {
+    if (!outputPath)
+        return streamDestination(stdout, 'JSON');
+    const fd = fs.openSync(outputPath, 'w');
+    return { close: () => fs.closeSync(fd), write: (value) => writeAll(fd, value, 'JSON') };
+}
+function createRecordWriter(destination) {
+    const digest = crypto.createHash('sha256');
+    let records = 0;
+    const encode = (record) => `${JSON.stringify(record)}\n`;
+    return {
+        get records() {
+            return records;
+        },
+        write(record) {
+            const line = encode(record);
+            digest.update(line);
+            destination.write(line);
+            records += 1;
+        },
+        finish(record) {
+            destination.write(encode({ ...record, preceding_records: records, records_sha256: digest.digest('hex') }));
+        },
+    };
+}

package/cli/export-stream.ts

@@ -0,0 +1,118 @@
+import crypto = require('crypto');
+import fs = require('fs');
+
+export interface ExportStream {
+  readonly fd?: number;
+  write(value: string): unknown;
+}
+
+export interface Destination {
+  close(): void;
+  write(value: string): void;
+}
+
+export interface RecordWriter {
+  readonly records: number;
+  finish(record: Record<string, unknown>): void;
+  write(record: Record<string, unknown>): void;
+}
+
+export function nullableString(value: unknown): string | null {
+  return typeof value === 'string' ? value : null;
+}
+
+export function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+export function compareText(left: string, right: string): number {
+  if (left < right) return -1;
+  return left > right ? 1 : 0;
+}
+
+export function sameFileSnapshot(before: fs.BigIntStats, after: fs.BigIntStats): boolean {
+  return (
+    before.dev === after.dev &&
+    before.ino === after.ino &&
+    before.size === after.size &&
+    before.mtimeNs === after.mtimeNs &&
+    before.ctimeNs === after.ctimeNs
+  );
+}
+
+function writeAll(fd: number, value: string, label: string): void {
+  const bytes = Buffer.from(value);
+  let offset = 0;
+  while (offset < bytes.length) {
+    const written = fs.writeSync(fd, bytes, offset, bytes.length - offset);
+    if (!Number.isInteger(written) || written <= 0) {
+      throw new Error(`${label} export destination stopped accepting bytes`);
+    }
+    offset += written;
+  }
+}
+
+function streamDestination(stdout: ExportStream, label: string): Destination {
+  if (typeof stdout.fd === 'number' && Number.isInteger(stdout.fd)) {
+    const fd = stdout.fd;
+    return { close(): void {}, write: (value) => writeAll(fd, value, label) };
+  }
+  return {
+    close(): void {},
+    write(value): void {
+      stdout.write(value);
+    },
+  };
+}
+
+export function createExclusiveDestination(
+  outputPath: string | null | undefined,
+  stdout: ExportStream,
+  label: string
+): Destination {
+  if (!outputPath) return streamDestination(stdout, label);
+  const flags =
+    fs.constants.O_WRONLY |
+    fs.constants.O_CREAT |
+    fs.constants.O_EXCL |
+    (fs.constants.O_NOFOLLOW || 0);
+  const fd = fs.openSync(outputPath, flags, 0o600);
+  try {
+    fs.fchmodSync(fd, 0o600);
+  } catch (error) {
+    fs.closeSync(fd);
+    throw error;
+  }
+  return { close: () => fs.closeSync(fd), write: (value) => writeAll(fd, value, label) };
+}
+
+export function createReplacingDestination(
+  outputPath: string | null | undefined,
+  stdout: ExportStream
+): Destination {
+  if (!outputPath) return streamDestination(stdout, 'JSON');
+  const fd = fs.openSync(outputPath, 'w');
+  return { close: () => fs.closeSync(fd), write: (value) => writeAll(fd, value, 'JSON') };
+}
+
+export function createRecordWriter(destination: Destination): RecordWriter {
+  const digest = crypto.createHash('sha256');
+  let records = 0;
+  const encode = (record: Record<string, unknown>): string => `${JSON.stringify(record)}\n`;
+  return {
+    get records(): number {
+      return records;
+    },
+    write(record): void {
+      const line = encode(record);
+      digest.update(line);
+      destination.write(line);
+      records += 1;
+    },
+    finish(record): void {
+      destination.write(
+        encode({ ...record, preceding_records: records, records_sha256: digest.digest('hex') })
+      );
+    },
+  };
+}

package/cli/index.js

@@ -59,6 +59,7 @@ const {
 } = require('../lib/provider-names');
 const { commandExists } = require('../lib/provider-detection');
 const { getProvider, parseProviderChunk } = require('../src/providers');
+const { decodeTaskLogLine } = require('../src/task-log-line');
 const { readClustersFileSync } = require('../lib/clusters-registry');
 const { MOUNT_PRESETS, resolveEnvs } = require('../lib/docker-config');
 const {
@@ -1430,19 +1431,10 @@ function startClusterDbPolling(cluster, clusterId, isActive, options) {
 }
 
 function parseIncrementalTaskLogLine(line) {
-  const trimmed = line.trim();
-  if (!trimmed.startsWith('{')) {
-    return null;
-  }
-
-  let timestamp = Date.now();
-  let jsonContent = trimmed;
-
-  const timestampMatch = jsonContent.match(/^\[(\d{13})\](.*)$/);
-  if (timestampMatch) {
-    timestamp = parseInt(timestampMatch[1], 10);
-    jsonContent = timestampMatch[2];
-  }
+  const decoded = decodeTaskLogLine(line.trim());
+  if (!decoded.providerOutput) return null;
+  const timestamp = decoded.timestamp ?? Date.now();
+  const jsonContent = decoded.content.trim();
 
   if (!jsonContent.startsWith('{')) {
     return null;
@@ -2619,37 +2611,7 @@ function findAgentFromLifecycle(agents, taskId, lifecycleMessages) {
 }
 
 function parseTaskLogLine(line) {
-  const trimmed = line.trim();
-  if (!trimmed.startsWith('[')) {
-    return null;
-  }
-
-  const parsed = parseTimestampedLine(trimmed);
-  if (!parsed.jsonContent.startsWith('{')) {
-    return null;
-  }
-
-  try {
-    const json = JSON.parse(parsed.jsonContent);
-    if (json.type === 'system' && json.subtype === 'init') {
-      return null;
-    }
-  } catch {
-    return null;
-  }
-
-  return parsed;
-}
-
-function parseTimestampedLine(line) {
-  const timestampMatch = line.match(/^\[(\d{13})\](.*)$/);
-  if (!timestampMatch) {
-    return { timestamp: Date.now(), jsonContent: line };
-  }
-  return {
-    timestamp: parseInt(timestampMatch[1], 10),
-    jsonContent: timestampMatch[2],
-  };
+  return parseIncrementalTaskLogLine(line);
 }
 
 function buildTaskLogMessage({ taskId, timestamp, jsonContent, cluster, agent, iteration }) {
@@ -3405,9 +3367,9 @@ program
   .command('export <cluster-id>')
   .helpGroup('Maintenance:')
   .description('Export cluster conversation')
-  .option('-f, --format <format>', 'Export format: json, markdown, html', 'html')
+  .option('-f, --format <format>', 'Export format: json, trace, semantic, markdown, html', 'html')
   .option('-o, --output <file>', 'Output file (auto-generated for html)')
-  .action((clusterId, options) => {
+  .action(async (clusterId, options) => {
     try {
       // Get messages from DB
       const Ledger = require('../src/ledger');
@@ -3418,14 +3380,7 @@ program
         throw new Error(`Cluster ${clusterId} not found (no DB file)`);
       }
 
-      // JSON export
-      if (options.format === 'json') {
-        exportClusterJson(dbPath, clusterId, options.output || null);
-        if (options.output) {
-          console.log(`Exported to ${options.output}`);
-        }
-        return;
-      }
+      if (await exportClusterDataFormat(dbPath, clusterId, options)) return;
 
       const ledger = new Ledger(dbPath);
       const messages = ledger.getAll(clusterId);
@@ -6202,6 +6157,98 @@ function exportClusterJson(dbPath, clusterId, outputPath) {
   );
 }
 
+function tryExportClusterTraceSnapshot(dbPath, clusterId, outputPath, readTask, allowedLogRoot) {
+  const Ledger = require('../src/ledger');
+  const { streamClusterTraceExport } = require('./trace-export');
+  const ledger = new Ledger(dbPath, { readonly: true });
+  try {
+    return ledger.withReadSnapshot(() => {
+      if (ledger.needsAgentOutputReconciliation(clusterId)) return false;
+      streamClusterTraceExport({
+        ledger,
+        clusterId,
+        readTask,
+        allowedLogRoot,
+        outputPath,
+      });
+      return true;
+    });
+  } finally {
+    ledger.close();
+  }
+}
+
+function exportClusterTrace(dbPath, clusterId, outputPath, readTask, allowedLogRoot) {
+  const Ledger = require('../src/ledger');
+  const maxReconciliationAttempts = 3;
+  for (let attempt = 0; attempt <= maxReconciliationAttempts; attempt += 1) {
+    if (tryExportClusterTraceSnapshot(dbPath, clusterId, outputPath, readTask, allowedLogRoot)) {
+      return;
+    }
+    if (attempt === maxReconciliationAttempts) break;
+    const reconciliationLedger = new Ledger(dbPath);
+    reconciliationLedger.close();
+  }
+  throw new Error(
+    `Cluster ${clusterId} output changed during ${maxReconciliationAttempts} reconciliation attempts`
+  );
+}
+
+function tryExportClusterSemanticSnapshot(dbPath, clusterId, outputPath, readTask, allowedLogRoot) {
+  const Ledger = require('../src/ledger');
+  const { streamClusterSemanticExport } = require('./semantic-export');
+  const ledger = new Ledger(dbPath, { readonly: true });
+  try {
+    return ledger.withReadSnapshot(() => {
+      if (ledger.needsAgentOutputReconciliation(clusterId)) return false;
+      streamClusterSemanticExport({
+        ledger,
+        clusterId,
+        readTask,
+        allowedLogRoot,
+        outputPath,
+      });
+      return true;
+    });
+  } finally {
+    ledger.close();
+  }
+}
+
+function exportClusterSemantic(dbPath, clusterId, outputPath, readTask, allowedLogRoot) {
+  const Ledger = require('../src/ledger');
+  const maxReconciliationAttempts = 3;
+  for (let attempt = 0; attempt <= maxReconciliationAttempts; attempt += 1) {
+    if (tryExportClusterSemanticSnapshot(dbPath, clusterId, outputPath, readTask, allowedLogRoot)) {
+      return;
+    }
+    if (attempt === maxReconciliationAttempts) break;
+    const reconciliationLedger = new Ledger(dbPath);
+    reconciliationLedger.close();
+  }
+  throw new Error(
+    `Cluster ${clusterId} output changed during ${maxReconciliationAttempts} reconciliation attempts`
+  );
+}
+
+async function exportClusterDataFormat(dbPath, clusterId, options) {
+  if (options.format === 'json') {
+    exportClusterJson(dbPath, clusterId, options.output || null);
+    if (options.output) console.log(`Exported to ${options.output}`);
+    return true;
+  }
+  if (options.format !== 'trace' && options.format !== 'semantic') return false;
+
+  const [{ getTask }, { LOGS_DIR }] = await Promise.all([
+    import('../task-lib/store.js'),
+    import('../task-lib/config.js'),
+  ]);
+  const exporter = options.format === 'trace' ? exportClusterTrace : exportClusterSemantic;
+  exporter(dbPath, clusterId, options.output || null, getTask, LOGS_DIR);
+  if (options.output) console.log(`Exported ${options.format} to ${options.output}`);
+  return true;
+}
+
 // Main entry point
 async function main() {
   printLegacyDistroNotice();
@@ -6258,4 +6305,5 @@ module.exports = {
   buildCompletionPrompt,
   resolveRunMode,
   killRunningClusters,
+  parseTaskLogLine,
 };

package/cli/json-export.js

@@ -1,8 +1,5 @@
 "use strict";
-const fs = require("fs");
-function isInteger(value) {
-    return Number.isInteger(value);
-}
+const export_stream_1 = require("./export-stream");
 function indentJson(value, spaces) {
     const prefix = ' '.repeat(spaces);
     return JSON.stringify(value, null, 2)
@@ -10,41 +7,8 @@ function indentJson(value, spaces) {
         .map((line) => `${prefix}${line}`)
         .join('\n');
 }
-function writeAll(fd, value) {
-    const bytes = Buffer.from(value);
-    let offset = 0;
-    while (offset < bytes.length) {
-        const written = fs.writeSync(fd, bytes, offset, bytes.length - offset);
-        if (!Number.isInteger(written) || written <= 0) {
-            throw new Error('JSON export destination stopped accepting bytes');
-        }
-        offset += written;
-    }
-}
-function createDestination(outputPath, stdout) {
-    if (outputPath) {
-        const fd = fs.openSync(outputPath, 'w');
-        return {
-            close: () => fs.closeSync(fd),
-            write: (value) => writeAll(fd, value),
-        };
-    }
-    if (isInteger(stdout.fd)) {
-        const fd = stdout.fd;
-        return {
-            close() { },
-            write: (value) => writeAll(fd, value),
-        };
-    }
-    return {
-        close() { },
-        write: (value) => {
-            stdout.write(value);
-        },
-    };
-}
 function streamClusterJsonExport({ ledger, clusterId, outputPath = null, stdout = process.stdout, }) {
-    const destination = createDestination(outputPath, stdout);
+    const destination = (0, export_stream_1.createReplacingDestination)(outputPath, stdout);
     const iterator = ledger.iterateAll(clusterId);
     try {
         destination.write(`{\n  "cluster_id": ${JSON.stringify(clusterId)},\n  "messages": `);

package/cli/json-export.ts

@@ -1,4 +1,4 @@
-import fs = require('fs');
+import { createReplacingDestination, type ExportStream } from './export-stream';
 
 interface LedgerMessageIterator extends Iterator<unknown> {
   return?: () => IteratorResult<unknown>;
@@ -8,11 +8,6 @@ interface ClusterLedger {
   iterateAll(clusterId: string): LedgerMessageIterator;
 }
 
-interface ExportStream {
-  readonly fd?: number;
-  write(value: string): unknown;
-}
-
 interface JsonExportOptions {
   ledger: ClusterLedger;
   clusterId: string;
@@ -20,15 +15,6 @@ interface JsonExportOptions {
   stdout?: ExportStream;
 }
 
-interface Destination {
-  close(): void;
-  write(value: string): void;
-}
-
-function isInteger(value: unknown): value is number {
-  return Number.isInteger(value);
-}
-
 function indentJson(value: unknown, spaces: number): string {
   const prefix = ' '.repeat(spaces);
   return JSON.stringify(value, null, 2)
@@ -37,53 +23,13 @@ function indentJson(value: unknown, spaces: number): string {
     .join('\n');
 }
 
-function writeAll(fd: number, value: string): void {
-  const bytes = Buffer.from(value);
-  let offset = 0;
-  while (offset < bytes.length) {
-    const written = fs.writeSync(fd, bytes, offset, bytes.length - offset);
-    if (!Number.isInteger(written) || written <= 0) {
-      throw new Error('JSON export destination stopped accepting bytes');
-    }
-    offset += written;
-  }
-}
-
-function createDestination(
-  outputPath: string | null | undefined,
-  stdout: ExportStream
-): Destination {
-  if (outputPath) {
-    const fd = fs.openSync(outputPath, 'w');
-    return {
-      close: () => fs.closeSync(fd),
-      write: (value) => writeAll(fd, value),
-    };
-  }
-
-  if (isInteger(stdout.fd)) {
-    const fd = stdout.fd;
-    return {
-      close(): void {},
-      write: (value) => writeAll(fd, value),
-    };
-  }
-
-  return {
-    close(): void {},
-    write: (value): void => {
-      stdout.write(value);
-    },
-  };
-}
-
 function streamClusterJsonExport({
   ledger,
   clusterId,
   outputPath = null,
   stdout = process.stdout,
 }: JsonExportOptions): void {
-  const destination = createDestination(outputPath, stdout);
+  const destination = createReplacingDestination(outputPath, stdout);
   const iterator = ledger.iterateAll(clusterId);
   try {
     destination.write(`{\n  "cluster_id": ${JSON.stringify(clusterId)},\n  "messages": `);