@tonyclaw/agent-inspector 3.0.45 → 3.0.47

package/src/proxy/jsonlRecovery.ts

@@ -0,0 +1,554 @@
+import { createHash, randomUUID } from "node:crypto";
+import { Buffer } from "node:buffer";
+import { constants, existsSync, lstatSync } from "node:fs";
+import { link, lstat, open, readFile, readdir, stat, unlink } from "node:fs/promises";
+import { basename, join, resolve } from "node:path";
+import { CapturedLogSchema } from "./schemas";
+import {
+  ensurePrivateDirectorySync,
+  PRIVATE_FILE_MODE,
+  securePrivateFileSync,
+} from "./privateDataPath";
+
+const READ_CHUNK_BYTES = 64 * 1024;
+const QUARANTINE_DIRECTORY = "quarantine";
+
+export type CapturedLogRecoveryLevel = "ok" | "degraded" | "unavailable";
+
+export type CapturedLogRecoveryHealth = {
+  level: CapturedLogRecoveryLevel;
+  diagnostic: string | null;
+  repairedFiles: number;
+  delimiterRepairs: number;
+  quarantinedTails: number;
+};
+
+type FileSignature = {
+  size: number;
+  modifiedAt: number;
+  changedAt: number;
+  device: number;
+  inode: number;
+};
+
+type RecoveryContext = {
+  root: string;
+  health: CapturedLogRecoveryHealth;
+  startup: Promise<CapturedLogRecoveryHealth> | null;
+  startupComplete: boolean;
+  safeFiles: Map<string, FileSignature>;
+};
+
+type RecordValidation =
+  | { valid: true }
+  | { valid: false; reason: "invalid-json" | "invalid-log-schema" };
+
+type InvalidRecordReason = "invalid-json" | "invalid-log-schema";
+
+type FileRecoveryResult =
+  | { status: "clean"; signature: FileSignature }
+  | { status: "delimiter-appended"; signature: FileSignature }
+  | { status: "tail-quarantined"; signature: FileSignature }
+  | {
+      status: "interior-corruption";
+      line: number;
+      byteOffset: number;
+      recordHash: string;
+      reason: InvalidRecordReason;
+    }
+  | { status: "failed"; reason: string };
+
+type ScanResult =
+  | {
+      status: "complete";
+      pending: Buffer;
+      pendingOffset: number;
+      nextLine: number;
+    }
+  | {
+      status: "interior-corruption";
+      line: number;
+      byteOffset: number;
+      recordHash: string;
+      reason: InvalidRecordReason;
+    }
+  | { status: "failed"; reason: string };
+
+type QuarantineRecord = {
+  schemaVersion: 1;
+  reason: "invalid-final-jsonl-record";
+  sourceFile: string;
+  sourcePathHash: string;
+  recoveredAt: string;
+  tailOffset: number;
+  tailBytes: number;
+  tailSha256: string;
+  tailBase64: string;
+};
+
+type ExistingQuarantineState = "missing" | "matching" | "conflicting";
+
+let activeContext: RecoveryContext | null = null;
+
+function emptyHealth(): CapturedLogRecoveryHealth {
+  return {
+    level: "ok",
+    diagnostic: null,
+    repairedFiles: 0,
+    delimiterRepairs: 0,
+    quarantinedTails: 0,
+  };
+}
+
+function contextFor(logDir: string): RecoveryContext {
+  const root = resolve(logDir);
+  if (activeContext === null || activeContext.root !== root) {
+    activeContext = {
+      root,
+      health: emptyHealth(),
+      startup: null,
+      startupComplete: false,
+      safeFiles: new Map(),
+    };
+  }
+  return activeContext;
+}
+
+function copyHealth(health: CapturedLogRecoveryHealth): CapturedLogRecoveryHealth {
+  return { ...health };
+}
+
+function hashBytes(value: Uint8Array): string {
+  return createHash("sha256").update(value).digest("hex");
+}
+
+function signatureFromStats(stats: {
+  size: number;
+  mtimeMs: number;
+  ctimeMs: number;
+  dev: number;
+  ino: number;
+}): FileSignature {
+  return {
+    size: stats.size,
+    modifiedAt: stats.mtimeMs,
+    changedAt: stats.ctimeMs,
+    device: stats.dev,
+    inode: stats.ino,
+  };
+}
+
+function signaturesMatch(left: FileSignature, right: FileSignature): boolean {
+  return (
+    left.size === right.size &&
+    left.modifiedAt === right.modifiedAt &&
+    left.changedAt === right.changedAt &&
+    left.device === right.device &&
+    left.inode === right.inode
+  );
+}
+
+function validateRecord(bytes: Buffer): RecordValidation {
+  const withoutCarriageReturn =
+    bytes.length > 0 && bytes[bytes.length - 1] === 0x0d ? bytes.subarray(0, -1) : bytes;
+  const text = withoutCarriageReturn.toString("utf-8");
+  if (text.trim() === "") return { valid: true };
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    return { valid: false, reason: "invalid-json" };
+  }
+  return CapturedLogSchema.safeParse(parsed).success
+    ? { valid: true }
+    : { valid: false, reason: "invalid-log-schema" };
+}
+
+async function scanFile(
+  handle: Awaited<ReturnType<typeof open>>,
+  size: number,
+): Promise<ScanResult> {
+  const readBuffer = Buffer.alloc(READ_CHUNK_BYTES);
+  let position = 0;
+  let pending = Buffer.alloc(0);
+  let pendingOffset = 0;
+  let line = 1;
+
+  while (position < size) {
+    const length = Math.min(READ_CHUNK_BYTES, size - position);
+    let bytesRead: number;
+    try {
+      const result = await handle.read(readBuffer, 0, length, position);
+      bytesRead = result.bytesRead;
+    } catch (error) {
+      return { status: "failed", reason: `read failed: ${String(error)}` };
+    }
+    if (bytesRead === 0) {
+      return { status: "failed", reason: "captured-log file changed while being scanned" };
+    }
+
+    const chunk = readBuffer.subarray(0, bytesRead);
+    const combined = pending.length === 0 ? Buffer.from(chunk) : Buffer.concat([pending, chunk]);
+    let consumed = 0;
+    let newline = combined.indexOf(0x0a, consumed);
+    while (newline !== -1) {
+      const record = combined.subarray(consumed, newline);
+      const validation = validateRecord(record);
+      if (!validation.valid) {
+        return {
+          status: "interior-corruption",
+          line,
+          byteOffset: pendingOffset + consumed,
+          recordHash: hashBytes(record),
+          reason: validation.reason,
+        };
+      }
+      consumed = newline + 1;
+      line += 1;
+      newline = combined.indexOf(0x0a, consumed);
+    }
+
+    pending = Buffer.from(combined.subarray(consumed));
+    pendingOffset += consumed;
+    position += bytesRead;
+  }
+
+  return { status: "complete", pending, pendingOffset, nextLine: line };
+}
+
+async function syncDirectory(directory: string): Promise<void> {
+  let handle: Awaited<ReturnType<typeof open>> | null = null;
+  try {
+    handle = await open(directory, "r");
+    await handle.sync();
+  } catch {
+    // Directory fsync is unavailable on some supported Windows filesystems.
+  } finally {
+    await handle?.close().catch(() => undefined);
+  }
+}
+
+async function removeTemporaryFile(filePath: string): Promise<void> {
+  await unlink(filePath).catch(() => undefined);
+}
+
+function readOwnProperty(value: unknown, name: string): unknown {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined;
+  return Object.getOwnPropertyDescriptor(value, name)?.value;
+}
+
+async function inspectExistingQuarantine(
+  filePath: string,
+  expected: QuarantineRecord,
+): Promise<ExistingQuarantineState> {
+  if (!existsSync(filePath)) return "missing";
+  try {
+    const parsed: unknown = JSON.parse(await readFile(filePath, "utf-8"));
+    return readOwnProperty(parsed, "schemaVersion") === expected.schemaVersion &&
+      readOwnProperty(parsed, "reason") === expected.reason &&
+      readOwnProperty(parsed, "sourceFile") === expected.sourceFile &&
+      readOwnProperty(parsed, "sourcePathHash") === expected.sourcePathHash &&
+      readOwnProperty(parsed, "tailOffset") === expected.tailOffset &&
+      readOwnProperty(parsed, "tailBytes") === expected.tailBytes &&
+      readOwnProperty(parsed, "tailSha256") === expected.tailSha256 &&
+      readOwnProperty(parsed, "tailBase64") === expected.tailBase64
+      ? "matching"
+      : "conflicting";
+  } catch {
+    return "conflicting";
+  }
+}
+
+async function quarantineTail(
+  context: RecoveryContext,
+  sourceFile: string,
+  tail: Buffer,
+  tailOffset: number,
+): Promise<string | null> {
+  const quarantineDirectory = join(context.root, QUARANTINE_DIRECTORY);
+  if (!ensurePrivateDirectorySync(quarantineDirectory)) return null;
+
+  const tailSha256 = hashBytes(tail);
+  const sourceFileName = basename(sourceFile);
+  const destination = join(quarantineDirectory, `${sourceFileName}.tail-${tailSha256}.json`);
+  const temporary = join(
+    quarantineDirectory,
+    `.${sourceFileName}.${String(process.pid)}.${randomUUID()}.tmp`,
+  );
+  const record: QuarantineRecord = {
+    schemaVersion: 1,
+    reason: "invalid-final-jsonl-record",
+    sourceFile: sourceFileName,
+    sourcePathHash: hashBytes(Buffer.from(resolve(sourceFile), "utf-8")),
+    recoveredAt: new Date().toISOString(),
+    tailOffset,
+    tailBytes: tail.length,
+    tailSha256,
+    tailBase64: tail.toString("base64"),
+  };
+
+  const existing = await inspectExistingQuarantine(destination, record);
+  if (existing === "matching") {
+    return securePrivateFileSync(destination) ? destination : null;
+  }
+  if (existing === "conflicting") return null;
+
+  let handle: Awaited<ReturnType<typeof open>> | null = null;
+  try {
+    handle = await open(temporary, "wx", PRIVATE_FILE_MODE);
+    await handle.writeFile(`${JSON.stringify(record)}\n`, "utf-8");
+    await handle.sync();
+    await handle.close();
+    handle = null;
+    if (!securePrivateFileSync(temporary)) {
+      await removeTemporaryFile(temporary);
+      return null;
+    }
+    // A hard-link install has create-if-absent semantics on every supported platform. Unlike
+    // rename(), it cannot overwrite a conflicting quarantine artifact that appears after the
+    // preflight check on POSIX.
+    await link(temporary, destination);
+    await unlink(temporary);
+    if (!securePrivateFileSync(destination)) return null;
+    await syncDirectory(quarantineDirectory);
+    return destination;
+  } catch {
+    await handle?.close().catch(() => undefined);
+    await removeTemporaryFile(temporary);
+    const racedExisting = await inspectExistingQuarantine(destination, record);
+    return racedExisting === "matching" && securePrivateFileSync(destination) ? destination : null;
+  }
+}
+
+async function recoverFile(
+  context: RecoveryContext,
+  filePath: string,
+): Promise<FileRecoveryResult> {
+  if (!existsSync(filePath)) {
+    context.safeFiles.delete(filePath);
+    return {
+      status: "clean",
+      signature: { size: 0, modifiedAt: 0, changedAt: 0, device: 0, inode: 0 },
+    };
+  }
+
+  let fileStats: Awaited<ReturnType<typeof lstat>>;
+  try {
+    fileStats = await lstat(filePath);
+  } catch (error) {
+    return { status: "failed", reason: `stat failed: ${String(error)}` };
+  }
+  if (fileStats.isSymbolicLink() || !fileStats.isFile()) {
+    return { status: "failed", reason: "captured-log path is not a regular private file" };
+  }
+
+  const initialSignature = signatureFromStats(fileStats);
+  const safeSignature = context.safeFiles.get(filePath);
+  if (safeSignature !== undefined && signaturesMatch(safeSignature, initialSignature)) {
+    return { status: "clean", signature: initialSignature };
+  }
+
+  let handle: Awaited<ReturnType<typeof open>> | null = null;
+  try {
+    handle = await open(filePath, constants.O_RDWR | constants.O_NOFOLLOW);
+    const openedStats = await handle.stat();
+    if (!openedStats.isFile()) {
+      return { status: "failed", reason: "captured-log handle is not a regular file" };
+    }
+    const openedSignature = signatureFromStats(openedStats);
+    if (!signaturesMatch(initialSignature, openedSignature)) {
+      return {
+        status: "failed",
+        reason: "captured-log path changed between metadata validation and open",
+      };
+    }
+    const scan = await scanFile(handle, openedStats.size);
+    const scannedSignature = signatureFromStats(await handle.stat());
+    if (!signaturesMatch(openedSignature, scannedSignature)) {
+      return {
+        status: "failed",
+        reason: "captured-log file changed while recovery was scanning it",
+      };
+    }
+    switch (scan.status) {
+      case "failed":
+        return scan;
+      case "interior-corruption":
+        return scan;
+      case "complete": {
+        if (scan.pending.length === 0) {
+          return { status: "clean", signature: signatureFromStats(await handle.stat()) };
+        }
+
+        const validation = validateRecord(scan.pending);
+        if (validation.valid) {
+          const write = await handle.write(Buffer.from("\n", "utf-8"), 0, 1, openedStats.size);
+          if (write.bytesWritten !== 1) {
+            return { status: "failed", reason: "captured-log delimiter append was incomplete" };
+          }
+          await handle.sync();
+          return {
+            status: "delimiter-appended",
+            signature: signatureFromStats(await handle.stat()),
+          };
+        }
+
+        const quarantinePath = await quarantineTail(
+          context,
+          filePath,
+          scan.pending,
+          scan.pendingOffset,
+        );
+        if (quarantinePath === null) {
+          return { status: "failed", reason: "invalid tail could not be privately quarantined" };
+        }
+        const beforeTruncate = signatureFromStats(await handle.stat());
+        if (!signaturesMatch(openedSignature, beforeTruncate)) {
+          return {
+            status: "failed",
+            reason: "captured-log file changed before invalid-tail truncation",
+          };
+        }
+        await handle.truncate(scan.pendingOffset);
+        await handle.sync();
+        return {
+          status: "tail-quarantined",
+          signature: signatureFromStats(await handle.stat()),
+        };
+      }
+    }
+    return { status: "failed", reason: "unsupported captured-log scan state" };
+  } catch (error) {
+    return { status: "failed", reason: String(error) };
+  } finally {
+    await handle?.close().catch(() => undefined);
+  }
+}
+
+function applyFileResult(
+  context: RecoveryContext,
+  filePath: string,
+  result: FileRecoveryResult,
+): void {
+  const file = basename(filePath);
+  switch (result.status) {
+    case "clean":
+      context.safeFiles.set(filePath, result.signature);
+      return;
+    case "delimiter-appended":
+      context.safeFiles.set(filePath, result.signature);
+      context.health.delimiterRepairs += 1;
+      context.health.repairedFiles += 1;
+      return;
+    case "tail-quarantined":
+      context.safeFiles.set(filePath, result.signature);
+      context.health.quarantinedTails += 1;
+      context.health.repairedFiles += 1;
+      if (context.health.level !== "unavailable") {
+        context.health.level = "degraded";
+        context.health.diagnostic = `Recovered and quarantined an invalid JSONL tail in ${file}`;
+      }
+      return;
+    case "interior-corruption":
+      context.safeFiles.delete(filePath);
+      context.health.level = "unavailable";
+      context.health.diagnostic =
+        `JSONL corruption in ${file} at line ${String(result.line)} ` +
+        `(offset ${String(result.byteOffset)}, ${result.reason}, sha256 ${result.recordHash})`;
+      return;
+    case "failed":
+      context.safeFiles.delete(filePath);
+      context.health.level = "unavailable";
+      context.health.diagnostic = `JSONL recovery failed for ${file}: ${result.reason}`;
+  }
+}
+
+async function runStartupRecovery(context: RecoveryContext): Promise<CapturedLogRecoveryHealth> {
+  if (!ensurePrivateDirectorySync(context.root)) {
+    context.health.level = "unavailable";
+    context.health.diagnostic = "Captured-log directory could not be made private";
+    return copyHealth(context.health);
+  }
+
+  let entries: string[];
+  try {
+    entries = await readdir(context.root);
+  } catch (error) {
+    context.health.level = "unavailable";
+    context.health.diagnostic = `Captured-log directory could not be listed: ${String(error)}`;
+    return copyHealth(context.health);
+  }
+
+  const files = entries.filter((entry) => entry.endsWith(".jsonl")).sort();
+  for (const file of files) {
+    const filePath = join(context.root, file);
+    const result = await recoverFile(context, filePath);
+    applyFileResult(context, filePath, result);
+    if (context.health.level === "unavailable") break;
+  }
+  return copyHealth(context.health);
+}
+
+/** Recover all captured-log ledgers once before startup reads, indexing, readiness, or appends. */
+export async function recoverCapturedLogStorage(
+  logDir: string,
+): Promise<CapturedLogRecoveryHealth> {
+  const context = contextFor(logDir);
+  if (context.startup === null) {
+    context.startup = runStartupRecovery(context).then((health) => {
+      context.startupComplete = true;
+      return health;
+    });
+  }
+  return await context.startup;
+}
+
+/** Re-check a ledger immediately before appending; unchanged files use a stat-only fast path. */
+export async function recoverCapturedLogFileBeforeAppend(
+  logDir: string,
+  filePath: string,
+): Promise<CapturedLogRecoveryHealth> {
+  const context = contextFor(logDir);
+  await recoverCapturedLogStorage(logDir);
+  if (context.health.level === "unavailable") return copyHealth(context.health);
+
+  const result = await recoverFile(context, filePath);
+  applyFileResult(context, filePath, result);
+  return copyHealth(context.health);
+}
+
+/** Refresh the trusted signature after this process successfully appends a complete JSONL record. */
+export async function markCapturedLogFileAppended(logDir: string, filePath: string): Promise<void> {
+  const context = contextFor(logDir);
+  try {
+    const fileStats = await stat(filePath);
+    context.safeFiles.set(filePath, signatureFromStats(fileStats));
+  } catch {
+    context.safeFiles.delete(filePath);
+  }
+}
+
+export function getCapturedLogRecoveryHealth(logDir: string): CapturedLogRecoveryHealth {
+  return copyHealth(contextFor(logDir).health);
+}
+
+/** Fail-closed guard for the process-exit writer, where asynchronous recovery is unavailable. */
+export function canAppendCapturedLogFileOnExit(logDir: string, filePath: string): boolean {
+  const context = contextFor(logDir);
+  if (!context.startupComplete || context.health.level === "unavailable") return false;
+  const trusted = context.safeFiles.get(filePath);
+  if (trusted === undefined) return false;
+
+  try {
+    const observed = lstatSync(filePath);
+    if (observed === undefined || observed.isSymbolicLink() || !observed.isFile()) return false;
+    return signaturesMatch(trusted, signatureFromStats(observed));
+  } catch {
+    return false;
+  }
+}
+
+export function _resetCapturedLogRecoveryForTests(): void {
+  activeContext = null;
+}

package/src/proxy/logFinalizer.ts

@@ -13,7 +13,7 @@ import {
   getSessionProcessPendingCount,
   isSessionProcessAvailable,
 } from "./sessionProcess";
-import { finalizeLogUpdate } from "./store";
+import { persistFinalizedLogUpdate } from "./store";
 import { computeTokensPerSecond } from "./streamTiming";
 import { analyzeToolSchemaWarnings } from "./toolSchemaWarnings";
 import { isPlainRecord, safeGetOwnProperty } from "../lib/objectUtils";
@@ -37,6 +37,7 @@ export type FinalizeStreamingLogJob = BaseFinalizeLogJob & {
   collectStreamingChunks: boolean;
   firstChunkMs: number | null;
   totalStreamMs: number;
+  terminalError?: string | null;
 };
 
 export type FinalizeStreamAbortLogJob = BaseFinalizeLogJob & {
@@ -99,6 +100,10 @@ export function buildFileLogEntry(log: CapturedLog, upstreamUrl: string): Record
     clientProjectFolder: log.clientProjectFolder,
     streamingChunks: log.streamingChunks,
     streamingChunksPath: log.streamingChunksPath,
+    captureIncomplete: log.captureIncomplete ?? false,
+    droppedChunks: log.droppedChunks ?? 0,
+    droppedBytes: log.droppedBytes ?? 0,
+    captureIncompleteReason: log.captureIncompleteReason ?? null,
     warnings: log.warnings,
     error: log.error,
   };
@@ -198,7 +203,7 @@ async function runToolSchemaWarningAnalysis(job: ToolSchemaWarningAnalysisJob):
   if (updatedLog === null) return;
 
   appendLogEntry({ ...buildFileLogEntry(updatedLog, job.upstreamUrl), error: job.error });
-  finalizeLogUpdate(updatedLog);
+  await persistFinalizedLogUpdate(updatedLog);
 }
 
 function scheduleToolSchemaWarningAnalysis(job: ToolSchemaWarningAnalysisJob): void {
@@ -455,8 +460,13 @@ export function buildFinalizeLogResult(job: FinalizeLogJob): FinalizeLogResult {
   switch (job.type) {
     case "non-streaming":
       return finalizeNonStreaming(job, log);
-    case "streaming":
-      return finalizeStreaming(job, log);
+    case "streaming": {
+      const result = finalizeStreaming(job, log);
+      if (job.terminalError !== undefined && job.terminalError !== null) {
+        result.log.error = job.terminalError;
+      }
+      return result;
+    }
     case "stream-abort":
       return finalizeStreamAbort(job, log);
     default: {
@@ -472,13 +482,18 @@ export function buildFinalizeLogResult(job: FinalizeLogJob): FinalizeLogResult {
   }
 }
 
-export function commitFinalizeLogResult(result: FinalizeLogResult): void {
-  appendLogEntry({ ...buildFileLogEntry(result.log, result.upstreamUrl), error: result.error });
-  finalizeLogUpdate(result.log);
+export async function commitFinalizeLogResult(result: FinalizeLogResult): Promise<void> {
+  const persistedError =
+    result.log.captureIncompleteReason === "upstream-stream-error"
+      ? (result.log.error ?? result.error ?? null)
+      : (result.error ?? result.log.error ?? null);
+  result.log.error = persistedError;
+  appendLogEntry({ ...buildFileLogEntry(result.log, result.upstreamUrl), error: persistedError });
+  await persistFinalizedLogUpdate(result.log);
   scheduleToolSchemaWarningAnalysis({
     log: cloneLog(result.log),
     upstreamUrl: result.upstreamUrl,
-    error: result.error,
+    error: persistedError,
   });
   if (result.cleanupRawStreamPath !== undefined && result.cleanupRawStreamPath !== null) {
     try {
@@ -665,8 +680,8 @@ export function executeBuildInWorker(job: FinalizeLogJob): Promise<FinalizeLogRe
 }
 
 export function executeFinalizeLogJob(job: FinalizeLogJob): Promise<void> {
-  return resolveBuildPromise(job).then((result) => {
-    commitFinalizeLogResult(result);
+  return resolveBuildPromise(job).then(async (result) => {
+    await commitFinalizeLogResult(result);
     if (result.error !== null && result.error !== "Client aborted") {
       return Promise.reject(new Error(result.error));
     }

package/src/proxy/logImporter.ts

@@ -4,6 +4,7 @@ import JSZip from "jszip";
 import { z } from "zod";
 import {
   CapturedLogSchema,
+  LogIdSchema,
   StreamingChunkSchema,
   type ApiFormat,
   type CapturedLog,
@@ -58,7 +59,7 @@ const ManifestSchema = z
     exportedAt: z.string().optional(),
     mode: z.string().optional(),
     redacted: z.boolean().optional(),
-    logIds: z.array(z.number().int().positive()).optional(),
+    logIds: z.array(LogIdSchema).optional(),
   })
   .passthrough();