@tonyclaw/agent-inspector 3.1.32 → 4.0.2

package/src/proxy/jsonlRecovery.ts

@@ -1,554 +0,0 @@
-import { createHash, randomUUID } from "node:crypto";
-import { Buffer } from "node:buffer";
-import { constants, existsSync, lstatSync } from "node:fs";
-import { link, lstat, open, readFile, readdir, stat, unlink } from "node:fs/promises";
-import { basename, join, resolve } from "node:path";
-import { CapturedLogSchema } from "./schemas";
-import {
-  ensurePrivateDirectorySync,
-  PRIVATE_FILE_MODE,
-  securePrivateFileSync,
-} from "./privateDataPath";
-
-const READ_CHUNK_BYTES = 64 * 1024;
-const QUARANTINE_DIRECTORY = "quarantine";
-
-export type CapturedLogRecoveryLevel = "ok" | "degraded" | "unavailable";
-
-export type CapturedLogRecoveryHealth = {
-  level: CapturedLogRecoveryLevel;
-  diagnostic: string | null;
-  repairedFiles: number;
-  delimiterRepairs: number;
-  quarantinedTails: number;
-};
-
-type FileSignature = {
-  size: number;
-  modifiedAt: number;
-  changedAt: number;
-  device: number;
-  inode: number;
-};
-
-type RecoveryContext = {
-  root: string;
-  health: CapturedLogRecoveryHealth;
-  startup: Promise<CapturedLogRecoveryHealth> | null;
-  startupComplete: boolean;
-  safeFiles: Map<string, FileSignature>;
-};
-
-type RecordValidation =
-  | { valid: true }
-  | { valid: false; reason: "invalid-json" | "invalid-log-schema" };
-
-type InvalidRecordReason = "invalid-json" | "invalid-log-schema";
-
-type FileRecoveryResult =
-  | { status: "clean"; signature: FileSignature }
-  | { status: "delimiter-appended"; signature: FileSignature }
-  | { status: "tail-quarantined"; signature: FileSignature }
-  | {
-      status: "interior-corruption";
-      line: number;
-      byteOffset: number;
-      recordHash: string;
-      reason: InvalidRecordReason;
-    }
-  | { status: "failed"; reason: string };
-
-type ScanResult =
-  | {
-      status: "complete";
-      pending: Buffer;
-      pendingOffset: number;
-      nextLine: number;
-    }
-  | {
-      status: "interior-corruption";
-      line: number;
-      byteOffset: number;
-      recordHash: string;
-      reason: InvalidRecordReason;
-    }
-  | { status: "failed"; reason: string };
-
-type QuarantineRecord = {
-  schemaVersion: 1;
-  reason: "invalid-final-jsonl-record";
-  sourceFile: string;
-  sourcePathHash: string;
-  recoveredAt: string;
-  tailOffset: number;
-  tailBytes: number;
-  tailSha256: string;
-  tailBase64: string;
-};
-
-type ExistingQuarantineState = "missing" | "matching" | "conflicting";
-
-let activeContext: RecoveryContext | null = null;
-
-function emptyHealth(): CapturedLogRecoveryHealth {
-  return {
-    level: "ok",
-    diagnostic: null,
-    repairedFiles: 0,
-    delimiterRepairs: 0,
-    quarantinedTails: 0,
-  };
-}
-
-function contextFor(logDir: string): RecoveryContext {
-  const root = resolve(logDir);
-  if (activeContext === null || activeContext.root !== root) {
-    activeContext = {
-      root,
-      health: emptyHealth(),
-      startup: null,
-      startupComplete: false,
-      safeFiles: new Map(),
-    };
-  }
-  return activeContext;
-}
-
-function copyHealth(health: CapturedLogRecoveryHealth): CapturedLogRecoveryHealth {
-  return { ...health };
-}
-
-function hashBytes(value: Uint8Array): string {
-  return createHash("sha256").update(value).digest("hex");
-}
-
-function signatureFromStats(stats: {
-  size: number;
-  mtimeMs: number;
-  ctimeMs: number;
-  dev: number;
-  ino: number;
-}): FileSignature {
-  return {
-    size: stats.size,
-    modifiedAt: stats.mtimeMs,
-    changedAt: stats.ctimeMs,
-    device: stats.dev,
-    inode: stats.ino,
-  };
-}
-
-function signaturesMatch(left: FileSignature, right: FileSignature): boolean {
-  return (
-    left.size === right.size &&
-    left.modifiedAt === right.modifiedAt &&
-    left.changedAt === right.changedAt &&
-    left.device === right.device &&
-    left.inode === right.inode
-  );
-}
-
-function validateRecord(bytes: Buffer): RecordValidation {
-  const withoutCarriageReturn =
-    bytes.length > 0 && bytes[bytes.length - 1] === 0x0d ? bytes.subarray(0, -1) : bytes;
-  const text = withoutCarriageReturn.toString("utf-8");
-  if (text.trim() === "") return { valid: true };
-
-  let parsed: unknown;
-  try {
-    parsed = JSON.parse(text);
-  } catch {
-    return { valid: false, reason: "invalid-json" };
-  }
-  return CapturedLogSchema.safeParse(parsed).success
-    ? { valid: true }
-    : { valid: false, reason: "invalid-log-schema" };
-}
-
-async function scanFile(
-  handle: Awaited<ReturnType<typeof open>>,
-  size: number,
-): Promise<ScanResult> {
-  const readBuffer = Buffer.alloc(READ_CHUNK_BYTES);
-  let position = 0;
-  let pending = Buffer.alloc(0);
-  let pendingOffset = 0;
-  let line = 1;
-
-  while (position < size) {
-    const length = Math.min(READ_CHUNK_BYTES, size - position);
-    let bytesRead: number;
-    try {
-      const result = await handle.read(readBuffer, 0, length, position);
-      bytesRead = result.bytesRead;
-    } catch (error) {
-      return { status: "failed", reason: `read failed: ${String(error)}` };
-    }
-    if (bytesRead === 0) {
-      return { status: "failed", reason: "captured-log file changed while being scanned" };
-    }
-
-    const chunk = readBuffer.subarray(0, bytesRead);
-    const combined = pending.length === 0 ? Buffer.from(chunk) : Buffer.concat([pending, chunk]);
-    let consumed = 0;
-    let newline = combined.indexOf(0x0a, consumed);
-    while (newline !== -1) {
-      const record = combined.subarray(consumed, newline);
-      const validation = validateRecord(record);
-      if (!validation.valid) {
-        return {
-          status: "interior-corruption",
-          line,
-          byteOffset: pendingOffset + consumed,
-          recordHash: hashBytes(record),
-          reason: validation.reason,
-        };
-      }
-      consumed = newline + 1;
-      line += 1;
-      newline = combined.indexOf(0x0a, consumed);
-    }
-
-    pending = Buffer.from(combined.subarray(consumed));
-    pendingOffset += consumed;
-    position += bytesRead;
-  }
-
-  return { status: "complete", pending, pendingOffset, nextLine: line };
-}
-
-async function syncDirectory(directory: string): Promise<void> {
-  let handle: Awaited<ReturnType<typeof open>> | null = null;
-  try {
-    handle = await open(directory, "r");
-    await handle.sync();
-  } catch {
-    // Directory fsync is unavailable on some supported Windows filesystems.
-  } finally {
-    await handle?.close().catch(() => undefined);
-  }
-}
-
-async function removeTemporaryFile(filePath: string): Promise<void> {
-  await unlink(filePath).catch(() => undefined);
-}
-
-function readOwnProperty(value: unknown, name: string): unknown {
-  if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined;
-  return Object.getOwnPropertyDescriptor(value, name)?.value;
-}
-
-async function inspectExistingQuarantine(
-  filePath: string,
-  expected: QuarantineRecord,
-): Promise<ExistingQuarantineState> {
-  if (!existsSync(filePath)) return "missing";
-  try {
-    const parsed: unknown = JSON.parse(await readFile(filePath, "utf-8"));
-    return readOwnProperty(parsed, "schemaVersion") === expected.schemaVersion &&
-      readOwnProperty(parsed, "reason") === expected.reason &&
-      readOwnProperty(parsed, "sourceFile") === expected.sourceFile &&
-      readOwnProperty(parsed, "sourcePathHash") === expected.sourcePathHash &&
-      readOwnProperty(parsed, "tailOffset") === expected.tailOffset &&
-      readOwnProperty(parsed, "tailBytes") === expected.tailBytes &&
-      readOwnProperty(parsed, "tailSha256") === expected.tailSha256 &&
-      readOwnProperty(parsed, "tailBase64") === expected.tailBase64
-      ? "matching"
-      : "conflicting";
-  } catch {
-    return "conflicting";
-  }
-}
-
-async function quarantineTail(
-  context: RecoveryContext,
-  sourceFile: string,
-  tail: Buffer,
-  tailOffset: number,
-): Promise<string | null> {
-  const quarantineDirectory = join(context.root, QUARANTINE_DIRECTORY);
-  if (!ensurePrivateDirectorySync(quarantineDirectory)) return null;
-
-  const tailSha256 = hashBytes(tail);
-  const sourceFileName = basename(sourceFile);
-  const destination = join(quarantineDirectory, `${sourceFileName}.tail-${tailSha256}.json`);
-  const temporary = join(
-    quarantineDirectory,
-    `.${sourceFileName}.${String(process.pid)}.${randomUUID()}.tmp`,
-  );
-  const record: QuarantineRecord = {
-    schemaVersion: 1,
-    reason: "invalid-final-jsonl-record",
-    sourceFile: sourceFileName,
-    sourcePathHash: hashBytes(Buffer.from(resolve(sourceFile), "utf-8")),
-    recoveredAt: new Date().toISOString(),
-    tailOffset,
-    tailBytes: tail.length,
-    tailSha256,
-    tailBase64: tail.toString("base64"),
-  };
-
-  const existing = await inspectExistingQuarantine(destination, record);
-  if (existing === "matching") {
-    return securePrivateFileSync(destination) ? destination : null;
-  }
-  if (existing === "conflicting") return null;
-
-  let handle: Awaited<ReturnType<typeof open>> | null = null;
-  try {
-    handle = await open(temporary, "wx", PRIVATE_FILE_MODE);
-    await handle.writeFile(`${JSON.stringify(record)}\n`, "utf-8");
-    await handle.sync();
-    await handle.close();
-    handle = null;
-    if (!securePrivateFileSync(temporary)) {
-      await removeTemporaryFile(temporary);
-      return null;
-    }
-    // A hard-link install has create-if-absent semantics on every supported platform. Unlike
-    // rename(), it cannot overwrite a conflicting quarantine artifact that appears after the
-    // preflight check on POSIX.
-    await link(temporary, destination);
-    await unlink(temporary);
-    if (!securePrivateFileSync(destination)) return null;
-    await syncDirectory(quarantineDirectory);
-    return destination;
-  } catch {
-    await handle?.close().catch(() => undefined);
-    await removeTemporaryFile(temporary);
-    const racedExisting = await inspectExistingQuarantine(destination, record);
-    return racedExisting === "matching" && securePrivateFileSync(destination) ? destination : null;
-  }
-}
-
-async function recoverFile(
-  context: RecoveryContext,
-  filePath: string,
-): Promise<FileRecoveryResult> {
-  if (!existsSync(filePath)) {
-    context.safeFiles.delete(filePath);
-    return {
-      status: "clean",
-      signature: { size: 0, modifiedAt: 0, changedAt: 0, device: 0, inode: 0 },
-    };
-  }
-
-  let fileStats: Awaited<ReturnType<typeof lstat>>;
-  try {
-    fileStats = await lstat(filePath);
-  } catch (error) {
-    return { status: "failed", reason: `stat failed: ${String(error)}` };
-  }
-  if (fileStats.isSymbolicLink() || !fileStats.isFile()) {
-    return { status: "failed", reason: "captured-log path is not a regular private file" };
-  }
-
-  const initialSignature = signatureFromStats(fileStats);
-  const safeSignature = context.safeFiles.get(filePath);
-  if (safeSignature !== undefined && signaturesMatch(safeSignature, initialSignature)) {
-    return { status: "clean", signature: initialSignature };
-  }
-
-  let handle: Awaited<ReturnType<typeof open>> | null = null;
-  try {
-    handle = await open(filePath, constants.O_RDWR | constants.O_NOFOLLOW);
-    const openedStats = await handle.stat();
-    if (!openedStats.isFile()) {
-      return { status: "failed", reason: "captured-log handle is not a regular file" };
-    }
-    const openedSignature = signatureFromStats(openedStats);
-    if (!signaturesMatch(initialSignature, openedSignature)) {
-      return {
-        status: "failed",
-        reason: "captured-log path changed between metadata validation and open",
-      };
-    }
-    const scan = await scanFile(handle, openedStats.size);
-    const scannedSignature = signatureFromStats(await handle.stat());
-    if (!signaturesMatch(openedSignature, scannedSignature)) {
-      return {
-        status: "failed",
-        reason: "captured-log file changed while recovery was scanning it",
-      };
-    }
-    switch (scan.status) {
-      case "failed":
-        return scan;
-      case "interior-corruption":
-        return scan;
-      case "complete": {
-        if (scan.pending.length === 0) {
-          return { status: "clean", signature: signatureFromStats(await handle.stat()) };
-        }
-
-        const validation = validateRecord(scan.pending);
-        if (validation.valid) {
-          const write = await handle.write(Buffer.from("\n", "utf-8"), 0, 1, openedStats.size);
-          if (write.bytesWritten !== 1) {
-            return { status: "failed", reason: "captured-log delimiter append was incomplete" };
-          }
-          await handle.sync();
-          return {
-            status: "delimiter-appended",
-            signature: signatureFromStats(await handle.stat()),
-          };
-        }
-
-        const quarantinePath = await quarantineTail(
-          context,
-          filePath,
-          scan.pending,
-          scan.pendingOffset,
-        );
-        if (quarantinePath === null) {
-          return { status: "failed", reason: "invalid tail could not be privately quarantined" };
-        }
-        const beforeTruncate = signatureFromStats(await handle.stat());
-        if (!signaturesMatch(openedSignature, beforeTruncate)) {
-          return {
-            status: "failed",
-            reason: "captured-log file changed before invalid-tail truncation",
-          };
-        }
-        await handle.truncate(scan.pendingOffset);
-        await handle.sync();
-        return {
-          status: "tail-quarantined",
-          signature: signatureFromStats(await handle.stat()),
-        };
-      }
-    }
-    return { status: "failed", reason: "unsupported captured-log scan state" };
-  } catch (error) {
-    return { status: "failed", reason: String(error) };
-  } finally {
-    await handle?.close().catch(() => undefined);
-  }
-}
-
-function applyFileResult(
-  context: RecoveryContext,
-  filePath: string,
-  result: FileRecoveryResult,
-): void {
-  const file = basename(filePath);
-  switch (result.status) {
-    case "clean":
-      context.safeFiles.set(filePath, result.signature);
-      return;
-    case "delimiter-appended":
-      context.safeFiles.set(filePath, result.signature);
-      context.health.delimiterRepairs += 1;
-      context.health.repairedFiles += 1;
-      return;
-    case "tail-quarantined":
-      context.safeFiles.set(filePath, result.signature);
-      context.health.quarantinedTails += 1;
-      context.health.repairedFiles += 1;
-      if (context.health.level !== "unavailable") {
-        context.health.level = "degraded";
-        context.health.diagnostic = `Recovered and quarantined an invalid JSONL tail in ${file}`;
-      }
-      return;
-    case "interior-corruption":
-      context.safeFiles.delete(filePath);
-      context.health.level = "unavailable";
-      context.health.diagnostic =
-        `JSONL corruption in ${file} at line ${String(result.line)} ` +
-        `(offset ${String(result.byteOffset)}, ${result.reason}, sha256 ${result.recordHash})`;
-      return;
-    case "failed":
-      context.safeFiles.delete(filePath);
-      context.health.level = "unavailable";
-      context.health.diagnostic = `JSONL recovery failed for ${file}: ${result.reason}`;
-  }
-}
-
-async function runStartupRecovery(context: RecoveryContext): Promise<CapturedLogRecoveryHealth> {
-  if (!ensurePrivateDirectorySync(context.root)) {
-    context.health.level = "unavailable";
-    context.health.diagnostic = "Captured-log directory could not be made private";
-    return copyHealth(context.health);
-  }
-
-  let entries: string[];
-  try {
-    entries = await readdir(context.root);
-  } catch (error) {
-    context.health.level = "unavailable";
-    context.health.diagnostic = `Captured-log directory could not be listed: ${String(error)}`;
-    return copyHealth(context.health);
-  }
-
-  const files = entries.filter((entry) => entry.endsWith(".jsonl")).sort();
-  for (const file of files) {
-    const filePath = join(context.root, file);
-    const result = await recoverFile(context, filePath);
-    applyFileResult(context, filePath, result);
-    if (context.health.level === "unavailable") break;
-  }
-  return copyHealth(context.health);
-}
-
-/** Recover all captured-log ledgers once before startup reads, indexing, readiness, or appends. */
-export async function recoverCapturedLogStorage(
-  logDir: string,
-): Promise<CapturedLogRecoveryHealth> {
-  const context = contextFor(logDir);
-  if (context.startup === null) {
-    context.startup = runStartupRecovery(context).then((health) => {
-      context.startupComplete = true;
-      return health;
-    });
-  }
-  return await context.startup;
-}
-
-/** Re-check a ledger immediately before appending; unchanged files use a stat-only fast path. */
-export async function recoverCapturedLogFileBeforeAppend(
-  logDir: string,
-  filePath: string,
-): Promise<CapturedLogRecoveryHealth> {
-  const context = contextFor(logDir);
-  await recoverCapturedLogStorage(logDir);
-  if (context.health.level === "unavailable") return copyHealth(context.health);
-
-  const result = await recoverFile(context, filePath);
-  applyFileResult(context, filePath, result);
-  return copyHealth(context.health);
-}
-
-/** Refresh the trusted signature after this process successfully appends a complete JSONL record. */
-export async function markCapturedLogFileAppended(logDir: string, filePath: string): Promise<void> {
-  const context = contextFor(logDir);
-  try {
-    const fileStats = await stat(filePath);
-    context.safeFiles.set(filePath, signatureFromStats(fileStats));
-  } catch {
-    context.safeFiles.delete(filePath);
-  }
-}
-
-export function getCapturedLogRecoveryHealth(logDir: string): CapturedLogRecoveryHealth {
-  return copyHealth(contextFor(logDir).health);
-}
-
-/** Fail-closed guard for the process-exit writer, where asynchronous recovery is unavailable. */
-export function canAppendCapturedLogFileOnExit(logDir: string, filePath: string): boolean {
-  const context = contextFor(logDir);
-  if (!context.startupComplete || context.health.level === "unavailable") return false;
-  const trusted = context.safeFiles.get(filePath);
-  if (trusted === undefined) return false;
-
-  try {
-    const observed = lstatSync(filePath);
-    if (observed === undefined || observed.isSymbolicLink() || !observed.isFile()) return false;
-    return signaturesMatch(trusted, signatureFromStats(observed));
-  } catch {
-    return false;
-  }
-}
-
-export function _resetCapturedLogRecoveryForTests(): void {
-  activeContext = null;
-}

package/src/proxy/logBodyChunks.ts

@@ -1,91 +0,0 @@
-import { Buffer } from "node:buffer";
-import type { CapturedLog, LogBodyChunk, LogBodyPart } from "../contracts";
-
-export const DEFAULT_LOG_BODY_CHUNK_BYTES = 256 * 1024;
-export const MAX_LOG_BODY_CHUNK_BYTES = 1024 * 1024;
-
-export function normalizeLogBodyChunkOffset(offset: number): number {
-  if (!Number.isInteger(offset) || offset < 0) return 0;
-  return offset;
-}
-
-export function normalizeLogBodyChunkLimit(limit: number): number {
-  if (!Number.isInteger(limit) || limit < 1) return DEFAULT_LOG_BODY_CHUNK_BYTES;
-  return Math.min(limit, MAX_LOG_BODY_CHUNK_BYTES);
-}
-
-function bodyTextForPart(log: CapturedLog, part: LogBodyPart): string | null {
-  switch (part) {
-    case "request":
-      return log.rawRequestBody;
-    case "response":
-      return log.responseText;
-  }
-}
-
-function isUtf8ContinuationByte(byte: number | undefined): boolean {
-  return byte !== undefined && (byte & 0b1100_0000) === 0b1000_0000;
-}
-
-function expandToUtf8Boundary(buffer: Buffer, end: number): number {
-  if (end >= buffer.byteLength) return buffer.byteLength;
-  let safeEnd = end;
-  while (safeEnd < buffer.byteLength && isUtf8ContinuationByte(buffer[safeEnd])) {
-    safeEnd++;
-  }
-  return safeEnd;
-}
-
-export function createLogBodyChunk({
-  log,
-  part,
-  offset,
-  limit,
-}: {
-  log: CapturedLog;
-  part: LogBodyPart;
-  offset: number;
-  limit: number;
-}): LogBodyChunk {
-  const text = bodyTextForPart(log, part);
-  const normalizedLimit = normalizeLogBodyChunkLimit(limit);
-  if (text === null || text === "") {
-    return {
-      logId: log.id,
-      part,
-      text: "",
-      offset: 0,
-      limit: normalizedLimit,
-      totalBytes: 0,
-      textBytes: 0,
-      nextOffset: null,
-      hasMore: false,
-      contentMode: "empty",
-    };
-  }
-
-  const bodyBytes = Buffer.from(text, "utf8");
-  const totalBytes = bodyBytes.byteLength;
-  const normalizedOffset = Math.min(normalizeLogBodyChunkOffset(offset), totalBytes);
-  const end = expandToUtf8Boundary(
-    bodyBytes,
-    Math.min(normalizedOffset + normalizedLimit, totalBytes),
-  );
-  const chunkText = bodyBytes.toString("utf8", normalizedOffset, end);
-  const textBytes = end - normalizedOffset;
-  const hasMore = end < totalBytes;
-  const contentMode = hasMore || normalizedOffset > 0 ? "partial" : "full";
-
-  return {
-    logId: log.id,
-    part,
-    text: chunkText,
-    offset: normalizedOffset,
-    limit: normalizedLimit,
-    totalBytes,
-    textBytes,
-    nextOffset: hasMore ? end : null,
-    hasMore,
-    contentMode,
-  };
-}