@tonyclaw/agent-inspector 2.1.2 → 2.1.4

package/src/proxy/logBodyChunks.ts

@@ -0,0 +1,91 @@
+import { Buffer } from "node:buffer";
+import type { CapturedLog, LogBodyChunk, LogBodyPart } from "../contracts";
+
+export const DEFAULT_LOG_BODY_CHUNK_BYTES = 256 * 1024;
+export const MAX_LOG_BODY_CHUNK_BYTES = 1024 * 1024;
+
+export function normalizeLogBodyChunkOffset(offset: number): number {
+  if (!Number.isInteger(offset) || offset < 0) return 0;
+  return offset;
+}
+
+export function normalizeLogBodyChunkLimit(limit: number): number {
+  if (!Number.isInteger(limit) || limit < 1) return DEFAULT_LOG_BODY_CHUNK_BYTES;
+  return Math.min(limit, MAX_LOG_BODY_CHUNK_BYTES);
+}
+
+function bodyTextForPart(log: CapturedLog, part: LogBodyPart): string | null {
+  switch (part) {
+    case "request":
+      return log.rawRequestBody;
+    case "response":
+      return log.responseText;
+  }
+}
+
+function isUtf8ContinuationByte(byte: number | undefined): boolean {
+  return byte !== undefined && (byte & 0b1100_0000) === 0b1000_0000;
+}
+
+function expandToUtf8Boundary(buffer: Buffer, end: number): number {
+  if (end >= buffer.byteLength) return buffer.byteLength;
+  let safeEnd = end;
+  while (safeEnd < buffer.byteLength && isUtf8ContinuationByte(buffer[safeEnd])) {
+    safeEnd++;
+  }
+  return safeEnd;
+}
+
+export function createLogBodyChunk({
+  log,
+  part,
+  offset,
+  limit,
+}: {
+  log: CapturedLog;
+  part: LogBodyPart;
+  offset: number;
+  limit: number;
+}): LogBodyChunk {
+  const text = bodyTextForPart(log, part);
+  const normalizedLimit = normalizeLogBodyChunkLimit(limit);
+  if (text === null || text === "") {
+    return {
+      logId: log.id,
+      part,
+      text: "",
+      offset: 0,
+      limit: normalizedLimit,
+      totalBytes: 0,
+      textBytes: 0,
+      nextOffset: null,
+      hasMore: false,
+      contentMode: "empty",
+    };
+  }
+
+  const bodyBytes = Buffer.from(text, "utf8");
+  const totalBytes = bodyBytes.byteLength;
+  const normalizedOffset = Math.min(normalizeLogBodyChunkOffset(offset), totalBytes);
+  const end = expandToUtf8Boundary(
+    bodyBytes,
+    Math.min(normalizedOffset + normalizedLimit, totalBytes),
+  );
+  const chunkText = bodyBytes.toString("utf8", normalizedOffset, end);
+  const textBytes = end - normalizedOffset;
+  const hasMore = end < totalBytes;
+  const contentMode = hasMore || normalizedOffset > 0 ? "partial" : "full";
+
+  return {
+    logId: log.id,
+    part,
+    text: chunkText,
+    offset: normalizedOffset,
+    limit: normalizedLimit,
+    totalBytes,
+    textBytes,
+    nextOffset: hasMore ? end : null,
+    hasMore,
+    contentMode,
+  };
+}

package/src/proxy/logImporter.ts

@@ -0,0 +1,491 @@
+import { basename } from "node:path";
+import JSZip from "jszip";
+import { z } from "zod";
+import {
+  CapturedLogSchema,
+  StreamingChunkSchema,
+  type ApiFormat,
+  type CapturedLog,
+  type StreamingChunk,
+} from "../contracts";
+import packageJson from "../../package.json";
+import { apiFormatForPath } from "./formats";
+import { analyzeToolSchemaWarnings } from "./toolSchemaWarnings";
+
+export type ImportedLogDraft = Omit<CapturedLog, "id">;
+
+export type ParsedLogImport =
+  | {
+      ok: true;
+      logs: ImportedLogDraft[];
+      skipped: number;
+      warnings: string[];
+    }
+  | {
+      ok: false;
+      message: string;
+      warnings: string[];
+    };
+
+const ManifestSchema = z
+  .object({
+    version: z.string().optional(),
+    exportedAt: z.string().optional(),
+    mode: z.string().optional(),
+    redacted: z.boolean().optional(),
+    logIds: z.array(z.number().int().positive()).optional(),
+  })
+  .passthrough();
+
+const LogsArraySchema = z.array(CapturedLogSchema);
+const LogsEnvelopeSchema = z.object({ logs: z.array(CapturedLogSchema) }).passthrough();
+const StreamingChunksImportSchema = z.object({
+  chunks: z.array(StreamingChunkSchema),
+  truncated: z.boolean().optional().default(false),
+});
+
+type ParsedManifest = z.infer<typeof ManifestSchema>;
+
+const SEMVER_MAJOR_PATTERN = /^([0-9]+)\.[0-9]+\.[0-9]+(?:[-+].*)?$/;
+
+function readRecordProperty(value: unknown, name: string): unknown {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined;
+  const desc = Object.getOwnPropertyDescriptor(value, name);
+  return desc === undefined ? undefined : desc.value;
+}
+
+function parseJson(text: string | null): unknown {
+  if (text === null) return null;
+  try {
+    const parsed: unknown = JSON.parse(text);
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+
+function parseJsonLogs(text: string): CapturedLog[] | null {
+  const parsed = parseJson(text);
+  const direct = LogsArraySchema.safeParse(parsed);
+  if (direct.success) return direct.data;
+
+  const envelope = LogsEnvelopeSchema.safeParse(parsed);
+  if (envelope.success) return envelope.data.logs;
+
+  return null;
+}
+
+function parseJsonlLogs(text: string): { logs: CapturedLog[]; skipped: number } {
+  const logs: CapturedLog[] = [];
+  let skipped = 0;
+  for (const line of text.split(/\r?\n/)) {
+    if (line.trim() === "") continue;
+    const parsed = CapturedLogSchema.safeParse(parseJson(line));
+    if (parsed.success) {
+      logs.push(parsed.data);
+    } else {
+      skipped += 1;
+    }
+  }
+  return { logs, skipped };
+}
+
+function readStringProperty(value: unknown, name: string): string | null {
+  const property = readRecordProperty(value, name);
+  return typeof property === "string" ? property : null;
+}
+
+function hasProperty(value: unknown, name: string): boolean {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
+  return Object.getOwnPropertyDescriptor(value, name) !== undefined;
+}
+
+function modelFromRequestText(requestText: string | null): string | null {
+  return readStringProperty(parseJson(requestText), "model");
+}
+
+function inferApiFormatFromBody(
+  path: string,
+  model: string | null,
+  requestText: string | null,
+): ApiFormat {
+  const pathFormat = apiFormatForPath(path);
+  if (pathFormat !== "unknown") return pathFormat;
+
+  const normalizedModel = model?.toLowerCase() ?? "";
+  if (normalizedModel.startsWith("claude-")) return "anthropic";
+  if (
+    normalizedModel.startsWith("gpt-") ||
+    normalizedModel.startsWith("o1-") ||
+    normalizedModel.startsWith("o3-") ||
+    normalizedModel.startsWith("o4-") ||
+    normalizedModel.startsWith("deepseek-") ||
+    normalizedModel.startsWith("glm-") ||
+    normalizedModel.startsWith("qwen")
+  ) {
+    return "openai";
+  }
+
+  const parsedRequest = parseJson(requestText);
+  if (hasProperty(parsedRequest, "system") || hasProperty(parsedRequest, "max_tokens")) {
+    return "anthropic";
+  }
+  return "openai";
+}
+
+function defaultPathForApiFormat(apiFormat: ApiFormat): string {
+  switch (apiFormat) {
+    case "anthropic":
+      return "/v1/messages";
+    case "openai":
+      return "/v1/chat/completions";
+    case "unknown":
+      return "/v1/chat/completions";
+  }
+}
+
+function normalizeTimestamp(value: string, fallback: string): string {
+  return Number.isNaN(new Date(value).getTime()) ? fallback : value;
+}
+
+function responseStatusFromText(responseText: string | null): number | null {
+  if (responseText === null) return null;
+  const parsed = parseJson(responseText);
+  return hasProperty(parsed, "error") ? 500 : 200;
+}
+
+function mergeWarnings(
+  existing: readonly string[] | undefined,
+  generated: readonly string[],
+): string[] | undefined {
+  const values: string[] = [];
+  for (const warning of existing ?? []) {
+    if (!values.includes(warning)) values.push(warning);
+  }
+  for (const warning of generated) {
+    if (!values.includes(warning)) values.push(warning);
+  }
+  return values.length === 0 ? undefined : values;
+}
+
+function withToolWarnings(log: ImportedLogDraft): ImportedLogDraft {
+  const warnings = analyzeToolSchemaWarnings({
+    rawRequestBody: log.rawRequestBody,
+    responseText: log.responseText,
+    apiFormat: log.apiFormat,
+  });
+  return {
+    ...log,
+    warnings: mergeWarnings(log.warnings, warnings),
+  };
+}
+
+function streamingChunksFromText(text: string | null):
+  | {
+      chunks: StreamingChunk[];
+      truncated: boolean;
+    }
+  | undefined {
+  const parsed = StreamingChunksImportSchema.safeParse(parseJson(text));
+  return parsed.success ? parsed.data : undefined;
+}
+
+function draftFromCapturedLog(
+  log: CapturedLog,
+  requestText: string | null,
+  responseText: string | null,
+  streamingChunks: { chunks: StreamingChunk[]; truncated: boolean } | undefined,
+): ImportedLogDraft {
+  const rawRequestBody = requestText ?? log.rawRequestBody;
+  const response = responseText ?? log.responseText;
+  const model = log.model ?? modelFromRequestText(rawRequestBody);
+  const apiFormat =
+    log.apiFormat === "unknown"
+      ? inferApiFormatFromBody(log.path, model, rawRequestBody)
+      : log.apiFormat;
+  const path = log.path.trim() === "" ? defaultPathForApiFormat(apiFormat) : log.path;
+
+  return withToolWarnings({
+    timestamp: normalizeTimestamp(log.timestamp, new Date().toISOString()),
+    method: log.method.trim() === "" ? "POST" : log.method,
+    path,
+    model,
+    sessionId: null,
+    rawRequestBody,
+    responseStatus: log.responseStatus ?? responseStatusFromText(response),
+    responseText: response,
+    inputTokens: log.inputTokens,
+    outputTokens: log.outputTokens,
+    cacheCreationInputTokens: log.cacheCreationInputTokens,
+    cacheReadInputTokens: log.cacheReadInputTokens,
+    elapsedMs: log.elapsedMs,
+    firstChunkMs: log.firstChunkMs ?? null,
+    totalStreamMs: log.totalStreamMs ?? null,
+    tokensPerSecond: log.tokensPerSecond ?? null,
+    streaming: log.streaming || streamingChunks !== undefined,
+    userAgent: log.userAgent,
+    origin: log.origin,
+    rawHeaders: log.rawHeaders,
+    headers: log.headers,
+    apiFormat,
+    isTest: false,
+    replayOfLogId: null,
+    providerName: log.providerName ?? null,
+    clientPort: log.clientPort ?? null,
+    clientPid: log.clientPid ?? null,
+    clientCwd: log.clientCwd ?? null,
+    clientProjectFolder: log.clientProjectFolder ?? null,
+    streamingChunks,
+    streamingChunksPath: null,
+    rawRequestBodyBytes: log.rawRequestBodyBytes,
+    responseTextBytes: log.responseTextBytes,
+    bodyContentMode: "full",
+    warnings: log.warnings,
+    error: log.error ?? null,
+  });
+}
+
+function legacyDraftFromParts(input: {
+  exportedAt: string;
+  requestText: string | null;
+  responseText: string | null;
+  streamingChunks: { chunks: StreamingChunk[]; truncated: boolean } | undefined;
+}): ImportedLogDraft {
+  const model = modelFromRequestText(input.requestText);
+  const apiFormat = inferApiFormatFromBody("", model, input.requestText);
+  return withToolWarnings({
+    timestamp: input.exportedAt,
+    method: "POST",
+    path: defaultPathForApiFormat(apiFormat),
+    model,
+    sessionId: null,
+    rawRequestBody: input.requestText,
+    responseStatus: responseStatusFromText(input.responseText),
+    responseText: input.responseText,
+    inputTokens: null,
+    outputTokens: null,
+    cacheCreationInputTokens: null,
+    cacheReadInputTokens: null,
+    elapsedMs: null,
+    firstChunkMs: null,
+    totalStreamMs: null,
+    tokensPerSecond: null,
+    streaming: input.streamingChunks !== undefined,
+    userAgent: "agent-inspector-import",
+    origin: null,
+    apiFormat,
+    isTest: false,
+    replayOfLogId: null,
+    providerName: null,
+    clientPort: null,
+    clientPid: null,
+    clientCwd: null,
+    clientProjectFolder: null,
+    streamingChunks: input.streamingChunks,
+    streamingChunksPath: null,
+    bodyContentMode: "full",
+    error: null,
+  });
+}
+
+async function readZipText(zip: JSZip, path: string): Promise<string | null> {
+  const file = zip.file(path);
+  return file === null ? null : await file.async("text");
+}
+
+async function readManifest(zip: JSZip): Promise<ParsedManifest | null> {
+  const text = await readZipText(zip, "manifest.json");
+  const parsed = ManifestSchema.safeParse(parseJson(text));
+  return parsed.success ? parsed.data : null;
+}
+
+function versionMajor(version: string): number | null {
+  const match = SEMVER_MAJOR_PATTERN.exec(version);
+  if (match === null) return null;
+  const major = match[1];
+  return major === undefined ? null : Number.parseInt(major, 10);
+}
+
+function validateManifestVersion(manifest: ParsedManifest | null):
+  | {
+      ok: true;
+      warning: string | null;
+    }
+  | {
+      ok: false;
+      message: string;
+    } {
+  const exportedVersion = manifest?.version;
+  if (exportedVersion === undefined) {
+    return {
+      ok: true,
+      warning:
+        "Imported legacy export without version metadata. Compatibility could not be verified.",
+    };
+  }
+
+  const exportedMajor = versionMajor(exportedVersion);
+  if (exportedMajor === null) {
+    return {
+      ok: false,
+      message: `Unsupported export version "${exportedVersion}".`,
+    };
+  }
+
+  const currentMajor = versionMajor(packageJson.version);
+  if (currentMajor === null) {
+    return {
+      ok: true,
+      warning: `Current Agent Inspector version "${packageJson.version}" could not be verified against export version "${exportedVersion}".`,
+    };
+  }
+
+  if (exportedMajor !== currentMajor) {
+    return {
+      ok: false,
+      message: `Export version mismatch: archive was created by Agent Inspector v${exportedVersion}, current version is v${packageJson.version}. Import with a matching major version.`,
+    };
+  }
+
+  if (exportedVersion !== packageJson.version) {
+    return {
+      ok: true,
+      warning: `Imported export from Agent Inspector v${exportedVersion}; current version is v${packageJson.version}.`,
+    };
+  }
+
+  return { ok: true, warning: null };
+}
+
+async function parseZipLogImport(data: Uint8Array): Promise<ParsedLogImport> {
+  const warnings: string[] = [];
+  const zip = await JSZip.loadAsync(data);
+  const manifest = await readManifest(zip);
+  const versionCheck = validateManifestVersion(manifest);
+  if (!versionCheck.ok) {
+    return {
+      ok: false,
+      message: versionCheck.message,
+      warnings,
+    };
+  }
+  if (versionCheck.warning !== null) {
+    warnings.push(versionCheck.warning);
+  }
+
+  const exportedAt = normalizeTimestamp(manifest?.exportedAt ?? "", new Date().toISOString());
+  if (manifest?.redacted === true) {
+    warnings.push("Imported redacted export. Secrets and some replay details may be unavailable.");
+  }
+
+  const metadataText = await readZipText(zip, "logs.json");
+  const metadataLogs = metadataText === null ? null : parseJsonLogs(metadataText);
+  const logs: ImportedLogDraft[] = [];
+  let skipped = 0;
+
+  if (metadataLogs !== null) {
+    for (const log of metadataLogs) {
+      const requestText = await readZipText(zip, `#${String(log.id)}.Request.json`);
+      const responseText = await readZipText(zip, `#${String(log.id)}.Response.json`);
+      const streamingText = await readZipText(zip, `#${String(log.id)}.SSE.Response.json`);
+      logs.push(
+        draftFromCapturedLog(
+          log,
+          requestText,
+          responseText,
+          streamingChunksFromText(streamingText),
+        ),
+      );
+    }
+    return { ok: true, logs, skipped, warnings };
+  }
+
+  for (const originalId of manifest?.logIds ?? []) {
+    const requestText = await readZipText(zip, `#${String(originalId)}.Request.json`);
+    const responseText = await readZipText(zip, `#${String(originalId)}.Response.json`);
+    const streamingText = await readZipText(zip, `#${String(originalId)}.SSE.Response.json`);
+    if (requestText === null && responseText === null && streamingText === null) {
+      skipped += 1;
+      continue;
+    }
+    logs.push(
+      legacyDraftFromParts({
+        exportedAt,
+        requestText,
+        responseText,
+        streamingChunks: streamingChunksFromText(streamingText),
+      }),
+    );
+  }
+
+  if (logs.length === 0) {
+    return {
+      ok: false,
+      message: "No importable logs were found in the archive.",
+      warnings,
+    };
+  }
+
+  return { ok: true, logs, skipped, warnings };
+}
+
+function looksLikeZip(data: Uint8Array, fileName: string): boolean {
+  if (fileName.toLowerCase().endsWith(".zip")) return true;
+  return data[0] === 0x50 && data[1] === 0x4b;
+}
+
+function parseTextLogImport(text: string): ParsedLogImport {
+  const jsonLogs = parseJsonLogs(text);
+  if (jsonLogs !== null) {
+    return {
+      ok: true,
+      logs: jsonLogs.map((log) => draftFromCapturedLog(log, null, null, undefined)),
+      skipped: 0,
+      warnings: [],
+    };
+  }
+
+  const jsonl = parseJsonlLogs(text);
+  if (jsonl.logs.length > 0) {
+    return {
+      ok: true,
+      logs: jsonl.logs.map((log) => draftFromCapturedLog(log, null, null, undefined)),
+      skipped: jsonl.skipped,
+      warnings: [],
+    };
+  }
+
+  return {
+    ok: false,
+    message:
+      "Unsupported import file. Use an Agent Inspector export ZIP, logs.json, or JSONL file.",
+    warnings: [],
+  };
+}
+
+export async function parseImportedLogFile(input: {
+  fileName: string;
+  data: Uint8Array;
+}): Promise<ParsedLogImport> {
+  try {
+    if (looksLikeZip(input.data, input.fileName)) {
+      return await parseZipLogImport(input.data);
+    }
+
+    return parseTextLogImport(new TextDecoder("utf-8").decode(input.data));
+  } catch (err) {
+    return {
+      ok: false,
+      message: err instanceof Error ? err.message : "Failed to parse import file.",
+      warnings: [],
+    };
+  }
+}
+
+export function sessionNameFromImportFileName(fileName: string): string {
+  const normalized = fileName.replaceAll("\\", "/");
+  const base = basename(normalized)
+    .replace(/\.(zip|json|jsonl)$/i, "")
+    .trim();
+  return base === "" ? `import-${new Date().toISOString().slice(0, 10)}` : base;
+}

package/src/proxy/logIndex.ts

@@ -5,6 +5,14 @@ import { createInterface } from "node:readline";
 import { Buffer } from "node:buffer";
 import { resolveLogDir, logger } from "./logger";
 import type { ApiFormat, CapturedLog } from "./schemas";
+import {
+  findSqliteLogIndexEntry,
+  getSqliteLogIndexMaxId,
+  listSqliteLogIndexEntries,
+  replaceSqliteLogIndexEntries,
+  syncSqliteLogIndexEntries,
+  upsertSqliteLogIndexEntry,
+} from "./sqliteLogIndex";
 
 export type LogIndexSummary = {
   id: number;
@@ -72,6 +80,7 @@ function getIndexPath(): string {
 }
 
 let cachedIndex: LogIndex | null = null;
+let sqliteBackfillScheduledMaxId = 0;
 
 function textByteLength(value: string | null): number | null {
   return value === null ? null : Buffer.byteLength(value, "utf8");
@@ -191,13 +200,15 @@ export async function addToIndex(
   metadata?: LogIndexEntryMetadata,
 ): Promise<void> {
   const index = await loadIndex();
-  index.entries[id] =
+  const entry: LogIndexEntry =
     metadata === undefined
       ? { id, file, byteOffset, byteLength }
       : { id, file, byteOffset, byteLength, ...metadata };
+  index.entries[id] = entry;
   if (id > index.maxId) {
     index.maxId = id;
   }
+  void upsertSqliteLogIndexEntry(entry);
   // Defer disk writes to reduce I/O - flush after a batch of updates
   scheduleIndexFlush();
 }
@@ -232,15 +243,62 @@ export async function flushIndex(): Promise<void> {
 }
 
 export async function findInIndex(id: number): Promise<LogIndexEntry | null> {
+  const sqliteEntry = await findSqliteLogIndexEntry(id);
+  if (sqliteEntry !== null) return sqliteEntry;
+
   const index = await loadIndex();
   return index.entries[id] ?? null;
 }
 
 export async function listIndexEntries(): Promise<LogIndexEntry[]> {
   const index = await loadIndex();
+  const sqliteMaxId = await getSqliteLogIndexMaxId();
+  if (sqliteMaxId !== null && sqliteMaxId >= index.maxId) {
+    const sqliteEntries = await listSqliteLogIndexEntries();
+    if (sqliteEntries !== null) return sqliteEntries;
+  }
+  maybeBackfillSqliteIndex(index, sqliteMaxId);
   return Object.values(index.entries).sort((left, right) => left.id - right.id);
 }
 
+export async function listFilteredIndexEntries({
+  sessionId,
+  model,
+}: {
+  sessionId?: string;
+  model?: string;
+}): Promise<LogIndexEntry[]> {
+  const index = await loadIndex();
+  const sqliteMaxId = await getSqliteLogIndexMaxId();
+  if (sqliteMaxId !== null && sqliteMaxId >= index.maxId) {
+    const sqliteEntries = await listSqliteLogIndexEntries({ sessionId, model });
+    if (sqliteEntries !== null) return sqliteEntries;
+  }
+  maybeBackfillSqliteIndex(index, sqliteMaxId);
+
+  return Object.values(index.entries)
+    .sort((left, right) => left.id - right.id)
+    .filter((entry) => {
+      if (sessionId !== undefined && entry.sessionId !== sessionId) return false;
+      if (model !== undefined && entry.model !== model) return false;
+      return true;
+    });
+}
+
+function maybeBackfillSqliteIndex(index: LogIndex, sqliteMaxId: number | null): void {
+  if (sqliteMaxId === null) return;
+  if (index.maxId <= sqliteMaxId) return;
+  if (index.maxId <= sqliteBackfillScheduledMaxId) return;
+
+  sqliteBackfillScheduledMaxId = index.maxId;
+  const entries = Object.values(index.entries);
+  void syncSqliteLogIndexEntries(entries).then((ok) => {
+    if (!ok && sqliteMaxId < sqliteBackfillScheduledMaxId) {
+      sqliteBackfillScheduledMaxId = sqliteMaxId;
+    }
+  });
+}
+
 type FileIndexResult = {
   entries: Record<number, LogIndexEntry>;
   maxId: number;
@@ -386,6 +444,7 @@ export async function rebuildIndex(): Promise<LogIndex> {
   if (!existsSync(logDir)) {
     cachedIndex = newIndex;
     await saveIndex(newIndex);
+    await replaceSqliteLogIndexEntries([]);
     return newIndex;
   }
 
@@ -405,6 +464,7 @@ export async function rebuildIndex(): Promise<LogIndex> {
 
   cachedIndex = newIndex;
   await saveIndex(newIndex);
+  await replaceSqliteLogIndexEntries(Object.values(newIndex.entries));
   return newIndex;
 }
 
@@ -441,7 +501,8 @@ export async function getNextLogId(): Promise<number> {
   await acquireLock();
   try {
     const index = await loadIndex();
-    const nextId = index.maxId + 1;
+    const sqliteMaxId = await getSqliteLogIndexMaxId();
+    const nextId = Math.max(index.maxId, sqliteMaxId ?? 0) + 1;
     index.maxId = nextId;
     // Synchronously update the index in memory (disk write is deferred via batching)
     cachedIndex = index;