@tracemarketplace/cli 0.0.21 → 0.0.23

package/src/flush.test.ts

@@ -7,6 +7,7 @@ import {
   createFreshSessionState,
   migrateLegacySessionState,
   planSessionUploads,
+  prepareTraceForUpload,
   verifyUnconfirmedChunks,
 } from "./flush.js";
 import { migrateSessionUploadState } from "./config.js";
@@ -185,6 +186,35 @@ describe("planSessionUploads", () => {
   });
 });
 
+describe("prepareTraceForUpload", () => {
+  it("always omits raw session payloads before submit", () => {
+    const trace = {
+      ...makeTrace("session-raw", [
+        makeTurn("u1", "user", "2026-03-21T00:00:00.000Z"),
+        makeTurn("a1", "assistant", "2026-03-21T00:01:00.000Z", 10),
+      ]),
+      submitted_by: "user@example.com",
+      raw_json: {
+        events: [
+          {
+            type: "event_msg",
+            text: "/Users/fleet/project/secrets.txt",
+          },
+        ],
+      },
+      raw_json_format: "codex_cli.jsonl",
+    } satisfies NormalizedTrace;
+
+    const prepared = prepareTraceForUpload(trace, { homeDir: "/Users/fleet" });
+
+    expect(prepared.raw_json).toBeNull();
+    expect(prepared.raw_json_format).toBeNull();
+    expect(prepared.submitted_by).toBe("[redacted]");
+    expect(prepared.turn_count).toBe(trace.turn_count);
+    expect(prepared.turns).toHaveLength(trace.turns.length);
+  });
+});
+
 describe("migrateSessionUploadState", () => {
   it("fills in missing confirmation fields from next/openChunk values", () => {
     const legacy = {
@@ -231,7 +261,13 @@ describe("verifyUnconfirmedChunks", () => {
     return { version: 2 as const, chunks: {}, sessions };
   }
 
-  function makeMockClient(responses: Record<string, { exists: boolean }>) {
+  function makeMockClient(
+    responses: Record<string, {
+      exists: boolean;
+      pending?: boolean;
+      status?: "queued" | "running" | "retry_wait" | "completed" | "duplicate" | "failed_terminal" | "payload_expired" | "missing";
+    }>,
+  ) {
     return {
       async get(path: string) {
         const url = new URL(`http://x${path}`);
@@ -326,6 +362,58 @@ describe("verifyUnconfirmedChunks", () => {
     expect(s.unconfirmedSince).toBeNull();
   });
 
+  it("does not reset timed-out chunks that are still pending on the server", async () => {
+    const state = makeSubmitState({
+      "codex_cli:sess_pending": makeSessionState({
+        sourceTool: "codex_cli",
+        sourceSessionId: "sess_pending",
+        locator: "/tmp/sess_pending.jsonl",
+        nextChunkIndex: 3,
+        openChunkStartTurn: 12,
+        confirmedChunkIndex: 1,
+        confirmedOpenChunkStartTurn: 4,
+        unconfirmedSince: "2026-03-21T00:00:00.000Z",
+      }),
+    });
+
+    const client = makeMockClient({
+      "codex_cli:sess_pending:1": { exists: false, pending: true, status: "retry_wait" },
+    });
+
+    await verifyUnconfirmedChunks(state, client, new Date("2026-03-21T02:01:00.000Z"));
+
+    const s = state.sessions["codex_cli:sess_pending"]!;
+    expect(s.nextChunkIndex).toBe(3);
+    expect(s.openChunkStartTurn).toBe(12);
+    expect(s.unconfirmedSince).toBe("2026-03-21T00:00:00.000Z");
+  });
+
+  it("immediately resets when the server reports a terminal ingest failure", async () => {
+    const state = makeSubmitState({
+      "codex_cli:sess_failed": makeSessionState({
+        sourceTool: "codex_cli",
+        sourceSessionId: "sess_failed",
+        locator: "/tmp/sess_failed.jsonl",
+        nextChunkIndex: 3,
+        openChunkStartTurn: 12,
+        confirmedChunkIndex: 1,
+        confirmedOpenChunkStartTurn: 4,
+        unconfirmedSince: "2026-03-21T00:00:00.000Z",
+      }),
+    });
+
+    const client = makeMockClient({
+      "codex_cli:sess_failed:1": { exists: false, pending: false, status: "failed_terminal" },
+    });
+
+    await verifyUnconfirmedChunks(state, client, new Date("2026-03-21T00:05:00.000Z"));
+
+    const s = state.sessions["codex_cli:sess_failed"]!;
+    expect(s.nextChunkIndex).toBe(1);
+    expect(s.openChunkStartTurn).toBe(4);
+    expect(s.unconfirmedSince).toBeNull();
+  });
+
   it("skips sessions that are already fully confirmed", async () => {
     const state = makeSubmitState({
       "codex_cli:sess4": makeSessionState({

package/src/flush.ts

@@ -75,6 +75,23 @@ interface IngestResponse {
   trace_id?: string;
 }
 
+export function prepareTraceForUpload(
+  trace: NormalizedTrace,
+  options: { homeDir?: string } = {},
+): NormalizedTrace {
+  const { homeDir = homedir() } = options;
+
+  // Keep session-sized raw payloads local until they have a separate upload path.
+  return redactTrace(
+    {
+      ...trace,
+      raw_json: null,
+      raw_json_format: null,
+    },
+    { homeDir },
+  );
+}
+
 export function collectIdleSessionSources(
   sessions: Record<string, SessionUploadState>,
   now = new Date()
@@ -90,7 +107,10 @@ export function collectIdleSessionSources(
 
 interface ChunkExistsResponse {
   exists: boolean;
+  pending?: boolean;
+  status?: "queued" | "running" | "retry_wait" | "completed" | "duplicate" | "failed_terminal" | "payload_expired" | "missing";
   trace_id?: string;
+  retry_after_ms?: number;
 }
 
 export async function verifyUnconfirmedChunks(
@@ -100,20 +120,9 @@ export async function verifyUnconfirmedChunks(
 ): Promise<void> {
   for (const [key, session] of Object.entries(state.sessions)) {
     if (session.confirmedChunkIndex >= session.nextChunkIndex) continue;
-
-    // Check if timed out — reset to re-submit from last confirmed point
-    if (session.unconfirmedSince) {
-      const age = now.getTime() - Date.parse(session.unconfirmedSince);
-      if (age >= UNCONFIRMED_RESUBMIT_MS) {
-        state.sessions[key] = {
-          ...session,
-          nextChunkIndex: session.confirmedChunkIndex,
-          openChunkStartTurn: session.confirmedOpenChunkStartTurn,
-          unconfirmedSince: null,
-        };
-        continue;
-      }
-    }
+    const timedOut = session.unconfirmedSince
+      ? now.getTime() - Date.parse(session.unconfirmedSince) >= UNCONFIRMED_RESUBMIT_MS
+      : false;
 
     // Check each unconfirmed chunk sequentially — stop at first missing one
     for (let i = session.confirmedChunkIndex; i < session.nextChunkIndex; i++) {
@@ -124,20 +133,52 @@ export async function verifyUnconfirmedChunks(
           chunk_index: String(i),
         });
         const result = await client.get(`/api/v1/traces/exists?${params}`) as ChunkExistsResponse;
-        if (!result.exists) break;
-        state.sessions[key] = {
-          ...state.sessions[key]!,
-          confirmedChunkIndex: i + 1,
-          confirmedOpenChunkStartTurn: state.sessions[key]!.openChunkStartTurn,
-          unconfirmedSince: i + 1 >= session.nextChunkIndex ? null : state.sessions[key]!.unconfirmedSince,
-        };
+        if (result.exists) {
+          state.sessions[key] = {
+            ...state.sessions[key]!,
+            confirmedChunkIndex: i + 1,
+            confirmedOpenChunkStartTurn: state.sessions[key]!.openChunkStartTurn,
+            unconfirmedSince: i + 1 >= session.nextChunkIndex ? null : state.sessions[key]!.unconfirmedSince,
+          };
+          continue;
+        }
+
+        if (result.pending) {
+          break;
+        }
+
+        if (result.status === "failed_terminal" || result.status === "payload_expired") {
+          resetSessionToConfirmedBaseline(state, key, session);
+          break;
+        }
+
+        if (timedOut) {
+          resetSessionToConfirmedBaseline(state, key, session);
+        }
+        break;
       } catch {
+        if (timedOut) {
+          resetSessionToConfirmedBaseline(state, key, session);
+        }
         break;
       }
     }
   }
 }
 
+function resetSessionToConfirmedBaseline(
+  state: ReturnType<typeof loadState>,
+  key: string,
+  session: SessionUploadState,
+): void {
+  state.sessions[key] = {
+    ...session,
+    nextChunkIndex: session.confirmedChunkIndex,
+    openChunkStartTurn: session.confirmedOpenChunkStartTurn,
+    unconfirmedSince: null,
+  };
+}
+
 export async function flushTrackedSessions(
   config: Config,
   sources: SessionSource[],
@@ -400,7 +441,7 @@ async function uploadTraceChunk(
   sync = false,
 ): Promise<ChunkUploadResult> {
   // Client-side regex redaction runs before transmission; Presidio runs server-side async.
-  const payloadTrace = redactTrace(trace, { homeDir: homedir() });
+  const payloadTrace = prepareTraceForUpload(trace);
   const jsonSize = JSON.stringify({ trace: payloadTrace, source_tool: payloadTrace.source_tool }).length;
   console.error(`[upload] ${payloadTrace.source_session_id} payload=${Math.round(jsonSize/1024)}KB turns=${payloadTrace.turn_count}`);