@tracemarketplace/cli 0.0.15 → 0.0.17

package/src/flush.test.ts

@@ -1,12 +1,15 @@
 import { describe, expect, it } from "vitest";
 import type { NormalizedTrace, Turn } from "@tracemarketplace/shared";
 import type { SessionSource } from "./flush.js";
+import type { SessionUploadState } from "./config.js";
 import {
   collectIdleSessionSources,
   createFreshSessionState,
   migrateLegacySessionState,
   planSessionUploads,
+  verifyUnconfirmedChunks,
 } from "./flush.js";
+import { migrateSessionUploadState } from "./config.js";
 
 function makeTurn(
   turnId: string,
@@ -81,6 +84,20 @@ function makeSource(tool: SessionSource["tool"], locator: string): SessionSource
   return { tool, locator, label: locator };
 }
 
+function makeSessionState(overrides: Partial<SessionUploadState> & Pick<SessionUploadState, "sourceTool" | "sourceSessionId" | "locator">): SessionUploadState {
+  return {
+    nextChunkIndex: 0,
+    openChunkStartTurn: 0,
+    lastSeenTurnCount: 0,
+    lastActivityAt: null,
+    lastFlushedTurnId: null,
+    confirmedChunkIndex: 0,
+    confirmedOpenChunkStartTurn: 0,
+    unconfirmedSince: null,
+    ...overrides,
+  };
+}
+
 describe("planSessionUploads", () => {
   it("flushes a sealed 100k chunk and keeps the tail pending", () => {
     const trace = makeTrace("session-100k", [
@@ -168,10 +185,174 @@ describe("planSessionUploads", () => {
   });
 });
 
+describe("migrateSessionUploadState", () => {
+  it("fills in missing confirmation fields from next/openChunk values", () => {
+    const legacy = {
+      sourceTool: "codex_cli" as const,
+      sourceSessionId: "s1",
+      locator: "/tmp/s1.jsonl",
+      nextChunkIndex: 3,
+      openChunkStartTurn: 10,
+      lastSeenTurnCount: 10,
+      lastActivityAt: "2026-03-21T00:00:00.000Z",
+      lastFlushedTurnId: "a3",
+      // missing: confirmedChunkIndex, confirmedOpenChunkStartTurn, unconfirmedSince
+    } as any;
+
+    const migrated = migrateSessionUploadState(legacy);
+
+    expect(migrated.confirmedChunkIndex).toBe(3);
+    expect(migrated.confirmedOpenChunkStartTurn).toBe(10);
+    expect(migrated.unconfirmedSince).toBeNull();
+  });
+
+  it("does not overwrite existing confirmation fields", () => {
+    const state = makeSessionState({
+      sourceTool: "codex_cli",
+      sourceSessionId: "s2",
+      locator: "/tmp/s2.jsonl",
+      nextChunkIndex: 5,
+      openChunkStartTurn: 20,
+      confirmedChunkIndex: 3,
+      confirmedOpenChunkStartTurn: 12,
+      unconfirmedSince: "2026-03-21T01:00:00.000Z",
+    });
+
+    const migrated = migrateSessionUploadState(state);
+
+    expect(migrated.confirmedChunkIndex).toBe(3);
+    expect(migrated.confirmedOpenChunkStartTurn).toBe(12);
+    expect(migrated.unconfirmedSince).toBe("2026-03-21T01:00:00.000Z");
+  });
+});
+
+describe("verifyUnconfirmedChunks", () => {
+  function makeSubmitState(sessions: Record<string, ReturnType<typeof makeSessionState>>) {
+    return { version: 2 as const, chunks: {}, sessions };
+  }
+
+  function makeMockClient(responses: Record<string, { exists: boolean }>) {
+    return {
+      async get(path: string) {
+        const url = new URL(`http://x${path}`);
+        const tool = url.searchParams.get("source_tool")!;
+        const id = url.searchParams.get("source_session_id")!;
+        const idx = url.searchParams.get("chunk_index")!;
+        const key = `${tool}:${id}:${idx}`;
+        const r = responses[key];
+        if (!r) throw new Error(`Unexpected exists check: ${key}`);
+        return r;
+      },
+    } as any;
+  }
+
+  it("advances confirmedChunkIndex when chunks are confirmed", async () => {
+    const state = makeSubmitState({
+      "codex_cli:sess1": makeSessionState({
+        sourceTool: "codex_cli",
+        sourceSessionId: "sess1",
+        locator: "/tmp/sess1.jsonl",
+        nextChunkIndex: 3,
+        openChunkStartTurn: 12,
+        confirmedChunkIndex: 1,
+        confirmedOpenChunkStartTurn: 4,
+        unconfirmedSince: "2026-03-21T00:00:00.000Z",
+      }),
+    });
+
+    const client = makeMockClient({
+      "codex_cli:sess1:1": { exists: true },
+      "codex_cli:sess1:2": { exists: true },
+    });
+
+    await verifyUnconfirmedChunks(state, client, new Date("2026-03-21T01:00:00.000Z"));
+
+    const s = state.sessions["codex_cli:sess1"]!;
+    expect(s.confirmedChunkIndex).toBe(3);
+    // all confirmed → unconfirmedSince cleared
+    expect(s.unconfirmedSince).toBeNull();
+  });
+
+  it("stops advancing at first missing chunk", async () => {
+    const state = makeSubmitState({
+      "codex_cli:sess2": makeSessionState({
+        sourceTool: "codex_cli",
+        sourceSessionId: "sess2",
+        locator: "/tmp/sess2.jsonl",
+        nextChunkIndex: 3,
+        openChunkStartTurn: 12,
+        confirmedChunkIndex: 1,
+        confirmedOpenChunkStartTurn: 4,
+        unconfirmedSince: "2026-03-21T00:00:00.000Z",
+      }),
+    });
+
+    const client = makeMockClient({
+      "codex_cli:sess2:1": { exists: true },
+      "codex_cli:sess2:2": { exists: false },
+    });
+
+    await verifyUnconfirmedChunks(state, client, new Date("2026-03-21T01:00:00.000Z"));
+
+    const s = state.sessions["codex_cli:sess2"]!;
+    expect(s.confirmedChunkIndex).toBe(2);
+    // still one unconfirmed chunk → unconfirmedSince preserved
+    expect(s.unconfirmedSince).toBe("2026-03-21T00:00:00.000Z");
+  });
+
+  it("resets to confirmed state after 2hr timeout", async () => {
+    const state = makeSubmitState({
+      "codex_cli:sess3": makeSessionState({
+        sourceTool: "codex_cli",
+        sourceSessionId: "sess3",
+        locator: "/tmp/sess3.jsonl",
+        nextChunkIndex: 3,
+        openChunkStartTurn: 12,
+        confirmedChunkIndex: 1,
+        confirmedOpenChunkStartTurn: 4,
+        unconfirmedSince: "2026-03-21T00:00:00.000Z",
+      }),
+    });
+
+    const client = makeMockClient({}); // should not be called
+
+    // now = 2h01m after unconfirmedSince
+    await verifyUnconfirmedChunks(state, client, new Date("2026-03-21T02:01:00.000Z"));
+
+    const s = state.sessions["codex_cli:sess3"]!;
+    // reset back to confirmed baseline so chunks get re-submitted
+    expect(s.nextChunkIndex).toBe(1);
+    expect(s.openChunkStartTurn).toBe(4);
+    expect(s.unconfirmedSince).toBeNull();
+  });
+
+  it("skips sessions that are already fully confirmed", async () => {
+    const state = makeSubmitState({
+      "codex_cli:sess4": makeSessionState({
+        sourceTool: "codex_cli",
+        sourceSessionId: "sess4",
+        locator: "/tmp/sess4.jsonl",
+        nextChunkIndex: 2,
+        openChunkStartTurn: 8,
+        confirmedChunkIndex: 2,
+        confirmedOpenChunkStartTurn: 8,
+        unconfirmedSince: null,
+      }),
+    });
+
+    let callCount = 0;
+    const client = { async get() { callCount++; return { exists: true }; } } as any;
+
+    await verifyUnconfirmedChunks(state, client, new Date("2026-03-21T01:00:00.000Z"));
+
+    expect(callCount).toBe(0);
+  });
+});
+
 describe("collectIdleSessionSources", () => {
   it("returns only tracked sessions with an open tail older than two days", () => {
     const sources = collectIdleSessionSources({
-      "codex_cli:idle": {
+      "codex_cli:idle": makeSessionState({
         sourceTool: "codex_cli",
         sourceSessionId: "idle",
         locator: "/tmp/idle.jsonl",
@@ -180,8 +361,8 @@ describe("collectIdleSessionSources", () => {
         lastSeenTurnCount: 4,
         lastActivityAt: "2026-03-21T00:00:00.000Z",
         lastFlushedTurnId: "a1",
-      },
-      "codex_cli:closed": {
+      }),
+      "codex_cli:closed": makeSessionState({
         sourceTool: "codex_cli",
         sourceSessionId: "closed",
         locator: "/tmp/closed.jsonl",
@@ -190,8 +371,8 @@ describe("collectIdleSessionSources", () => {
         lastSeenTurnCount: 4,
         lastActivityAt: "2026-03-21T00:00:00.000Z",
         lastFlushedTurnId: "a2",
-      },
-      "codex_cli:fresh": {
+      }),
+      "codex_cli:fresh": makeSessionState({
         sourceTool: "codex_cli",
         sourceSessionId: "fresh",
         locator: "/tmp/fresh.jsonl",
@@ -200,7 +381,7 @@ describe("collectIdleSessionSources", () => {
         lastSeenTurnCount: 2,
         lastActivityAt: "2026-03-22T23:59:59.000Z",
         lastFlushedTurnId: null,
-      },
+      }),
     }, new Date("2026-03-23T00:01:00.000Z"));
 
     expect(sources).toEqual([

package/src/flush.ts

@@ -7,11 +7,11 @@ import {
   extractCursor,
   redactTrace,
   type NormalizedTrace,
-  type Turn,
 } from "@tracemarketplace/shared";
 import { ApiClient } from "./api-client.js";
 import {
   loadState,
+  migrateSessionUploadState,
   saveState,
   stateKey,
   type Config,
@@ -21,6 +21,8 @@ import {
 import { CURSOR_DB_PATH } from "./sessions.js";
 
 const IDLE_FINALIZATION_MS = 2 * 24 * 60 * 60 * 1000;
+const UNCONFIRMED_RESUBMIT_MS = 2 * 60 * 60 * 1000; // re-submit after 2hr with no confirmation
+const INGEST_CONCURRENCY = 12;
 
 export interface SessionSource {
   tool: TrackedSessionTool;
@@ -67,6 +69,12 @@ interface ChunkUploadResult {
   error?: string;
 }
 
+interface IngestResponse {
+  queued?: boolean;
+  duplicate?: boolean;
+  trace_id?: string;
+}
+
 export function collectIdleSessionSources(
   sessions: Record<string, SessionUploadState>,
   now = new Date()
@@ -80,10 +88,60 @@ export function collectIdleSessionSources(
     }));
 }
 
+interface ChunkExistsResponse {
+  exists: boolean;
+  trace_id?: string;
+}
+
+export async function verifyUnconfirmedChunks(
+  state: ReturnType<typeof loadState>,
+  client: ApiClient,
+  now: Date,
+): Promise<void> {
+  for (const [key, session] of Object.entries(state.sessions)) {
+    if (session.confirmedChunkIndex >= session.nextChunkIndex) continue;
+
+    // Check if timed out — reset to re-submit from last confirmed point
+    if (session.unconfirmedSince) {
+      const age = now.getTime() - Date.parse(session.unconfirmedSince);
+      if (age >= UNCONFIRMED_RESUBMIT_MS) {
+        state.sessions[key] = {
+          ...session,
+          nextChunkIndex: session.confirmedChunkIndex,
+          openChunkStartTurn: session.confirmedOpenChunkStartTurn,
+          unconfirmedSince: null,
+        };
+        continue;
+      }
+    }
+
+    // Check each unconfirmed chunk sequentially — stop at first missing one
+    for (let i = session.confirmedChunkIndex; i < session.nextChunkIndex; i++) {
+      try {
+        const params = new URLSearchParams({
+          source_tool: session.sourceTool,
+          source_session_id: session.sourceSessionId,
+          chunk_index: String(i),
+        });
+        const result = await client.get(`/api/v1/traces/exists?${params}`) as ChunkExistsResponse;
+        if (!result.exists) break;
+        state.sessions[key] = {
+          ...state.sessions[key]!,
+          confirmedChunkIndex: i + 1,
+          confirmedOpenChunkStartTurn: state.sessions[key]!.openChunkStartTurn,
+          unconfirmedSince: i + 1 >= session.nextChunkIndex ? null : state.sessions[key]!.unconfirmedSince,
+        };
+      } catch {
+        break;
+      }
+    }
+  }
+}
+
 export async function flushTrackedSessions(
   config: Config,
   sources: SessionSource[],
-  opts: { includeIdleTracked?: boolean; now?: Date } = {}
+  opts: { includeIdleTracked?: boolean; now?: Date; prefetchedTraces?: Map<string, NormalizedTrace> } = {}
 ): Promise<FlushResult> {
   const now = opts.now ?? new Date();
   const state = loadState(config.profile);
@@ -94,12 +152,23 @@ export async function flushTrackedSessions(
     ...(opts.includeIdleTracked ? collectIdleSessionSources(state.sessions, now) : []),
   ]);
 
-  const results: SessionFlushResult[] = [];
-
-  for (const source of allSources) {
-    results.push(await processSessionSource(source, state, config, client, now));
+  // Migrate any existing sessions that predate the confirmation fields
+  for (const [key, session] of Object.entries(state.sessions)) {
+    state.sessions[key] = migrateSessionUploadState(session);
   }
 
+  // Verify previously queued chunks and handle 2hr re-submit timeout
+  await verifyUnconfirmedChunks(state, client, now);
+
+  // Process sessions concurrently — each session's chunks stay sequential internally
+  const results = await pLimit(
+    allSources.map((source) => () => processSessionSource(
+      source, state, config, client, now,
+      opts.prefetchedTraces?.get(`${source.tool}:${source.locator}`),
+    )),
+    INGEST_CONCURRENCY,
+  );
+
   saveState(state, config.profile);
 
   return {
@@ -174,6 +243,9 @@ export function createFreshSessionState(
     lastSeenTurnCount: 0,
     lastActivityAt: null,
     lastFlushedTurnId: null,
+    confirmedChunkIndex: 0,
+    confirmedOpenChunkStartTurn: 0,
+    unconfirmedSince: null,
   };
 }
 
@@ -182,15 +254,20 @@ export function migrateLegacySessionState(
   trace: NormalizedTrace,
   legacyChunkIndex: number
 ): SessionUploadState {
+  const nextChunkIndex = legacyChunkIndex + 1;
+  const openChunkStartTurn = trace.turn_count;
   return {
     sourceTool: source.tool,
     sourceSessionId: trace.source_session_id,
     locator: source.locator,
-    nextChunkIndex: legacyChunkIndex + 1,
-    openChunkStartTurn: trace.turn_count,
+    nextChunkIndex,
+    openChunkStartTurn,
     lastSeenTurnCount: trace.turn_count,
     lastActivityAt: getLastActivityAt(trace),
     lastFlushedTurnId: trace.turns[trace.turn_count - 1]?.turn_id ?? null,
+    confirmedChunkIndex: nextChunkIndex,
+    confirmedOpenChunkStartTurn: openChunkStartTurn,
+    unconfirmedSince: null,
   };
 }
 
@@ -199,12 +276,13 @@ async function processSessionSource(
   state: ReturnType<typeof loadState>,
   config: Config,
   client: ApiClient,
-  now: Date
+  now: Date,
+  prefetchedTrace?: NormalizedTrace,
 ): Promise<SessionFlushResult> {
   let trace: NormalizedTrace;
 
   try {
-    trace = await extractTraceFromSource(source, config.email);
+    trace = prefetchedTrace ?? await extractTraceFromSource(source, config.email);
   } catch (err) {
     return {
       source,
@@ -250,7 +328,9 @@ async function processSessionSource(
   let payoutCents = 0;
 
   for (const upload of plan.uploads) {
+    const tUpload = Date.now();
     const result = await uploadTraceChunk(upload.trace, client);
+    console.error(`[flush] ${key} chunk${upload.trace.chunk_index} done in ${Date.now()-tUpload}ms err=${result.error?.slice(0,60) ?? 'none'}`);
     if (result.error) {
       state.sessions[key] = workingState;
       if (workingState.nextChunkIndex > 0) {
@@ -315,42 +395,27 @@ async function uploadTraceChunk(
   trace: NormalizedTrace,
   client: ApiClient
 ): Promise<ChunkUploadResult> {
+  // Client-side regex redaction runs before transmission; Presidio runs server-side async.
+  const payloadTrace = redactTrace(trace, { homeDir: homedir() });
+
   try {
-    const result = await client.post("/api/v1/traces/batch", {
-      traces: [redactTrace(trace, { homeDir: homedir() })],
-      source_tool: trace.source_tool,
-    }) as {
-      accepted: number;
-      duplicate: number;
-      traces: Array<{
-        error?: string;
-        is_duplicate?: boolean;
-        payout_cents?: number;
-        trace_id?: string;
-      }>;
-    };
+    const result = await client.post("/api/v1/traces/ingest", {
+      trace: payloadTrace,
+      source_tool: payloadTrace.source_tool,
+    }) as IngestResponse;
 
-    const traceResult = result.traces?.[0] ?? {};
-    if (traceResult.error) {
-      return {
-        duplicate: false,
-        payoutCents: 0,
-        traceId: traceResult.trace_id ?? null,
-        error: traceResult.error,
-      };
+    if (result.duplicate) {
+      return { duplicate: true, payoutCents: 0, traceId: result.trace_id ?? null };
     }
 
-    return {
-      duplicate: traceResult.is_duplicate === true || (result.duplicate > 0 && result.accepted === 0),
-      payoutCents: traceResult.payout_cents ?? 0,
-      traceId: traceResult.trace_id ?? null,
-    };
+    // 202 queued — payout credited asynchronously; show $0 until processed
+    return { duplicate: false, payoutCents: 0, traceId: null };
   } catch (err) {
     return {
       duplicate: false,
       payoutCents: 0,
       traceId: null,
-      error: `Submit failed: ${err}`,
+      error: formatSubmitFailure(err),
     };
   }
 }
@@ -379,6 +444,7 @@ function applyUploadedChunk(
   const nextOpenChunkStartTurn = (uploadedChunk.chunk_start_turn ?? cursor.openChunkStartTurn)
     + (uploadedChunk.turn_count ?? uploadedChunk.turns.length);
 
+  const nowIso = new Date().toISOString();
   return {
     ...cursor,
     nextChunkIndex: (uploadedChunk.chunk_index ?? cursor.nextChunkIndex) + 1,
@@ -386,6 +452,7 @@ function applyUploadedChunk(
     lastSeenTurnCount: trace.turn_count,
     lastActivityAt: getLastActivityAt(trace),
     lastFlushedTurnId: trace.turns[nextOpenChunkStartTurn - 1]?.turn_id ?? cursor.lastFlushedTurnId,
+    unconfirmedSince: cursor.unconfirmedSince ?? nowIso,
   };
 }
 
@@ -485,6 +552,25 @@ function dedupeSources(sources: SessionSource[]): SessionSource[] {
   return [...unique.values()];
 }
 
+function formatSubmitFailure(err: unknown): string {
+  return err instanceof Error ? `Submit failed: ${err.message}` : `Submit failed: ${String(err)}`;
+}
+
+async function pLimit<T>(tasks: Array<() => Promise<T>>, concurrency: number): Promise<T[]> {
+  const results: T[] = new Array(tasks.length);
+  let index = 0;
+
+  async function worker() {
+    while (index < tasks.length) {
+      const i = index++;
+      results[i] = await tasks[i]!();
+    }
+  }
+
+  await Promise.all(Array.from({ length: Math.min(concurrency, tasks.length) }, worker));
+  return results;
+}
+
 export function buildFileSessionSource(
   tool: "claude_code" | "codex_cli",
   filePath: string

package/src/commands/register.ts

@@ -1,8 +0,0 @@
-import chalk from "chalk";
-import { CLI_NAME } from "../constants.js";
-import { loginCommand } from "./login.js";
-
-export async function registerCommand(opts: { profile?: string; serverUrl?: string }): Promise<void> {
-  console.log(chalk.yellow(`\`${CLI_NAME} register\` is now an alias for \`${CLI_NAME} login\`.`));
-  await loginCommand({ profile: opts.profile, serverUrl: opts.serverUrl });
-}