@tracemarketplace/cli 0.0.15 → 0.0.18

package/src/flush.ts

@@ -7,11 +7,11 @@ import {
   extractCursor,
   redactTrace,
   type NormalizedTrace,
-  type Turn,
 } from "@tracemarketplace/shared";
 import { ApiClient } from "./api-client.js";
 import {
   loadState,
+  migrateSessionUploadState,
   saveState,
   stateKey,
   type Config,
@@ -21,6 +21,8 @@ import {
 import { CURSOR_DB_PATH } from "./sessions.js";
 
 const IDLE_FINALIZATION_MS = 2 * 24 * 60 * 60 * 1000;
+const UNCONFIRMED_RESUBMIT_MS = 2 * 60 * 60 * 1000; // re-submit after 2hr with no confirmation
+const INGEST_CONCURRENCY = 12;
 
 export interface SessionSource {
   tool: TrackedSessionTool;
@@ -67,6 +69,12 @@ interface ChunkUploadResult {
   error?: string;
 }
 
+interface IngestResponse {
+  queued?: boolean;
+  duplicate?: boolean;
+  trace_id?: string;
+}
+
 export function collectIdleSessionSources(
   sessions: Record<string, SessionUploadState>,
   now = new Date()
@@ -80,10 +88,60 @@ export function collectIdleSessionSources(
     }));
 }
 
+interface ChunkExistsResponse {
+  exists: boolean;
+  trace_id?: string;
+}
+
+export async function verifyUnconfirmedChunks(
+  state: ReturnType<typeof loadState>,
+  client: ApiClient,
+  now: Date,
+): Promise<void> {
+  for (const [key, session] of Object.entries(state.sessions)) {
+    if (session.confirmedChunkIndex >= session.nextChunkIndex) continue;
+
+    // Check if timed out — reset to re-submit from last confirmed point
+    if (session.unconfirmedSince) {
+      const age = now.getTime() - Date.parse(session.unconfirmedSince);
+      if (age >= UNCONFIRMED_RESUBMIT_MS) {
+        state.sessions[key] = {
+          ...session,
+          nextChunkIndex: session.confirmedChunkIndex,
+          openChunkStartTurn: session.confirmedOpenChunkStartTurn,
+          unconfirmedSince: null,
+        };
+        continue;
+      }
+    }
+
+    // Check each unconfirmed chunk sequentially — stop at first missing one
+    for (let i = session.confirmedChunkIndex; i < session.nextChunkIndex; i++) {
+      try {
+        const params = new URLSearchParams({
+          source_tool: session.sourceTool,
+          source_session_id: session.sourceSessionId,
+          chunk_index: String(i),
+        });
+        const result = await client.get(`/api/v1/traces/exists?${params}`) as ChunkExistsResponse;
+        if (!result.exists) break;
+        state.sessions[key] = {
+          ...state.sessions[key]!,
+          confirmedChunkIndex: i + 1,
+          confirmedOpenChunkStartTurn: state.sessions[key]!.openChunkStartTurn,
+          unconfirmedSince: i + 1 >= session.nextChunkIndex ? null : state.sessions[key]!.unconfirmedSince,
+        };
+      } catch {
+        break;
+      }
+    }
+  }
+}
+
 export async function flushTrackedSessions(
   config: Config,
   sources: SessionSource[],
-  opts: { includeIdleTracked?: boolean; now?: Date } = {}
+  opts: { includeIdleTracked?: boolean; now?: Date; prefetchedTraces?: Map<string, NormalizedTrace>; sync?: boolean } = {}
 ): Promise<FlushResult> {
   const now = opts.now ?? new Date();
   const state = loadState(config.profile);
@@ -94,12 +152,24 @@ export async function flushTrackedSessions(
     ...(opts.includeIdleTracked ? collectIdleSessionSources(state.sessions, now) : []),
   ]);
 
-  const results: SessionFlushResult[] = [];
-
-  for (const source of allSources) {
-    results.push(await processSessionSource(source, state, config, client, now));
+  // Migrate any existing sessions that predate the confirmation fields
+  for (const [key, session] of Object.entries(state.sessions)) {
+    state.sessions[key] = migrateSessionUploadState(session);
   }
 
+  // Verify previously queued chunks and handle 2hr re-submit timeout
+  await verifyUnconfirmedChunks(state, client, now);
+
+  // Process sessions concurrently — each session's chunks stay sequential internally
+  const results = await pLimit(
+    allSources.map((source) => () => processSessionSource(
+      source, state, config, client, now,
+      opts.prefetchedTraces?.get(`${source.tool}:${source.locator}`),
+      opts.sync,
+    )),
+    INGEST_CONCURRENCY,
+  );
+
   saveState(state, config.profile);
 
   return {
@@ -174,6 +244,9 @@ export function createFreshSessionState(
     lastSeenTurnCount: 0,
     lastActivityAt: null,
     lastFlushedTurnId: null,
+    confirmedChunkIndex: 0,
+    confirmedOpenChunkStartTurn: 0,
+    unconfirmedSince: null,
   };
 }
 
@@ -182,15 +255,20 @@ export function migrateLegacySessionState(
   trace: NormalizedTrace,
   legacyChunkIndex: number
 ): SessionUploadState {
+  const nextChunkIndex = legacyChunkIndex + 1;
+  const openChunkStartTurn = trace.turn_count;
   return {
     sourceTool: source.tool,
     sourceSessionId: trace.source_session_id,
     locator: source.locator,
-    nextChunkIndex: legacyChunkIndex + 1,
-    openChunkStartTurn: trace.turn_count,
+    nextChunkIndex,
+    openChunkStartTurn,
     lastSeenTurnCount: trace.turn_count,
     lastActivityAt: getLastActivityAt(trace),
     lastFlushedTurnId: trace.turns[trace.turn_count - 1]?.turn_id ?? null,
+    confirmedChunkIndex: nextChunkIndex,
+    confirmedOpenChunkStartTurn: openChunkStartTurn,
+    unconfirmedSince: null,
   };
 }
 
@@ -199,12 +277,14 @@ async function processSessionSource(
   state: ReturnType<typeof loadState>,
   config: Config,
   client: ApiClient,
-  now: Date
+  now: Date,
+  prefetchedTrace?: NormalizedTrace,
+  sync = false,
 ): Promise<SessionFlushResult> {
   let trace: NormalizedTrace;
 
   try {
-    trace = await extractTraceFromSource(source, config.email);
+    trace = prefetchedTrace ?? await extractTraceFromSource(source, config.email);
   } catch (err) {
     return {
       source,
@@ -250,7 +330,10 @@ async function processSessionSource(
   let payoutCents = 0;
 
   for (const upload of plan.uploads) {
-    const result = await uploadTraceChunk(upload.trace, client);
+    const tUpload = Date.now();
+    const result = await uploadTraceChunk(upload.trace, client, sync);
+    if (!sync) console.error(`[flush] ${key} chunk${upload.trace.chunk_index} queued in ${Date.now()-tUpload}ms`);
+    else console.error(`[flush] ${key} chunk${upload.trace.chunk_index} done in ${Date.now()-tUpload}ms err=${result.error?.slice(0,60) ?? 'none'}`);
     if (result.error) {
       state.sessions[key] = workingState;
       if (workingState.nextChunkIndex > 0) {
@@ -313,44 +396,41 @@ async function extractTraceFromSource(
 
 async function uploadTraceChunk(
   trace: NormalizedTrace,
-  client: ApiClient
+  client: ApiClient,
+  sync = false,
 ): Promise<ChunkUploadResult> {
+  // Client-side regex redaction runs before transmission; Presidio runs server-side async.
+  const payloadTrace = redactTrace(trace, { homeDir: homedir() });
+  const jsonSize = JSON.stringify({ trace: payloadTrace, source_tool: payloadTrace.source_tool }).length;
+  console.error(`[upload] ${payloadTrace.source_session_id} payload=${Math.round(jsonSize/1024)}KB turns=${payloadTrace.turn_count}`);
+
+  if (!sync) {
+    // Fire and forget — optimistically treat as queued; verifyUnconfirmedChunks confirms later
+    client.post("/api/v1/traces/ingest", {
+      trace: payloadTrace,
+      source_tool: payloadTrace.source_tool,
+    }).catch(() => { /* errors handled by unconfirmed retry mechanism */ });
+    return { duplicate: false, payoutCents: 0, traceId: null };
+  }
+
   try {
-    const result = await client.post("/api/v1/traces/batch", {
-      traces: [redactTrace(trace, { homeDir: homedir() })],
-      source_tool: trace.source_tool,
-    }) as {
-      accepted: number;
-      duplicate: number;
-      traces: Array<{
-        error?: string;
-        is_duplicate?: boolean;
-        payout_cents?: number;
-        trace_id?: string;
-      }>;
-    };
+    const result = await client.post("/api/v1/traces/ingest", {
+      trace: payloadTrace,
+      source_tool: payloadTrace.source_tool,
+    }) as IngestResponse;
 
-    const traceResult = result.traces?.[0] ?? {};
-    if (traceResult.error) {
-      return {
-        duplicate: false,
-        payoutCents: 0,
-        traceId: traceResult.trace_id ?? null,
-        error: traceResult.error,
-      };
+    if (result.duplicate) {
+      return { duplicate: true, payoutCents: 0, traceId: result.trace_id ?? null };
     }
 
-    return {
-      duplicate: traceResult.is_duplicate === true || (result.duplicate > 0 && result.accepted === 0),
-      payoutCents: traceResult.payout_cents ?? 0,
-      traceId: traceResult.trace_id ?? null,
-    };
+    // 202 queued — payout credited asynchronously; show $0 until processed
+    return { duplicate: false, payoutCents: 0, traceId: null };
   } catch (err) {
     return {
       duplicate: false,
       payoutCents: 0,
       traceId: null,
-      error: `Submit failed: ${err}`,
+      error: formatSubmitFailure(err),
     };
   }
 }
@@ -379,6 +459,7 @@ function applyUploadedChunk(
   const nextOpenChunkStartTurn = (uploadedChunk.chunk_start_turn ?? cursor.openChunkStartTurn)
     + (uploadedChunk.turn_count ?? uploadedChunk.turns.length);
 
+  const nowIso = new Date().toISOString();
   return {
     ...cursor,
     nextChunkIndex: (uploadedChunk.chunk_index ?? cursor.nextChunkIndex) + 1,
@@ -386,6 +467,7 @@ function applyUploadedChunk(
     lastSeenTurnCount: trace.turn_count,
     lastActivityAt: getLastActivityAt(trace),
     lastFlushedTurnId: trace.turns[nextOpenChunkStartTurn - 1]?.turn_id ?? cursor.lastFlushedTurnId,
+    unconfirmedSince: cursor.unconfirmedSince ?? nowIso,
   };
 }
 
@@ -485,6 +567,25 @@ function dedupeSources(sources: SessionSource[]): SessionSource[] {
   return [...unique.values()];
 }
 
+function formatSubmitFailure(err: unknown): string {
+  return err instanceof Error ? `Submit failed: ${err.message}` : `Submit failed: ${String(err)}`;
+}
+
+async function pLimit<T>(tasks: Array<() => Promise<T>>, concurrency: number): Promise<T[]> {
+  const results: T[] = new Array(tasks.length);
+  let index = 0;
+
+  async function worker() {
+    while (index < tasks.length) {
+      const i = index++;
+      results[i] = await tasks[i]!();
+    }
+  }
+
+  await Promise.all(Array.from({ length: Math.min(concurrency, tasks.length) }, worker));
+  return results;
+}
+
 export function buildFileSessionSource(
   tool: "claude_code" | "codex_cli",
   filePath: string

package/src/commands/register.ts

@@ -1,8 +0,0 @@
-import chalk from "chalk";
-import { CLI_NAME } from "../constants.js";
-import { loginCommand } from "./login.js";
-
-export async function registerCommand(opts: { profile?: string; serverUrl?: string }): Promise<void> {
-  console.log(chalk.yellow(`\`${CLI_NAME} register\` is now an alias for \`${CLI_NAME} login\`.`));
-  await loginCommand({ profile: opts.profile, serverUrl: opts.serverUrl });
-}