@tracemarketplace/cli 0.0.13 → 0.0.17

package/src/flush.ts

@@ -0,0 +1,591 @@
+import { readFile } from "fs/promises";
+import { homedir } from "os";
+import {
+  chunkTrace,
+  extractClaudeCode,
+  extractCodex,
+  extractCursor,
+  redactTrace,
+  type NormalizedTrace,
+} from "@tracemarketplace/shared";
+import { ApiClient } from "./api-client.js";
+import {
+  loadState,
+  migrateSessionUploadState,
+  saveState,
+  stateKey,
+  type Config,
+  type SessionUploadState,
+  type TrackedSessionTool,
+} from "./config.js";
+import { CURSOR_DB_PATH } from "./sessions.js";
+
+const IDLE_FINALIZATION_MS = 2 * 24 * 60 * 60 * 1000;
+const UNCONFIRMED_RESUBMIT_MS = 2 * 60 * 60 * 1000; // re-submit after 2hr with no confirmation
+const INGEST_CONCURRENCY = 12;
+
+export interface SessionSource {
+  tool: TrackedSessionTool;
+  locator: string;
+  label: string;
+}
+
+export interface PlannedUpload {
+  trace: NormalizedTrace;
+  nextState: SessionUploadState;
+}
+
+export interface SessionPlan {
+  observedState: SessionUploadState;
+  uploads: PlannedUpload[];
+  pending: boolean;
+}
+
+export interface SessionFlushResult {
+  source: SessionSource;
+  sessionKey: string | null;
+  uploadedChunks: number;
+  duplicateChunks: number;
+  pending: boolean;
+  payoutCents: number;
+  turnCount: number;
+  migratedLegacyState: boolean;
+  error?: string;
+}
+
+export interface FlushResult {
+  processedSessions: number;
+  uploadedChunks: number;
+  duplicateChunks: number;
+  pendingSessions: number;
+  payoutCents: number;
+  results: SessionFlushResult[];
+}
+
+interface ChunkUploadResult {
+  duplicate: boolean;
+  payoutCents: number;
+  traceId: string | null;
+  error?: string;
+}
+
+interface IngestResponse {
+  queued?: boolean;
+  duplicate?: boolean;
+  trace_id?: string;
+}
+
+export function collectIdleSessionSources(
+  sessions: Record<string, SessionUploadState>,
+  now = new Date()
+): SessionSource[] {
+  return Object.values(sessions)
+    .filter((session) => isSessionIdlePending(session, now))
+    .map((session) => ({
+      tool: session.sourceTool,
+      locator: session.locator,
+      label: `${session.sourceTool}:${session.sourceSessionId}`,
+    }));
+}
+
+interface ChunkExistsResponse {
+  exists: boolean;
+  trace_id?: string;
+}
+
+export async function verifyUnconfirmedChunks(
+  state: ReturnType<typeof loadState>,
+  client: ApiClient,
+  now: Date,
+): Promise<void> {
+  for (const [key, session] of Object.entries(state.sessions)) {
+    if (session.confirmedChunkIndex >= session.nextChunkIndex) continue;
+
+    // Check if timed out — reset to re-submit from last confirmed point
+    if (session.unconfirmedSince) {
+      const age = now.getTime() - Date.parse(session.unconfirmedSince);
+      if (age >= UNCONFIRMED_RESUBMIT_MS) {
+        state.sessions[key] = {
+          ...session,
+          nextChunkIndex: session.confirmedChunkIndex,
+          openChunkStartTurn: session.confirmedOpenChunkStartTurn,
+          unconfirmedSince: null,
+        };
+        continue;
+      }
+    }
+
+    // Check each unconfirmed chunk sequentially — stop at first missing one
+    for (let i = session.confirmedChunkIndex; i < session.nextChunkIndex; i++) {
+      try {
+        const params = new URLSearchParams({
+          source_tool: session.sourceTool,
+          source_session_id: session.sourceSessionId,
+          chunk_index: String(i),
+        });
+        const result = await client.get(`/api/v1/traces/exists?${params}`) as ChunkExistsResponse;
+        if (!result.exists) break;
+        state.sessions[key] = {
+          ...state.sessions[key]!,
+          confirmedChunkIndex: i + 1,
+          confirmedOpenChunkStartTurn: state.sessions[key]!.openChunkStartTurn,
+          unconfirmedSince: i + 1 >= session.nextChunkIndex ? null : state.sessions[key]!.unconfirmedSince,
+        };
+      } catch {
+        break;
+      }
+    }
+  }
+}
+
+export async function flushTrackedSessions(
+  config: Config,
+  sources: SessionSource[],
+  opts: { includeIdleTracked?: boolean; now?: Date; prefetchedTraces?: Map<string, NormalizedTrace> } = {}
+): Promise<FlushResult> {
+  const now = opts.now ?? new Date();
+  const state = loadState(config.profile);
+  const client = new ApiClient(config.serverUrl, config.apiKey);
+
+  const allSources = dedupeSources([
+    ...sources,
+    ...(opts.includeIdleTracked ? collectIdleSessionSources(state.sessions, now) : []),
+  ]);
+
+  // Migrate any existing sessions that predate the confirmation fields
+  for (const [key, session] of Object.entries(state.sessions)) {
+    state.sessions[key] = migrateSessionUploadState(session);
+  }
+
+  // Verify previously queued chunks and handle 2hr re-submit timeout
+  await verifyUnconfirmedChunks(state, client, now);
+
+  // Process sessions concurrently — each session's chunks stay sequential internally
+  const results = await pLimit(
+    allSources.map((source) => () => processSessionSource(
+      source, state, config, client, now,
+      opts.prefetchedTraces?.get(`${source.tool}:${source.locator}`),
+    )),
+    INGEST_CONCURRENCY,
+  );
+
+  saveState(state, config.profile);
+
+  return {
+    processedSessions: results.length,
+    uploadedChunks: results.reduce((sum, result) => sum + result.uploadedChunks, 0),
+    duplicateChunks: results.reduce((sum, result) => sum + result.duplicateChunks, 0),
+    pendingSessions: results.filter((result) => result.pending).length,
+    payoutCents: results.reduce((sum, result) => sum + result.payoutCents, 0),
+    results,
+  };
+}
+
+export function planSessionUploads(
+  trace: NormalizedTrace,
+  cursor: SessionUploadState,
+  now = new Date()
+): SessionPlan {
+  const observedState = observeTrace(cursor, trace);
+
+  if (observedState.openChunkStartTurn >= trace.turn_count) {
+    return {
+      observedState,
+      uploads: [],
+      pending: false,
+    };
+  }
+
+  const suffixTrace = sliceTraceFromTurn(trace, observedState.openChunkStartTurn);
+  const relativeChunks = chunkTrace(suffixTrace);
+  const uploads: PlannedUpload[] = [];
+  const nowIso = now.toISOString();
+  let nextState = observedState;
+
+  for (let i = 0; i < relativeChunks.length; i++) {
+    const relativeChunk = relativeChunks[i];
+    const isLast = i === relativeChunks.length - 1;
+    const outputTokens = relativeChunk.total_output_tokens ?? 0;
+    const closeReason = !isLast
+      ? "100k_tokens"
+      : outputTokens >= 100_000
+        ? "100k_tokens"
+        : isTraceIdle(trace, now)
+          ? "idle_2d"
+          : null;
+
+    if (!closeReason) {
+      break;
+    }
+
+    const finalizedChunk = finalizeChunk(relativeChunk, nextState, nowIso, closeReason);
+    nextState = applyUploadedChunk(nextState, trace, finalizedChunk);
+    uploads.push({ trace: finalizedChunk, nextState });
+  }
+
+  return {
+    observedState,
+    uploads,
+    pending: nextState.openChunkStartTurn < trace.turn_count,
+  };
+}
+
+export function createFreshSessionState(
+  source: SessionSource,
+  trace: NormalizedTrace
+): SessionUploadState {
+  return {
+    sourceTool: source.tool,
+    sourceSessionId: trace.source_session_id,
+    locator: source.locator,
+    nextChunkIndex: 0,
+    openChunkStartTurn: 0,
+    lastSeenTurnCount: 0,
+    lastActivityAt: null,
+    lastFlushedTurnId: null,
+    confirmedChunkIndex: 0,
+    confirmedOpenChunkStartTurn: 0,
+    unconfirmedSince: null,
+  };
+}
+
+export function migrateLegacySessionState(
+  source: SessionSource,
+  trace: NormalizedTrace,
+  legacyChunkIndex: number
+): SessionUploadState {
+  const nextChunkIndex = legacyChunkIndex + 1;
+  const openChunkStartTurn = trace.turn_count;
+  return {
+    sourceTool: source.tool,
+    sourceSessionId: trace.source_session_id,
+    locator: source.locator,
+    nextChunkIndex,
+    openChunkStartTurn,
+    lastSeenTurnCount: trace.turn_count,
+    lastActivityAt: getLastActivityAt(trace),
+    lastFlushedTurnId: trace.turns[trace.turn_count - 1]?.turn_id ?? null,
+    confirmedChunkIndex: nextChunkIndex,
+    confirmedOpenChunkStartTurn: openChunkStartTurn,
+    unconfirmedSince: null,
+  };
+}
+
+async function processSessionSource(
+  source: SessionSource,
+  state: ReturnType<typeof loadState>,
+  config: Config,
+  client: ApiClient,
+  now: Date,
+  prefetchedTrace?: NormalizedTrace,
+): Promise<SessionFlushResult> {
+  let trace: NormalizedTrace;
+
+  try {
+    trace = prefetchedTrace ?? await extractTraceFromSource(source, config.email);
+  } catch (err) {
+    return {
+      source,
+      sessionKey: null,
+      uploadedChunks: 0,
+      duplicateChunks: 0,
+      pending: false,
+      payoutCents: 0,
+      turnCount: 0,
+      migratedLegacyState: false,
+      error: `Extraction failed: ${err}`,
+    };
+  }
+
+  if (trace.turn_count === 0) {
+    return {
+      source,
+      sessionKey: stateKey(source.tool, trace.source_session_id),
+      uploadedChunks: 0,
+      duplicateChunks: 0,
+      pending: false,
+      payoutCents: 0,
+      turnCount: 0,
+      migratedLegacyState: false,
+      error: "Empty session",
+    };
+  }
+
+  const key = stateKey(trace.source_tool, trace.source_session_id);
+  const existing = state.sessions[key];
+  const legacyChunkIndex = state.chunks[key];
+  const migratedLegacyState = !existing && typeof legacyChunkIndex === "number";
+  const cursor = existing
+    ? { ...existing, locator: source.locator, sourceTool: trace.source_tool, sourceSessionId: trace.source_session_id }
+    : migratedLegacyState
+      ? migrateLegacySessionState(source, trace, legacyChunkIndex)
+      : createFreshSessionState(source, trace);
+
+  const plan = planSessionUploads(trace, cursor, now);
+  let workingState = plan.observedState;
+  let uploadedChunks = 0;
+  let duplicateChunks = 0;
+  let payoutCents = 0;
+
+  for (const upload of plan.uploads) {
+    const tUpload = Date.now();
+    const result = await uploadTraceChunk(upload.trace, client);
+    console.error(`[flush] ${key} chunk${upload.trace.chunk_index} done in ${Date.now()-tUpload}ms err=${result.error?.slice(0,60) ?? 'none'}`);
+    if (result.error) {
+      state.sessions[key] = workingState;
+      if (workingState.nextChunkIndex > 0) {
+        state.chunks[key] = workingState.nextChunkIndex - 1;
+      }
+      return {
+        source,
+        sessionKey: key,
+        uploadedChunks,
+        duplicateChunks,
+        pending: workingState.openChunkStartTurn < workingState.lastSeenTurnCount,
+        payoutCents,
+        turnCount: trace.turn_count,
+        migratedLegacyState,
+        error: result.error,
+      };
+    }
+
+    workingState = upload.nextState;
+    payoutCents += result.payoutCents;
+    if (result.duplicate) {
+      duplicateChunks++;
+    } else {
+      uploadedChunks++;
+    }
+  }
+
+  state.sessions[key] = workingState;
+  if (workingState.nextChunkIndex > 0) {
+    state.chunks[key] = workingState.nextChunkIndex - 1;
+  }
+
+  return {
+    source,
+    sessionKey: key,
+    uploadedChunks,
+    duplicateChunks,
+    pending: workingState.openChunkStartTurn < workingState.lastSeenTurnCount,
+    payoutCents,
+    turnCount: trace.turn_count,
+    migratedLegacyState,
+  };
+}
+
+async function extractTraceFromSource(
+  source: SessionSource,
+  email: string
+): Promise<NormalizedTrace> {
+  switch (source.tool) {
+    case "claude_code":
+      return extractClaudeCode(source.locator, email);
+    case "codex_cli": {
+      const buf = await readFile(source.locator);
+      return extractCodex(buf, email);
+    }
+    case "cursor":
+      return extractCursor(CURSOR_DB_PATH, source.locator, email);
+  }
+}
+
+async function uploadTraceChunk(
+  trace: NormalizedTrace,
+  client: ApiClient
+): Promise<ChunkUploadResult> {
+  // Client-side regex redaction runs before transmission; Presidio runs server-side async.
+  const payloadTrace = redactTrace(trace, { homeDir: homedir() });
+
+  try {
+    const result = await client.post("/api/v1/traces/ingest", {
+      trace: payloadTrace,
+      source_tool: payloadTrace.source_tool,
+    }) as IngestResponse;
+
+    if (result.duplicate) {
+      return { duplicate: true, payoutCents: 0, traceId: result.trace_id ?? null };
+    }
+
+    // 202 queued — payout credited asynchronously; show $0 until processed
+    return { duplicate: false, payoutCents: 0, traceId: null };
+  } catch (err) {
+    return {
+      duplicate: false,
+      payoutCents: 0,
+      traceId: null,
+      error: formatSubmitFailure(err),
+    };
+  }
+}
+
+function finalizeChunk(
+  trace: NormalizedTrace,
+  cursor: SessionUploadState,
+  nowIso: string,
+  closeReason: "100k_tokens" | "idle_2d"
+): NormalizedTrace {
+  return {
+    ...trace,
+    chunk_index: cursor.nextChunkIndex,
+    chunk_start_turn: cursor.openChunkStartTurn + (trace.chunk_start_turn ?? 0),
+    chunk_complete: true,
+    chunk_close_reason: closeReason,
+    chunk_closed_at: nowIso,
+  };
+}
+
+function applyUploadedChunk(
+  cursor: SessionUploadState,
+  trace: NormalizedTrace,
+  uploadedChunk: NormalizedTrace
+): SessionUploadState {
+  const nextOpenChunkStartTurn = (uploadedChunk.chunk_start_turn ?? cursor.openChunkStartTurn)
+    + (uploadedChunk.turn_count ?? uploadedChunk.turns.length);
+
+  const nowIso = new Date().toISOString();
+  return {
+    ...cursor,
+    nextChunkIndex: (uploadedChunk.chunk_index ?? cursor.nextChunkIndex) + 1,
+    openChunkStartTurn: nextOpenChunkStartTurn,
+    lastSeenTurnCount: trace.turn_count,
+    lastActivityAt: getLastActivityAt(trace),
+    lastFlushedTurnId: trace.turns[nextOpenChunkStartTurn - 1]?.turn_id ?? cursor.lastFlushedTurnId,
+    unconfirmedSince: cursor.unconfirmedSince ?? nowIso,
+  };
+}
+
+function observeTrace(
+  cursor: SessionUploadState,
+  trace: NormalizedTrace
+): SessionUploadState {
+  return {
+    ...cursor,
+    sourceTool: trace.source_tool,
+    sourceSessionId: trace.source_session_id,
+    openChunkStartTurn: Math.min(cursor.openChunkStartTurn, trace.turn_count),
+    lastSeenTurnCount: trace.turn_count,
+    lastActivityAt: getLastActivityAt(trace),
+  };
+}
+
+function sliceTraceFromTurn(
+  trace: NormalizedTrace,
+  startTurn: number
+): NormalizedTrace {
+  const turns = trace.turns.slice(startTurn);
+  const inputTokens = turns.reduce((sum, turn) => sum + (turn.usage?.input_tokens ?? 0), 0);
+  const outputTokens = turns.reduce((sum, turn) => sum + (turn.usage?.output_tokens ?? 0), 0);
+  const toolCallCount = turns.reduce(
+    (sum, turn) => sum + turn.content.filter((block) => block.type === "tool_use").length,
+    0
+  );
+
+  return {
+    ...trace,
+    turns,
+    chunk_index: 0,
+    chunk_start_turn: 0,
+    chunk_complete: false,
+    chunk_close_reason: undefined,
+    chunk_closed_at: null,
+    turn_count: turns.length,
+    tool_call_count: toolCallCount,
+    has_tool_calls: toolCallCount > 0,
+    has_thinking_blocks: turns.some((turn) => turn.content.some((block) => block.type === "thinking")),
+    has_file_changes: turns.some((turn) =>
+      turn.content.some((block) =>
+        block.type === "tool_use"
+        && (block.tool_name === "Edit" || block.tool_name === "Write" || block.tool_name === "MultiEdit")
+      )
+    ),
+    has_shell_commands: turns.some((turn) =>
+      turn.content.some((block) => block.type === "tool_use" && block.tool_name === "Bash")
+    ),
+    total_input_tokens: inputTokens || null,
+    total_output_tokens: outputTokens || null,
+    started_at: turns.find((turn) => turn.timestamp)?.timestamp ?? trace.started_at,
+    ended_at: [...turns].reverse().find((turn) => turn.timestamp)?.timestamp ?? trace.ended_at,
+  };
+}
+
+function getLastActivityAt(trace: NormalizedTrace): string | null {
+  return trace.ended_at
+    ?? trace.turns[trace.turns.length - 1]?.timestamp
+    ?? trace.extracted_at
+    ?? null;
+}
+
+function isTraceIdle(trace: NormalizedTrace, now: Date): boolean {
+  const lastActivityAt = getLastActivityAt(trace);
+  if (!lastActivityAt) return false;
+
+  const lastActivityMs = Date.parse(lastActivityAt);
+  if (Number.isNaN(lastActivityMs)) return false;
+  return now.getTime() - lastActivityMs >= IDLE_FINALIZATION_MS;
+}
+
+function isSessionIdlePending(session: SessionUploadState, now: Date): boolean {
+  if (session.openChunkStartTurn >= session.lastSeenTurnCount) {
+    return false;
+  }
+  if (!session.lastActivityAt) {
+    return false;
+  }
+
+  const lastActivityMs = Date.parse(session.lastActivityAt);
+  if (Number.isNaN(lastActivityMs)) {
+    return false;
+  }
+
+  return now.getTime() - lastActivityMs >= IDLE_FINALIZATION_MS;
+}
+
+function dedupeSources(sources: SessionSource[]): SessionSource[] {
+  const unique = new Map<string, SessionSource>();
+
+  for (const source of sources) {
+    unique.set(`${source.tool}:${source.locator}`, source);
+  }
+
+  return [...unique.values()];
+}
+
+function formatSubmitFailure(err: unknown): string {
+  return err instanceof Error ? `Submit failed: ${err.message}` : `Submit failed: ${String(err)}`;
+}
+
+async function pLimit<T>(tasks: Array<() => Promise<T>>, concurrency: number): Promise<T[]> {
+  const results: T[] = new Array(tasks.length);
+  let index = 0;
+
+  async function worker() {
+    while (index < tasks.length) {
+      const i = index++;
+      results[i] = await tasks[i]!();
+    }
+  }
+
+  await Promise.all(Array.from({ length: Math.min(concurrency, tasks.length) }, worker));
+  return results;
+}
+
+export function buildFileSessionSource(
+  tool: "claude_code" | "codex_cli",
+  filePath: string
+): SessionSource {
+  return {
+    tool,
+    locator: filePath,
+    label: filePath,
+  };
+}
+
+export function buildCursorSessionSource(sessionId: string): SessionSource {
+  return {
+    tool: "cursor",
+    locator: sessionId,
+    label: sessionId,
+  };
+}

package/vitest.config.ts

@@ -0,0 +1,8 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    environment: "node",
+    include: ["src/**/*.test.ts"],
+  },
+});

package/src/commands/register.ts

@@ -1,52 +0,0 @@
-import inquirer from "inquirer";
-import chalk from "chalk";
-import { saveConfig } from "../config.js";
-
-export async function registerCommand(opts: { serverUrl?: string }): Promise<void> {
-  const { email } = await inquirer.prompt([
-    {
-      type: "input",
-      name: "email",
-      message: "Your email address:",
-      validate: (v: string) => v.includes("@") || "Enter a valid email",
-    },
-  ]);
-
-  const serverUrl =
-    opts.serverUrl ??
-    (
-      await inquirer.prompt([
-        {
-          type: "input",
-          name: "url",
-          message: "Server URL:",
-          default: "https://trace-marketplace-api.fly.dev",
-        },
-      ])
-    ).url;
-
-  const res = await fetch(`${serverUrl}/api/v1/register`, {
-    method: "POST",
-    headers: { "Content-Type": "application/json" },
-    body: JSON.stringify({ email }),
-  });
-
-  const data = (await res.json()) as { api_key?: string; error?: string };
-
-  if (!res.ok) {
-    if (res.status === 409 && data.api_key) {
-      console.log(chalk.yellow("Email already registered."));
-      console.log(chalk.cyan("Your API key:"), chalk.bold(data.api_key));
-      saveConfig({ apiKey: data.api_key, serverUrl, email });
-      return;
-    }
-    throw new Error(data.error ?? `HTTP ${res.status}`);
-  }
-
-  const apiKey = data.api_key!;
-  saveConfig({ apiKey, serverUrl, email });
-
-  console.log(chalk.green("Registered successfully!"));
-  console.log(chalk.cyan("Your API key:"), chalk.bold(apiKey));
-  console.log(chalk.gray("Config saved to ~/.config/tracemarketplace/config.json"));
-}