@tracemarketplace/cli 0.0.13 → 0.0.15

package/src/flush.ts

@@ -0,0 +1,505 @@
+import { readFile } from "fs/promises";
+import { homedir } from "os";
+import {
+  chunkTrace,
+  extractClaudeCode,
+  extractCodex,
+  extractCursor,
+  redactTrace,
+  type NormalizedTrace,
+  type Turn,
+} from "@tracemarketplace/shared";
+import { ApiClient } from "./api-client.js";
+import {
+  loadState,
+  saveState,
+  stateKey,
+  type Config,
+  type SessionUploadState,
+  type TrackedSessionTool,
+} from "./config.js";
+import { CURSOR_DB_PATH } from "./sessions.js";
+
+const IDLE_FINALIZATION_MS = 2 * 24 * 60 * 60 * 1000;
+
+export interface SessionSource {
+  tool: TrackedSessionTool;
+  locator: string;
+  label: string;
+}
+
+export interface PlannedUpload {
+  trace: NormalizedTrace;
+  nextState: SessionUploadState;
+}
+
+export interface SessionPlan {
+  observedState: SessionUploadState;
+  uploads: PlannedUpload[];
+  pending: boolean;
+}
+
+export interface SessionFlushResult {
+  source: SessionSource;
+  sessionKey: string | null;
+  uploadedChunks: number;
+  duplicateChunks: number;
+  pending: boolean;
+  payoutCents: number;
+  turnCount: number;
+  migratedLegacyState: boolean;
+  error?: string;
+}
+
+export interface FlushResult {
+  processedSessions: number;
+  uploadedChunks: number;
+  duplicateChunks: number;
+  pendingSessions: number;
+  payoutCents: number;
+  results: SessionFlushResult[];
+}
+
+interface ChunkUploadResult {
+  duplicate: boolean;
+  payoutCents: number;
+  traceId: string | null;
+  error?: string;
+}
+
+export function collectIdleSessionSources(
+  sessions: Record<string, SessionUploadState>,
+  now = new Date()
+): SessionSource[] {
+  return Object.values(sessions)
+    .filter((session) => isSessionIdlePending(session, now))
+    .map((session) => ({
+      tool: session.sourceTool,
+      locator: session.locator,
+      label: `${session.sourceTool}:${session.sourceSessionId}`,
+    }));
+}
+
+export async function flushTrackedSessions(
+  config: Config,
+  sources: SessionSource[],
+  opts: { includeIdleTracked?: boolean; now?: Date } = {}
+): Promise<FlushResult> {
+  const now = opts.now ?? new Date();
+  const state = loadState(config.profile);
+  const client = new ApiClient(config.serverUrl, config.apiKey);
+
+  const allSources = dedupeSources([
+    ...sources,
+    ...(opts.includeIdleTracked ? collectIdleSessionSources(state.sessions, now) : []),
+  ]);
+
+  const results: SessionFlushResult[] = [];
+
+  for (const source of allSources) {
+    results.push(await processSessionSource(source, state, config, client, now));
+  }
+
+  saveState(state, config.profile);
+
+  return {
+    processedSessions: results.length,
+    uploadedChunks: results.reduce((sum, result) => sum + result.uploadedChunks, 0),
+    duplicateChunks: results.reduce((sum, result) => sum + result.duplicateChunks, 0),
+    pendingSessions: results.filter((result) => result.pending).length,
+    payoutCents: results.reduce((sum, result) => sum + result.payoutCents, 0),
+    results,
+  };
+}
+
+export function planSessionUploads(
+  trace: NormalizedTrace,
+  cursor: SessionUploadState,
+  now = new Date()
+): SessionPlan {
+  const observedState = observeTrace(cursor, trace);
+
+  if (observedState.openChunkStartTurn >= trace.turn_count) {
+    return {
+      observedState,
+      uploads: [],
+      pending: false,
+    };
+  }
+
+  const suffixTrace = sliceTraceFromTurn(trace, observedState.openChunkStartTurn);
+  const relativeChunks = chunkTrace(suffixTrace);
+  const uploads: PlannedUpload[] = [];
+  const nowIso = now.toISOString();
+  let nextState = observedState;
+
+  for (let i = 0; i < relativeChunks.length; i++) {
+    const relativeChunk = relativeChunks[i];
+    const isLast = i === relativeChunks.length - 1;
+    const outputTokens = relativeChunk.total_output_tokens ?? 0;
+    const closeReason = !isLast
+      ? "100k_tokens"
+      : outputTokens >= 100_000
+        ? "100k_tokens"
+        : isTraceIdle(trace, now)
+          ? "idle_2d"
+          : null;
+
+    if (!closeReason) {
+      break;
+    }
+
+    const finalizedChunk = finalizeChunk(relativeChunk, nextState, nowIso, closeReason);
+    nextState = applyUploadedChunk(nextState, trace, finalizedChunk);
+    uploads.push({ trace: finalizedChunk, nextState });
+  }
+
+  return {
+    observedState,
+    uploads,
+    pending: nextState.openChunkStartTurn < trace.turn_count,
+  };
+}
+
+export function createFreshSessionState(
+  source: SessionSource,
+  trace: NormalizedTrace
+): SessionUploadState {
+  return {
+    sourceTool: source.tool,
+    sourceSessionId: trace.source_session_id,
+    locator: source.locator,
+    nextChunkIndex: 0,
+    openChunkStartTurn: 0,
+    lastSeenTurnCount: 0,
+    lastActivityAt: null,
+    lastFlushedTurnId: null,
+  };
+}
+
+export function migrateLegacySessionState(
+  source: SessionSource,
+  trace: NormalizedTrace,
+  legacyChunkIndex: number
+): SessionUploadState {
+  return {
+    sourceTool: source.tool,
+    sourceSessionId: trace.source_session_id,
+    locator: source.locator,
+    nextChunkIndex: legacyChunkIndex + 1,
+    openChunkStartTurn: trace.turn_count,
+    lastSeenTurnCount: trace.turn_count,
+    lastActivityAt: getLastActivityAt(trace),
+    lastFlushedTurnId: trace.turns[trace.turn_count - 1]?.turn_id ?? null,
+  };
+}
+
+async function processSessionSource(
+  source: SessionSource,
+  state: ReturnType<typeof loadState>,
+  config: Config,
+  client: ApiClient,
+  now: Date
+): Promise<SessionFlushResult> {
+  let trace: NormalizedTrace;
+
+  try {
+    trace = await extractTraceFromSource(source, config.email);
+  } catch (err) {
+    return {
+      source,
+      sessionKey: null,
+      uploadedChunks: 0,
+      duplicateChunks: 0,
+      pending: false,
+      payoutCents: 0,
+      turnCount: 0,
+      migratedLegacyState: false,
+      error: `Extraction failed: ${err}`,
+    };
+  }
+
+  if (trace.turn_count === 0) {
+    return {
+      source,
+      sessionKey: stateKey(source.tool, trace.source_session_id),
+      uploadedChunks: 0,
+      duplicateChunks: 0,
+      pending: false,
+      payoutCents: 0,
+      turnCount: 0,
+      migratedLegacyState: false,
+      error: "Empty session",
+    };
+  }
+
+  const key = stateKey(trace.source_tool, trace.source_session_id);
+  const existing = state.sessions[key];
+  const legacyChunkIndex = state.chunks[key];
+  const migratedLegacyState = !existing && typeof legacyChunkIndex === "number";
+  const cursor = existing
+    ? { ...existing, locator: source.locator, sourceTool: trace.source_tool, sourceSessionId: trace.source_session_id }
+    : migratedLegacyState
+      ? migrateLegacySessionState(source, trace, legacyChunkIndex)
+      : createFreshSessionState(source, trace);
+
+  const plan = planSessionUploads(trace, cursor, now);
+  let workingState = plan.observedState;
+  let uploadedChunks = 0;
+  let duplicateChunks = 0;
+  let payoutCents = 0;
+
+  for (const upload of plan.uploads) {
+    const result = await uploadTraceChunk(upload.trace, client);
+    if (result.error) {
+      state.sessions[key] = workingState;
+      if (workingState.nextChunkIndex > 0) {
+        state.chunks[key] = workingState.nextChunkIndex - 1;
+      }
+      return {
+        source,
+        sessionKey: key,
+        uploadedChunks,
+        duplicateChunks,
+        pending: workingState.openChunkStartTurn < workingState.lastSeenTurnCount,
+        payoutCents,
+        turnCount: trace.turn_count,
+        migratedLegacyState,
+        error: result.error,
+      };
+    }
+
+    workingState = upload.nextState;
+    payoutCents += result.payoutCents;
+    if (result.duplicate) {
+      duplicateChunks++;
+    } else {
+      uploadedChunks++;
+    }
+  }
+
+  state.sessions[key] = workingState;
+  if (workingState.nextChunkIndex > 0) {
+    state.chunks[key] = workingState.nextChunkIndex - 1;
+  }
+
+  return {
+    source,
+    sessionKey: key,
+    uploadedChunks,
+    duplicateChunks,
+    pending: workingState.openChunkStartTurn < workingState.lastSeenTurnCount,
+    payoutCents,
+    turnCount: trace.turn_count,
+    migratedLegacyState,
+  };
+}
+
+async function extractTraceFromSource(
+  source: SessionSource,
+  email: string
+): Promise<NormalizedTrace> {
+  switch (source.tool) {
+    case "claude_code":
+      return extractClaudeCode(source.locator, email);
+    case "codex_cli": {
+      const buf = await readFile(source.locator);
+      return extractCodex(buf, email);
+    }
+    case "cursor":
+      return extractCursor(CURSOR_DB_PATH, source.locator, email);
+  }
+}
+
+async function uploadTraceChunk(
+  trace: NormalizedTrace,
+  client: ApiClient
+): Promise<ChunkUploadResult> {
+  try {
+    const result = await client.post("/api/v1/traces/batch", {
+      traces: [redactTrace(trace, { homeDir: homedir() })],
+      source_tool: trace.source_tool,
+    }) as {
+      accepted: number;
+      duplicate: number;
+      traces: Array<{
+        error?: string;
+        is_duplicate?: boolean;
+        payout_cents?: number;
+        trace_id?: string;
+      }>;
+    };
+
+    const traceResult = result.traces?.[0] ?? {};
+    if (traceResult.error) {
+      return {
+        duplicate: false,
+        payoutCents: 0,
+        traceId: traceResult.trace_id ?? null,
+        error: traceResult.error,
+      };
+    }
+
+    return {
+      duplicate: traceResult.is_duplicate === true || (result.duplicate > 0 && result.accepted === 0),
+      payoutCents: traceResult.payout_cents ?? 0,
+      traceId: traceResult.trace_id ?? null,
+    };
+  } catch (err) {
+    return {
+      duplicate: false,
+      payoutCents: 0,
+      traceId: null,
+      error: `Submit failed: ${err}`,
+    };
+  }
+}
+
+function finalizeChunk(
+  trace: NormalizedTrace,
+  cursor: SessionUploadState,
+  nowIso: string,
+  closeReason: "100k_tokens" | "idle_2d"
+): NormalizedTrace {
+  return {
+    ...trace,
+    chunk_index: cursor.nextChunkIndex,
+    chunk_start_turn: cursor.openChunkStartTurn + (trace.chunk_start_turn ?? 0),
+    chunk_complete: true,
+    chunk_close_reason: closeReason,
+    chunk_closed_at: nowIso,
+  };
+}
+
+function applyUploadedChunk(
+  cursor: SessionUploadState,
+  trace: NormalizedTrace,
+  uploadedChunk: NormalizedTrace
+): SessionUploadState {
+  const nextOpenChunkStartTurn = (uploadedChunk.chunk_start_turn ?? cursor.openChunkStartTurn)
+    + (uploadedChunk.turn_count ?? uploadedChunk.turns.length);
+
+  return {
+    ...cursor,
+    nextChunkIndex: (uploadedChunk.chunk_index ?? cursor.nextChunkIndex) + 1,
+    openChunkStartTurn: nextOpenChunkStartTurn,
+    lastSeenTurnCount: trace.turn_count,
+    lastActivityAt: getLastActivityAt(trace),
+    lastFlushedTurnId: trace.turns[nextOpenChunkStartTurn - 1]?.turn_id ?? cursor.lastFlushedTurnId,
+  };
+}
+
+function observeTrace(
+  cursor: SessionUploadState,
+  trace: NormalizedTrace
+): SessionUploadState {
+  return {
+    ...cursor,
+    sourceTool: trace.source_tool,
+    sourceSessionId: trace.source_session_id,
+    openChunkStartTurn: Math.min(cursor.openChunkStartTurn, trace.turn_count),
+    lastSeenTurnCount: trace.turn_count,
+    lastActivityAt: getLastActivityAt(trace),
+  };
+}
+
+function sliceTraceFromTurn(
+  trace: NormalizedTrace,
+  startTurn: number
+): NormalizedTrace {
+  const turns = trace.turns.slice(startTurn);
+  const inputTokens = turns.reduce((sum, turn) => sum + (turn.usage?.input_tokens ?? 0), 0);
+  const outputTokens = turns.reduce((sum, turn) => sum + (turn.usage?.output_tokens ?? 0), 0);
+  const toolCallCount = turns.reduce(
+    (sum, turn) => sum + turn.content.filter((block) => block.type === "tool_use").length,
+    0
+  );
+
+  return {
+    ...trace,
+    turns,
+    chunk_index: 0,
+    chunk_start_turn: 0,
+    chunk_complete: false,
+    chunk_close_reason: undefined,
+    chunk_closed_at: null,
+    turn_count: turns.length,
+    tool_call_count: toolCallCount,
+    has_tool_calls: toolCallCount > 0,
+    has_thinking_blocks: turns.some((turn) => turn.content.some((block) => block.type === "thinking")),
+    has_file_changes: turns.some((turn) =>
+      turn.content.some((block) =>
+        block.type === "tool_use"
+        && (block.tool_name === "Edit" || block.tool_name === "Write" || block.tool_name === "MultiEdit")
+      )
+    ),
+    has_shell_commands: turns.some((turn) =>
+      turn.content.some((block) => block.type === "tool_use" && block.tool_name === "Bash")
+    ),
+    total_input_tokens: inputTokens || null,
+    total_output_tokens: outputTokens || null,
+    started_at: turns.find((turn) => turn.timestamp)?.timestamp ?? trace.started_at,
+    ended_at: [...turns].reverse().find((turn) => turn.timestamp)?.timestamp ?? trace.ended_at,
+  };
+}
+
+function getLastActivityAt(trace: NormalizedTrace): string | null {
+  return trace.ended_at
+    ?? trace.turns[trace.turns.length - 1]?.timestamp
+    ?? trace.extracted_at
+    ?? null;
+}
+
+function isTraceIdle(trace: NormalizedTrace, now: Date): boolean {
+  const lastActivityAt = getLastActivityAt(trace);
+  if (!lastActivityAt) return false;
+
+  const lastActivityMs = Date.parse(lastActivityAt);
+  if (Number.isNaN(lastActivityMs)) return false;
+  return now.getTime() - lastActivityMs >= IDLE_FINALIZATION_MS;
+}
+
+function isSessionIdlePending(session: SessionUploadState, now: Date): boolean {
+  if (session.openChunkStartTurn >= session.lastSeenTurnCount) {
+    return false;
+  }
+  if (!session.lastActivityAt) {
+    return false;
+  }
+
+  const lastActivityMs = Date.parse(session.lastActivityAt);
+  if (Number.isNaN(lastActivityMs)) {
+    return false;
+  }
+
+  return now.getTime() - lastActivityMs >= IDLE_FINALIZATION_MS;
+}
+
+function dedupeSources(sources: SessionSource[]): SessionSource[] {
+  const unique = new Map<string, SessionSource>();
+
+  for (const source of sources) {
+    unique.set(`${source.tool}:${source.locator}`, source);
+  }
+
+  return [...unique.values()];
+}
+
+export function buildFileSessionSource(
+  tool: "claude_code" | "codex_cli",
+  filePath: string
+): SessionSource {
+  return {
+    tool,
+    locator: filePath,
+    label: filePath,
+  };
+}
+
+export function buildCursorSessionSource(sessionId: string): SessionSource {
+  return {
+    tool: "cursor",
+    locator: sessionId,
+    label: sessionId,
+  };
+}

package/vitest.config.ts

@@ -0,0 +1,8 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    environment: "node",
+    include: ["src/**/*.test.ts"],
+  },
+});

package/src/submitter.ts

@@ -1,110 +0,0 @@
-/**
- * submitter.ts — extract one session file and submit it to the API.
- * Shared by auto-submit (hook), submit (batch), and daemon (watch).
- */
-import { readFile } from "fs/promises";
-import { homedir } from "os";
-import { extractClaudeCode, extractCodex, extractCursor, redactTrace, chunkTrace, type NormalizedTrace } from "@tracemarketplace/shared";
-import { ApiClient } from "./api-client.js";
-import { CURSOR_DB_PATH } from "./sessions.js";
-import { loadState, saveState, stateKey } from "./config.js";
-import type { Config } from "./config.js";
-
-export interface SubmitResult {
-  accepted: boolean;
-  superseded: boolean;
-  duplicate: boolean;
-  turnCount: number;
-  payoutCents: number;
-  traceId: string | null;
-  error?: string;
-}
-
-/**
- * Extract a claude_code or codex_cli session from a file path and submit it.
- */
-export async function submitFile(
-  tool: "claude_code" | "codex_cli",
-  filePath: string,
-  config: Config
-): Promise<SubmitResult> {
-  let trace;
-  try {
-    if (tool === "claude_code") {
-      trace = await extractClaudeCode(filePath, config.email);
-    } else {
-      const buf = await readFile(filePath);
-      trace = await extractCodex(buf, config.email);
-    }
-  } catch (err) {
-    return { accepted: false, superseded: false, duplicate: false, turnCount: 0, payoutCents: 0, traceId: null, error: `Extraction failed: ${err}` };
-  }
-
-  if (trace.turn_count === 0) {
-    return { accepted: false, superseded: false, duplicate: false, turnCount: 0, payoutCents: 0, traceId: null, error: "Empty session" };
-  }
-
-  return submitTrace(trace, config);
-}
-
-/**
- * Extract a Cursor session by session ID and submit it.
- */
-export async function submitCursorSession(
-  sessionId: string,
-  config: Config
-): Promise<SubmitResult> {
-  let trace;
-  try {
-    trace = await extractCursor(CURSOR_DB_PATH, sessionId, config.email);
-  } catch (err) {
-    return { accepted: false, superseded: false, duplicate: false, turnCount: 0, payoutCents: 0, traceId: null, error: `Cursor extraction failed: ${err}` };
-  }
-  return submitTrace(trace, config);
-}
-
-async function submitTrace(trace: Awaited<ReturnType<typeof extractClaudeCode>>, config: Config): Promise<SubmitResult> {
-  const state = loadState();
-  const key = stateKey(trace.source_tool, trace.source_session_id);
-  const lastSubmittedChunk = state.chunks[key] ?? -1;
-
-  // Chunk the session and only send new chunks
-  const chunks = chunkTrace(trace);
-  const newChunks = chunks.filter((c) => (c.chunk_index ?? 0) > lastSubmittedChunk);
-
-  if (newChunks.length === 0) {
-    return { accepted: false, superseded: false, duplicate: true, turnCount: trace.turn_count, payoutCents: 0, traceId: null };
-  }
-
-  const home = homedir();
-  const client = new ApiClient(config.serverUrl, config.apiKey);
-
-  try {
-    const result = await client.post("/api/v1/traces/batch", {
-      traces: newChunks.map((c) => redactTrace(c, { homeDir: home })),
-      source_tool: trace.source_tool,
-    }) as {
-      accepted: number;
-      duplicate: number;
-      traces: Array<{ trace_id?: string; payout_cents?: number }>;
-    };
-
-    // Persist the highest chunk index we successfully submitted
-    const maxChunk = Math.max(...newChunks.map((c) => c.chunk_index ?? 0));
-    state.chunks[key] = maxChunk;
-    saveState(state);
-
-    const totalPayout = result.traces?.reduce((s, t) => s + (t.payout_cents ?? 0), 0) ?? 0;
-    const first = result.traces?.[0];
-    return {
-      accepted: result.accepted > 0,
-      superseded: false,
-      duplicate: result.duplicate > 0 && result.accepted === 0,
-      turnCount: trace.turn_count,
-      payoutCents: totalPayout,
-      traceId: first?.trace_id ?? null,
-    };
-  } catch (err) {
-    return { accepted: false, superseded: false, duplicate: false, turnCount: trace.turn_count, payoutCents: 0, traceId: null, error: `Submit failed: ${err}` };
-  }
-}