@tracemarketplace/cli 0.0.17 → 0.0.19

package/src/commands/submit.ts

@@ -1,10 +1,12 @@
-import { existsSync } from "fs";
+import { closeSync, existsSync, mkdirSync, openSync, unlinkSync, writeFileSync } from "fs";
 import { readFile } from "fs/promises";
+import { spawn } from "child_process";
+import { join } from "path";
 import chalk from "chalk";
 import ora from "ora";
 import inquirer from "inquirer";
 import { extractClaudeCode, extractCodex, extractCursor, type NormalizedTrace } from "@tracemarketplace/shared";
-import { loadConfig, loadState, resolveProfile, stateKey } from "../config.js";
+import { getConfigDir, getSubmitLogPath, loadConfig, loadState, resolveProfile, stateKey } from "../config.js";
 import { loginCommandForProfile } from "../constants.js";
 import {
   buildCursorSessionSource,
@@ -16,6 +18,7 @@ import {
   type SessionSource,
 } from "../flush.js";
 import { CURSOR_DB_PATH, findFiles } from "../sessions.js";
+import { createSubmitWorkerState, removeSubmitWorkerState, writeSubmitWorkerState } from "../submit-worker-state.js";
 
 interface SubmitOptions {
   profile?: string;
@@ -23,6 +26,7 @@ interface SubmitOptions {
   session?: string;
   dryRun?: boolean;
   since?: string;
+  sync?: boolean;
 }
 
 interface DiscoveredSession {
@@ -35,6 +39,18 @@ interface PlannedSession extends DiscoveredSession {
   pending: boolean;
 }
 
+interface SubmitWorkerOptions {
+  profile?: string;
+  manifest: string;
+}
+
+interface SubmitWorkerManifest {
+  createdAt: string;
+  readyChunkCount: number;
+  readySessionCount: number;
+  sources: SessionSource[];
+}
+
 function pushSessionIfNonEmpty(
   sessions: DiscoveredSession[],
   source: SessionSource,
@@ -268,41 +284,128 @@ export async function submitCommand(opts: SubmitOptions): Promise<void> {
     return;
   }
 
-  const uploadSpinner = ora(`Submitting ${readyChunkCount} finalized chunk(s) to ${config.profile}...`).start();
+  const readySessions = plannedSessions.filter((session) => session.readyChunks > 0);
 
-  try {
-    const readySessions = plannedSessions.filter((session) => session.readyChunks > 0);
+  if (opts.sync) {
+    const uploadSpinner = ora(`Submitting ${readyChunkCount} finalized chunk(s) to ${config.profile}...`).start();
     const prefetchedTraces = new Map(readySessions.map((s) => [`${s.source.tool}:${s.source.locator}`, s.trace]));
-    const result = await flushTrackedSessions(
-      config,
-      readySessions.map((session) => session.source),
-      { includeIdleTracked: false, prefetchedTraces }
-    );
+    try {
+      const result = await flushTrackedSessions(
+        config,
+        readySessions.map((session) => session.source),
+        { includeIdleTracked: false, prefetchedTraces, sync: true }
+      );
+
+      uploadSpinner.stop();
+      const failedSessions = result.results.filter((session) => session.error && session.error !== "Empty session");
+      console.log(chalk.green("\nSubmission complete!"));
+      console.log(`  Uploaded chunks:   ${chalk.bold(result.uploadedChunks)}`);
+      console.log(`  Duplicate chunks:  ${chalk.gray(result.duplicateChunks)}`);
+      console.log(`  Pending sessions:  ${result.pendingSessions}`);
+      console.log(`  Payout:            ${chalk.green("$" + (result.payoutCents / 100).toFixed(2))}`);
+      if (failedSessions.length > 0) {
+        console.log(chalk.yellow(`\n${failedSessions.length} session(s) failed during submit:`));
+        failedSessions.slice(0, 3).forEach((session) => {
+          console.log(chalk.gray(`  ${session.source.label}: ${session.error}`));
+        });
+        if (failedSessions.length > 3) {
+          console.log(chalk.gray(`  ...and ${failedSessions.length - 3} more`));
+        }
+      }
+    } catch (e) {
+      uploadSpinner.fail("Submission failed");
+      console.error(chalk.red(String(e)));
+      process.exit(1);
+    }
+    return;
+  }
+
+  const launchSpinner = ora(`Starting background submit for ${readyChunkCount} finalized chunk(s)...`).start();
 
-    uploadSpinner.stop();
+  try {
+    const { logPath, pid } = spawnDetachedSubmitWorker(config.profile, {
+      createdAt: new Date().toISOString(),
+      readyChunkCount,
+      readySessionCount,
+      sources: readySessions.map((session) => session.source),
+    });
+
+    launchSpinner.stop();
+    console.log(chalk.green(`\nStarted background submit for ${readyChunkCount} chunk(s).`));
+    if (pid) {
+      console.log(chalk.gray(`  Worker PID: ${pid}`));
+    }
+    console.log(chalk.gray(`  Log: ${logPath}`));
+    console.log(chalk.gray("  Run `tracemp status` later to check confirmation status."));
+  } catch (e) {
+    launchSpinner.fail("Could not start background submit");
+    console.error(chalk.red(String(e)));
+    process.exit(1);
+  }
+}
 
+export async function submitWorkerCommand(opts: SubmitWorkerOptions): Promise<void> {
+  const profile = resolveProfile(opts.profile);
+  try {
+    const manifest = parseSubmitWorkerManifest(await readFile(opts.manifest, "utf-8"));
+    const config = loadConfig(profile);
+    if (!config) {
+      throw new Error(`Not authenticated for profile '${profile}'. Run: ${loginCommandForProfile(profile)}`);
+    }
+
+    console.log(
+      `[${new Date().toISOString()}] submit worker starting profile=${profile} ready_chunks=${manifest.readyChunkCount} sessions=${manifest.readySessionCount}`
+    );
+
+    const result = await flushTrackedSessions(config, manifest.sources, { includeIdleTracked: false, sync: false });
     const failedSessions = result.results.filter((session) => session.error && session.error !== "Empty session");
 
-    console.log(chalk.green("\nSubmission complete!"));
-    console.log(`  Uploaded chunks:   ${chalk.bold(result.uploadedChunks)}`);
-    console.log(`  Duplicate chunks:  ${chalk.gray(result.duplicateChunks)}`);
-    console.log(`  Pending sessions:  ${result.pendingSessions}`);
-    console.log(`  Payout:            ${chalk.green("$" + (result.payoutCents / 100).toFixed(2))}`);
+    console.log(
+      `[${new Date().toISOString()}] submit worker queued chunks=${result.uploadedChunks} pending_sessions=${result.pendingSessions}`
+    );
 
     if (failedSessions.length > 0) {
-      console.log(chalk.yellow(`\n${failedSessions.length} session(s) failed during submit:`));
-      failedSessions.slice(0, 3).forEach((session) => {
-        console.log(chalk.gray(`  ${session.source.label}: ${session.error}`));
+      console.log(`[${new Date().toISOString()}] ${failedSessions.length} session(s) failed during submit:`);
+      failedSessions.slice(0, 10).forEach((session) => {
+        console.log(`  ${session.source.label}: ${session.error}`);
       });
-      if (failedSessions.length > 3) {
-        console.log(chalk.gray(`  ...and ${failedSessions.length - 3} more`));
+      if (failedSessions.length > 10) {
+        console.log(`  ...and ${failedSessions.length - 10} more`);
       }
     }
-  } catch (e) {
-    uploadSpinner.fail("Submission failed");
-    console.error(chalk.red(String(e)));
-    process.exit(1);
+  } finally {
+    removeSubmitWorkerState(profile, process.pid);
+    try {
+      unlinkSync(opts.manifest);
+    } catch {}
+  }
+}
+
+export function buildSubmitWorkerRuntime(profile: string, manifestPath: string): { command: string; args: string[] } {
+  const cliEntrypoint = process.argv[1];
+  if (!cliEntrypoint) {
+    throw new Error("Unable to resolve tracemp CLI entrypoint for background submit.");
   }
+
+  const execArgv = process.execArgv.filter((arg) => !arg.startsWith("--inspect"));
+  return {
+    command: process.execPath,
+    args: [...execArgv, cliEntrypoint, "submit-worker", "--profile", profile, "--manifest", manifestPath],
+  };
+}
+
+export function parseSubmitWorkerManifest(raw: string): SubmitWorkerManifest {
+  const parsed = JSON.parse(raw) as Partial<SubmitWorkerManifest>;
+  if (!Array.isArray(parsed.sources) || parsed.sources.some((source) => !isSessionSource(source))) {
+    throw new Error("Invalid submit worker manifest.");
+  }
+
+  return {
+    createdAt: typeof parsed.createdAt === "string" ? parsed.createdAt : new Date(0).toISOString(),
+    readyChunkCount: typeof parsed.readyChunkCount === "number" ? parsed.readyChunkCount : parsed.sources.length,
+    readySessionCount: typeof parsed.readySessionCount === "number" ? parsed.readySessionCount : parsed.sources.length,
+    sources: parsed.sources,
+  };
 }
 
 function describeSessionStatus(session: PlannedSession): string {
@@ -311,3 +414,64 @@ function describeSessionStatus(session: PlannedSession): string {
   if (session.pending) return "pending";
   return "up-to-date";
 }
+
+function isSessionSource(value: unknown): value is SessionSource {
+  if (!value || typeof value !== "object") {
+    return false;
+  }
+
+  const source = value as Record<string, unknown>;
+  return (
+    (source.tool === "claude_code" || source.tool === "codex_cli" || source.tool === "cursor")
+    && typeof source.locator === "string"
+    && typeof source.label === "string"
+  );
+}
+
+function spawnDetachedSubmitWorker(
+  profile: string,
+  manifest: SubmitWorkerManifest
+): { logPath: string; pid: number | null } {
+  mkdirSync(getConfigDir(), { recursive: true });
+  const manifestPath = join(
+    getConfigDir(),
+    `submit-job-${Date.now()}-${process.pid}-${Math.random().toString(36).slice(2, 8)}.json`
+  );
+  writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + "\n", "utf-8");
+
+  const logPath = getSubmitLogPath(profile);
+  const logFd = openSync(logPath, "a");
+
+  try {
+    const { command, args } = buildSubmitWorkerRuntime(profile, manifestPath);
+    const child = spawn(command, args, {
+      cwd: process.cwd(),
+      detached: true,
+      env: process.env,
+      stdio: ["ignore", logFd, logFd],
+    });
+    child.unref();
+
+    if (child.pid) {
+      writeSubmitWorkerState(
+        profile,
+        createSubmitWorkerState(
+          profile,
+          child.pid,
+          manifest.createdAt,
+          manifest.readyChunkCount,
+          manifest.readySessionCount,
+        ),
+      );
+    }
+
+    return { logPath, pid: child.pid ?? null };
+  } catch (err) {
+    try {
+      unlinkSync(manifestPath);
+    } catch {}
+    throw err;
+  } finally {
+    closeSync(logFd);
+  }
+}

package/src/config.ts

@@ -60,6 +60,14 @@ export function getAutoSubmitLogPath(profile?: string): string {
   return join(getConfigDir(), `auto-submit${profileSuffix(profile)}.log`);
 }
 
+export function getSubmitLogPath(profile?: string): string {
+  return join(getConfigDir(), `submit${profileSuffix(profile)}.log`);
+}
+
+export function getSubmitWorkerStateDir(profile?: string): string {
+  return join(getConfigDir(), `submit-workers${profileSuffix(profile)}`);
+}
+
 export function getDaemonStatePath(profile?: string): string {
   return join(getConfigDir(), `daemon-state${profileSuffix(profile)}.json`);
 }

package/src/flush.ts

@@ -141,7 +141,7 @@ export async function verifyUnconfirmedChunks(
 export async function flushTrackedSessions(
   config: Config,
   sources: SessionSource[],
-  opts: { includeIdleTracked?: boolean; now?: Date; prefetchedTraces?: Map<string, NormalizedTrace> } = {}
+  opts: { includeIdleTracked?: boolean; now?: Date; prefetchedTraces?: Map<string, NormalizedTrace>; sync?: boolean } = {}
 ): Promise<FlushResult> {
   const now = opts.now ?? new Date();
   const state = loadState(config.profile);
@@ -165,6 +165,7 @@ export async function flushTrackedSessions(
     allSources.map((source) => () => processSessionSource(
       source, state, config, client, now,
       opts.prefetchedTraces?.get(`${source.tool}:${source.locator}`),
+      opts.sync,
     )),
     INGEST_CONCURRENCY,
   );
@@ -278,6 +279,7 @@ async function processSessionSource(
   client: ApiClient,
   now: Date,
   prefetchedTrace?: NormalizedTrace,
+  sync = false,
 ): Promise<SessionFlushResult> {
   let trace: NormalizedTrace;
 
@@ -329,8 +331,9 @@ async function processSessionSource(
 
   for (const upload of plan.uploads) {
     const tUpload = Date.now();
-    const result = await uploadTraceChunk(upload.trace, client);
-    console.error(`[flush] ${key} chunk${upload.trace.chunk_index} done in ${Date.now()-tUpload}ms err=${result.error?.slice(0,60) ?? 'none'}`);
+    const result = await uploadTraceChunk(upload.trace, client, sync);
+    if (!sync) console.error(`[flush] ${key} chunk${upload.trace.chunk_index} queued in ${Date.now()-tUpload}ms`);
+    else console.error(`[flush] ${key} chunk${upload.trace.chunk_index} done in ${Date.now()-tUpload}ms err=${result.error?.slice(0,60) ?? 'none'}`);
     if (result.error) {
       state.sessions[key] = workingState;
       if (workingState.nextChunkIndex > 0) {
@@ -393,10 +396,22 @@ async function extractTraceFromSource(
 
 async function uploadTraceChunk(
   trace: NormalizedTrace,
-  client: ApiClient
+  client: ApiClient,
+  sync = false,
 ): Promise<ChunkUploadResult> {
   // Client-side regex redaction runs before transmission; Presidio runs server-side async.
   const payloadTrace = redactTrace(trace, { homeDir: homedir() });
+  const jsonSize = JSON.stringify({ trace: payloadTrace, source_tool: payloadTrace.source_tool }).length;
+  console.error(`[upload] ${payloadTrace.source_session_id} payload=${Math.round(jsonSize/1024)}KB turns=${payloadTrace.turn_count}`);
+
+  if (!sync) {
+    // Fire and forget — optimistically treat as queued; verifyUnconfirmedChunks confirms later
+    client.post("/api/v1/traces/ingest", {
+      trace: payloadTrace,
+      source_tool: payloadTrace.source_tool,
+    }).catch(() => { /* errors handled by unconfirmed retry mechanism */ });
+    return { duplicate: false, payoutCents: 0, traceId: null };
+  }
 
   try {
     const result = await client.post("/api/v1/traces/ingest", {

package/src/submit-worker-state.test.ts

@@ -0,0 +1,61 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { mkdtempSync, readFileSync, writeFileSync } from "fs";
+import { tmpdir } from "os";
+import { join } from "path";
+import {
+  createSubmitWorkerState,
+  listActiveSubmitWorkers,
+  removeSubmitWorkerState,
+  writeSubmitWorkerState,
+} from "./submit-worker-state.js";
+
+describe("submit worker state", () => {
+  const tempDirs: string[] = [];
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it("lists active workers and prunes stale ones", () => {
+    const dir = mkdtempSync(join(tmpdir(), "tracemp-submit-workers-"));
+    tempDirs.push(dir);
+
+    writeSubmitWorkerState("prod", createSubmitWorkerState("prod", 111, "2026-03-21T00:00:00.000Z", 3, 2), { dir });
+    writeSubmitWorkerState("prod", createSubmitWorkerState("prod", 222, "2026-03-21T01:00:00.000Z", 1, 1), { dir });
+
+    vi.spyOn(process, "kill").mockImplementation((pid: number | NodeJS.Signals) => {
+      if (pid === 111) {
+        return true as never;
+      }
+      const err = new Error("missing") as NodeJS.ErrnoException;
+      err.code = "ESRCH";
+      throw err;
+    });
+
+    const workers = listActiveSubmitWorkers("prod", { dir });
+
+    expect(workers).toEqual([
+      createSubmitWorkerState("prod", 111, "2026-03-21T00:00:00.000Z", 3, 2),
+    ]);
+    expect(() => readFileSync(join(dir, "222.json"), "utf-8")).toThrow();
+  });
+
+  it("removes worker state files", () => {
+    const dir = mkdtempSync(join(tmpdir(), "tracemp-submit-workers-"));
+    tempDirs.push(dir);
+
+    writeSubmitWorkerState("prod", createSubmitWorkerState("prod", 333, "2026-03-21T00:00:00.000Z", 2, 1), { dir });
+    removeSubmitWorkerState("prod", 333, { dir });
+
+    expect(() => readFileSync(join(dir, "333.json"), "utf-8")).toThrow();
+  });
+
+  it("prunes invalid state payloads", () => {
+    const dir = mkdtempSync(join(tmpdir(), "tracemp-submit-workers-"));
+    tempDirs.push(dir);
+    writeFileSync(join(dir, "bad.json"), JSON.stringify({ nope: true }), "utf-8");
+
+    expect(listActiveSubmitWorkers("prod", { dir })).toEqual([]);
+    expect(() => readFileSync(join(dir, "bad.json"), "utf-8")).toThrow();
+  });
+});

package/src/submit-worker-state.ts

@@ -0,0 +1,127 @@
+import { existsSync, mkdirSync, readdirSync, readFileSync, unlinkSync, writeFileSync } from "fs";
+import { join } from "path";
+import { getSubmitLogPath, getSubmitWorkerStateDir } from "./config.js";
+
+export interface SubmitWorkerState {
+  pid: number;
+  startedAt: string;
+  readyChunkCount: number;
+  readySessionCount: number;
+  logPath: string;
+}
+
+interface SubmitWorkerStateOptions {
+  dir?: string;
+}
+
+export function isProcessRunning(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    return (err as NodeJS.ErrnoException).code === "EPERM";
+  }
+}
+
+export function writeSubmitWorkerState(
+  profile: string,
+  state: SubmitWorkerState,
+  opts: SubmitWorkerStateOptions = {},
+): void {
+  const dir = resolveStateDir(profile, opts.dir);
+  mkdirSync(dir, { recursive: true });
+  writeFileSync(join(dir, `${state.pid}.json`), JSON.stringify(state, null, 2) + "\n", "utf-8");
+}
+
+export function removeSubmitWorkerState(
+  profile: string,
+  pid: number,
+  opts: SubmitWorkerStateOptions = {},
+): void {
+  const dir = resolveStateDir(profile, opts.dir);
+  const path = join(dir, `${pid}.json`);
+  if (!existsSync(path)) {
+    return;
+  }
+
+  try {
+    unlinkSync(path);
+  } catch {}
+}
+
+export function listActiveSubmitWorkers(
+  profile: string,
+  opts: SubmitWorkerStateOptions = {},
+): SubmitWorkerState[] {
+  const dir = resolveStateDir(profile, opts.dir);
+  if (!existsSync(dir)) {
+    return [];
+  }
+
+  const workers: SubmitWorkerState[] = [];
+  for (const entry of readdirSync(dir)) {
+    if (!entry.endsWith(".json")) {
+      continue;
+    }
+
+    const path = join(dir, entry);
+    const parsed = readSubmitWorkerStateFile(path);
+    if (!parsed || !isProcessRunning(parsed.pid)) {
+      try {
+        unlinkSync(path);
+      } catch {}
+      continue;
+    }
+
+    workers.push(parsed);
+  }
+
+  return workers.sort((a, b) => a.startedAt.localeCompare(b.startedAt));
+}
+
+export function createSubmitWorkerState(
+  profile: string,
+  pid: number,
+  startedAt: string,
+  readyChunkCount: number,
+  readySessionCount: number,
+): SubmitWorkerState {
+  return {
+    pid,
+    startedAt,
+    readyChunkCount,
+    readySessionCount,
+    logPath: getSubmitLogPath(profile),
+  };
+}
+
+function resolveStateDir(profile: string, dir?: string): string {
+  return dir ?? getSubmitWorkerStateDir(profile);
+}
+
+function readSubmitWorkerStateFile(path: string): SubmitWorkerState | null {
+  try {
+    const value = JSON.parse(readFileSync(path, "utf-8")) as Partial<SubmitWorkerState>;
+    if (
+      typeof value.pid !== "number"
+      || !Number.isInteger(value.pid)
+      || value.pid <= 0
+      || typeof value.startedAt !== "string"
+      || typeof value.readyChunkCount !== "number"
+      || typeof value.readySessionCount !== "number"
+      || typeof value.logPath !== "string"
+    ) {
+      return null;
+    }
+
+    return {
+      pid: value.pid,
+      startedAt: value.startedAt,
+      readyChunkCount: value.readyChunkCount,
+      readySessionCount: value.readySessionCount,
+      logPath: value.logPath,
+    };
+  } catch {
+    return null;
+  }
+}