@tracemarketplace/cli 0.0.18 → 0.0.20

package/src/commands/status.ts

@@ -1,9 +1,23 @@
 import chalk from "chalk";
-import { loadConfig, resolveProfile } from "../config.js";
+import { loadConfig, loadState, migrateSessionUploadState, resolveProfile, type SubmitState } from "../config.js";
 import { ApiClient } from "../api-client.js";
 import { loginCommandForProfile } from "../constants.js";
+import { listActiveSubmitWorkers, type SubmitWorkerState } from "../submit-worker-state.js";
+import { formatRemoteSubmissionDisplay, isSubmissionPipelinePending } from "../submission-status.js";
 
-export async function statusCommand(opts: { profile?: string } = {}): Promise<void> {
+interface StatusOptions {
+  profile?: string;
+  remote?: boolean;
+}
+
+interface LocalSubmitStatusSummary {
+  trackedSessions: number;
+  pendingSessions: number;
+  awaitingConfirmationSessions: number;
+  awaitingConfirmationChunks: number;
+}
+
+export async function statusCommand(opts: StatusOptions = {}): Promise<void> {
   const profile = resolveProfile(opts.profile);
   const config = loadConfig(profile);
   if (!config) {
@@ -11,20 +25,85 @@ export async function statusCommand(opts: { profile?: string } = {}): Promise<vo
     process.exit(1);
   }
 
+  const localState = loadState(config.profile);
+  const activeWorkers = listActiveSubmitWorkers(config.profile);
+  const summary = summarizeLocalSubmitState(localState, activeWorkers);
+
+  console.log(chalk.gray("Profile:"), config.profile);
+  console.log(
+    chalk.bold("Local upload worker:"),
+    activeWorkers.length > 0
+      ? chalk.yellow(`running (${activeWorkers.length})`)
+      : chalk.gray("idle"),
+  );
+  console.log(chalk.bold("Tracked sessions:"), summary.trackedSessions);
+  console.log(chalk.bold("Sessions still in progress:"), summary.pendingSessions);
+  console.log(
+    chalk.bold("Queued uploads awaiting server confirmation:"),
+    `${summary.awaitingConfirmationChunks} chunk(s) across ${summary.awaitingConfirmationSessions} session(s)`,
+  );
+
+  if (activeWorkers.length > 0) {
+    activeWorkers.slice(0, 3).forEach((worker) => {
+      console.log(
+        chalk.gray(
+          `  pid ${worker.pid} started ${worker.startedAt} ready ${worker.readyChunkCount} chunk(s) from ${worker.readySessionCount} session(s)`
+        )
+      );
+    });
+    if (activeWorkers.length > 3) {
+      console.log(chalk.gray(`  ...and ${activeWorkers.length - 3} more worker(s)`));
+    }
+    console.log(chalk.gray(`  Log: ${activeWorkers[0]!.logPath}`));
+  }
+
+  if (!opts.remote) {
+    return;
+  }
+
   const client = new ApiClient(config.serverUrl, config.apiKey);
   const [me, subs] = await Promise.all([
     client.get("/api/v1/me") as Promise<{ email: string; balanceCents?: number; balance_cents?: number }>,
     client.get("/api/v1/submissions") as Promise<any[]>,
   ]);
-
+  const sortedSubs = [...(subs as any[])].sort(
+    (a, b) => Date.parse(String(b.submittedAt)) - Date.parse(String(a.submittedAt)),
+  );
   const balance = ((me as any).balanceCents ?? (me as any).balance_cents ?? 0) / 100;
-  console.log(chalk.gray("Profile:"), config.profile);
-  console.log(chalk.gray("Server:"), config.serverUrl);
-  console.log(chalk.bold("Balance:"), chalk.green(`$${balance.toFixed(2)}`));
-  console.log(chalk.bold("Submissions:"), (subs as any[]).length);
+  const pending = sortedSubs.filter((s) => isSubmissionPipelinePending(s));
 
-  const pending = (subs as any[]).filter((s) => s.acceptedCount === null);
+  console.log();
+  console.log(chalk.gray("Remote server:"), config.serverUrl);
+  console.log(chalk.bold("Balance:"), chalk.green(`$${balance.toFixed(2)}`));
+  console.log(chalk.bold("Submission batches:"), sortedSubs.length);
   if (pending.length > 0) {
-    console.log(chalk.yellow(`\n${pending.length} pending submission(s)`));
+    console.log(chalk.yellow(`Server data pipeline: processing ${pending.length} submission batch(es)`));
+  } else if (sortedSubs.length > 0) {
+    console.log(chalk.green("Server data pipeline: idle"));
   }
+
+  if (sortedSubs.length > 0) {
+    const latest = formatRemoteSubmissionDisplay(sortedSubs[0]);
+    console.log(chalk.gray(`Latest batch: ${latest.statusLabel} (${latest.ingestedLabel})`));
+  }
+}
+
+export function summarizeLocalSubmitState(
+  state: SubmitState,
+  _activeWorkers: SubmitWorkerState[],
+): LocalSubmitStatusSummary {
+  const sessions = Object.values(state.sessions).map((session) => migrateSessionUploadState(session));
+  const pendingSessions = sessions.filter((session) => session.openChunkStartTurn < session.lastSeenTurnCount).length;
+  const awaitingConfirmationSessions = sessions.filter((session) => session.confirmedChunkIndex < session.nextChunkIndex).length;
+  const awaitingConfirmationChunks = sessions.reduce(
+    (sum, session) => sum + Math.max(0, session.nextChunkIndex - session.confirmedChunkIndex),
+    0,
+  );
+
+  return {
+    trackedSessions: sessions.length,
+    pendingSessions,
+    awaitingConfirmationSessions,
+    awaitingConfirmationChunks,
+  };
 }

package/src/commands/submit.test.ts

@@ -0,0 +1,64 @@
+import { afterEach, describe, expect, it } from "vitest";
+import { buildSubmitWorkerRuntime, parseSubmitWorkerManifest } from "./submit.js";
+
+const originalArgv = [...process.argv];
+const originalExecArgv = [...process.execArgv];
+
+describe("parseSubmitWorkerManifest", () => {
+  it("accepts valid session sources", () => {
+    const manifest = parseSubmitWorkerManifest(JSON.stringify({
+      createdAt: "2026-03-21T00:00:00.000Z",
+      readyChunkCount: 3,
+      readySessionCount: 2,
+      sources: [
+        {
+          tool: "codex_cli",
+          locator: "/tmp/session.jsonl",
+          label: "codex_cli:/tmp/session.jsonl",
+        },
+      ],
+    }));
+
+    expect(manifest.readyChunkCount).toBe(3);
+    expect(manifest.readySessionCount).toBe(2);
+    expect(manifest.sources).toHaveLength(1);
+  });
+
+  it("rejects invalid source payloads", () => {
+    expect(() => parseSubmitWorkerManifest(JSON.stringify({
+      sources: [{ tool: "not-a-tool", locator: 123, label: null }],
+    }))).toThrow("Invalid submit worker manifest.");
+  });
+});
+
+describe("buildSubmitWorkerRuntime", () => {
+  afterEach(() => {
+    process.argv = [...originalArgv];
+    Object.defineProperty(process, "execArgv", {
+      configurable: true,
+      value: [...originalExecArgv],
+    });
+  });
+
+  it("targets the hidden submit-worker command and strips inspect flags", () => {
+    process.argv = [process.execPath, "/tmp/tracemp-cli.js"];
+    Object.defineProperty(process, "execArgv", {
+      configurable: true,
+      value: ["--inspect=127.0.0.1:9229", "--loader", "tsx"],
+    });
+
+    expect(buildSubmitWorkerRuntime("prod", "/tmp/submit-job.json")).toEqual({
+      command: process.execPath,
+      args: [
+        "--loader",
+        "tsx",
+        "/tmp/tracemp-cli.js",
+        "submit-worker",
+        "--profile",
+        "prod",
+        "--manifest",
+        "/tmp/submit-job.json",
+      ],
+    });
+  });
+});

package/src/commands/submit.ts

@@ -1,10 +1,12 @@
-import { existsSync } from "fs";
+import { closeSync, existsSync, mkdirSync, openSync, unlinkSync, writeFileSync } from "fs";
 import { readFile } from "fs/promises";
+import { spawn } from "child_process";
+import { join } from "path";
 import chalk from "chalk";
 import ora from "ora";
 import inquirer from "inquirer";
 import { extractClaudeCode, extractCodex, extractCursor, type NormalizedTrace } from "@tracemarketplace/shared";
-import { loadConfig, loadState, resolveProfile, stateKey } from "../config.js";
+import { getConfigDir, getSubmitLogPath, loadConfig, loadState, resolveProfile, stateKey } from "../config.js";
 import { loginCommandForProfile } from "../constants.js";
 import {
   buildCursorSessionSource,
@@ -16,6 +18,7 @@ import {
   type SessionSource,
 } from "../flush.js";
 import { CURSOR_DB_PATH, findFiles } from "../sessions.js";
+import { createSubmitWorkerState, removeSubmitWorkerState, writeSubmitWorkerState } from "../submit-worker-state.js";
 
 interface SubmitOptions {
   profile?: string;
@@ -36,6 +39,18 @@ interface PlannedSession extends DiscoveredSession {
   pending: boolean;
 }
 
+interface SubmitWorkerOptions {
+  profile?: string;
+  manifest: string;
+}
+
+interface SubmitWorkerManifest {
+  createdAt: string;
+  readyChunkCount: number;
+  readySessionCount: number;
+  sources: SessionSource[];
+}
+
 function pushSessionIfNonEmpty(
   sessions: DiscoveredSession[],
   source: SessionSource,
@@ -269,24 +284,19 @@ export async function submitCommand(opts: SubmitOptions): Promise<void> {
     return;
   }
 
-  const uploadSpinner = ora(
-    opts.sync
-      ? `Submitting ${readyChunkCount} finalized chunk(s) to ${config.profile}...`
-      : `Queuing ${readyChunkCount} finalized chunk(s) to ${config.profile}...`
-  ).start();
+  const readySessions = plannedSessions.filter((session) => session.readyChunks > 0);
 
-  try {
-    const readySessions = plannedSessions.filter((session) => session.readyChunks > 0);
+  if (opts.sync) {
+    const uploadSpinner = ora(`Submitting ${readyChunkCount} finalized chunk(s) to ${config.profile}...`).start();
     const prefetchedTraces = new Map(readySessions.map((s) => [`${s.source.tool}:${s.source.locator}`, s.trace]));
-    const result = await flushTrackedSessions(
-      config,
-      readySessions.map((session) => session.source),
-      { includeIdleTracked: false, prefetchedTraces, sync: opts.sync }
-    );
-
-    uploadSpinner.stop();
+    try {
+      const result = await flushTrackedSessions(
+        config,
+        readySessions.map((session) => session.source),
+        { includeIdleTracked: false, prefetchedTraces, sync: true }
+      );
 
-    if (opts.sync) {
+      uploadSpinner.stop();
       const failedSessions = result.results.filter((session) => session.error && session.error !== "Empty session");
       console.log(chalk.green("\nSubmission complete!"));
       console.log(`  Uploaded chunks:   ${chalk.bold(result.uploadedChunks)}`);
@@ -302,20 +312,166 @@ export async function submitCommand(opts: SubmitOptions): Promise<void> {
           console.log(chalk.gray(`  ...and ${failedSessions.length - 3} more`));
         }
       }
-    } else {
-      console.log(chalk.green(`\nQueued ${readyChunkCount} chunk(s) — processing in background.`));
-      console.log(chalk.gray("  Run again to check confirmation status."));
+    } catch (e) {
+      uploadSpinner.fail("Submission failed");
+      console.error(chalk.red(String(e)));
+      process.exit(1);
+    }
+    return;
+  }
+
+  const launchSpinner = ora(`Starting background submit for ${readyChunkCount} finalized chunk(s)...`).start();
+
+  try {
+    const { logPath, pid } = spawnDetachedSubmitWorker(config.profile, {
+      createdAt: new Date().toISOString(),
+      readyChunkCount,
+      readySessionCount,
+      sources: readySessions.map((session) => session.source),
+    });
+
+    launchSpinner.stop();
+    console.log(chalk.green(`\nStarted background submit for ${readyChunkCount} chunk(s).`));
+    if (pid) {
+      console.log(chalk.gray(`  Worker PID: ${pid}`));
     }
+    console.log(chalk.gray(`  Log: ${logPath}`));
+    console.log(chalk.gray("  Run `tracemp status` later to check confirmation status."));
   } catch (e) {
-    uploadSpinner.fail("Submission failed");
+    launchSpinner.fail("Could not start background submit");
     console.error(chalk.red(String(e)));
     process.exit(1);
   }
 }
 
+export async function submitWorkerCommand(opts: SubmitWorkerOptions): Promise<void> {
+  const profile = resolveProfile(opts.profile);
+  try {
+    const manifest = parseSubmitWorkerManifest(await readFile(opts.manifest, "utf-8"));
+    const config = loadConfig(profile);
+    if (!config) {
+      throw new Error(`Not authenticated for profile '${profile}'. Run: ${loginCommandForProfile(profile)}`);
+    }
+
+    console.log(
+      `[${new Date().toISOString()}] submit worker starting profile=${profile} ready_chunks=${manifest.readyChunkCount} sessions=${manifest.readySessionCount}`
+    );
+
+    const result = await flushTrackedSessions(config, manifest.sources, { includeIdleTracked: false, sync: false });
+    const failedSessions = result.results.filter((session) => session.error && session.error !== "Empty session");
+
+    console.log(
+      `[${new Date().toISOString()}] submit worker queued chunks=${result.uploadedChunks} pending_sessions=${result.pendingSessions}`
+    );
+
+    if (failedSessions.length > 0) {
+      console.log(`[${new Date().toISOString()}] ${failedSessions.length} session(s) failed during submit:`);
+      failedSessions.slice(0, 10).forEach((session) => {
+        console.log(`  ${session.source.label}: ${session.error}`);
+      });
+      if (failedSessions.length > 10) {
+        console.log(`  ...and ${failedSessions.length - 10} more`);
+      }
+    }
+  } finally {
+    removeSubmitWorkerState(profile, process.pid);
+    try {
+      unlinkSync(opts.manifest);
+    } catch {}
+  }
+}
+
+export function buildSubmitWorkerRuntime(profile: string, manifestPath: string): { command: string; args: string[] } {
+  const cliEntrypoint = process.argv[1];
+  if (!cliEntrypoint) {
+    throw new Error("Unable to resolve tracemp CLI entrypoint for background submit.");
+  }
+
+  const execArgv = process.execArgv.filter((arg) => !arg.startsWith("--inspect"));
+  return {
+    command: process.execPath,
+    args: [...execArgv, cliEntrypoint, "submit-worker", "--profile", profile, "--manifest", manifestPath],
+  };
+}
+
+export function parseSubmitWorkerManifest(raw: string): SubmitWorkerManifest {
+  const parsed = JSON.parse(raw) as Partial<SubmitWorkerManifest>;
+  if (!Array.isArray(parsed.sources) || parsed.sources.some((source) => !isSessionSource(source))) {
+    throw new Error("Invalid submit worker manifest.");
+  }
+
+  return {
+    createdAt: typeof parsed.createdAt === "string" ? parsed.createdAt : new Date(0).toISOString(),
+    readyChunkCount: typeof parsed.readyChunkCount === "number" ? parsed.readyChunkCount : parsed.sources.length,
+    readySessionCount: typeof parsed.readySessionCount === "number" ? parsed.readySessionCount : parsed.sources.length,
+    sources: parsed.sources,
+  };
+}
+
 function describeSessionStatus(session: PlannedSession): string {
   if (session.readyChunks > 0 && session.pending) return "ready + pending";
   if (session.readyChunks > 0) return "ready";
   if (session.pending) return "pending";
   return "up-to-date";
 }
+
+function isSessionSource(value: unknown): value is SessionSource {
+  if (!value || typeof value !== "object") {
+    return false;
+  }
+
+  const source = value as Record<string, unknown>;
+  return (
+    (source.tool === "claude_code" || source.tool === "codex_cli" || source.tool === "cursor")
+    && typeof source.locator === "string"
+    && typeof source.label === "string"
+  );
+}
+
+function spawnDetachedSubmitWorker(
+  profile: string,
+  manifest: SubmitWorkerManifest
+): { logPath: string; pid: number | null } {
+  mkdirSync(getConfigDir(), { recursive: true });
+  const manifestPath = join(
+    getConfigDir(),
+    `submit-job-${Date.now()}-${process.pid}-${Math.random().toString(36).slice(2, 8)}.json`
+  );
+  writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + "\n", "utf-8");
+
+  const logPath = getSubmitLogPath(profile);
+  const logFd = openSync(logPath, "a");
+
+  try {
+    const { command, args } = buildSubmitWorkerRuntime(profile, manifestPath);
+    const child = spawn(command, args, {
+      cwd: process.cwd(),
+      detached: true,
+      env: process.env,
+      stdio: ["ignore", logFd, logFd],
+    });
+    child.unref();
+
+    if (child.pid) {
+      writeSubmitWorkerState(
+        profile,
+        createSubmitWorkerState(
+          profile,
+          child.pid,
+          manifest.createdAt,
+          manifest.readyChunkCount,
+          manifest.readySessionCount,
+        ),
+      );
+    }
+
+    return { logPath, pid: child.pid ?? null };
+  } catch (err) {
+    try {
+      unlinkSync(manifestPath);
+    } catch {}
+    throw err;
+  } finally {
+    closeSync(logFd);
+  }
+}

package/src/config.ts

@@ -60,6 +60,14 @@ export function getAutoSubmitLogPath(profile?: string): string {
   return join(getConfigDir(), `auto-submit${profileSuffix(profile)}.log`);
 }
 
+export function getSubmitLogPath(profile?: string): string {
+  return join(getConfigDir(), `submit${profileSuffix(profile)}.log`);
+}
+
+export function getSubmitWorkerStateDir(profile?: string): string {
+  return join(getConfigDir(), `submit-workers${profileSuffix(profile)}`);
+}
+
 export function getDaemonStatePath(profile?: string): string {
   return join(getConfigDir(), `daemon-state${profileSuffix(profile)}.json`);
 }

package/src/submission-status.test.ts

@@ -0,0 +1,35 @@
+import { describe, expect, it } from "vitest";
+import { formatRemoteSubmissionDisplay, isSubmissionPipelinePending } from "./submission-status.js";
+
+describe("submission status display", () => {
+  it("describes pipeline-pending submissions clearly", () => {
+    expect(formatRemoteSubmissionDisplay({
+      sessionCount: 5,
+      acceptedCount: null,
+      totalPayoutCents: null,
+    })).toEqual({
+      ingestedLabel: "—/5",
+      statusLabel: "pending data pipeline",
+      payoutLabel: "processing",
+      pendingPipeline: true,
+    });
+  });
+
+  it("describes fully ingested submissions clearly", () => {
+    expect(formatRemoteSubmissionDisplay({
+      sessionCount: 5,
+      acceptedCount: 5,
+      totalPayoutCents: 125,
+    })).toEqual({
+      ingestedLabel: "5/5",
+      statusLabel: "ingested successfully",
+      payoutLabel: "$1.25",
+      pendingPipeline: false,
+    });
+  });
+
+  it("detects whether a remote submission is still in the pipeline", () => {
+    expect(isSubmissionPipelinePending({ sessionCount: 1, acceptedCount: null, totalPayoutCents: null })).toBe(true);
+    expect(isSubmissionPipelinePending({ sessionCount: 1, acceptedCount: 1, totalPayoutCents: 0 })).toBe(false);
+  });
+});

package/src/submission-status.ts

@@ -0,0 +1,48 @@
+export interface RemoteSubmissionLike {
+  sessionCount: number;
+  acceptedCount: number | null;
+  totalPayoutCents: number | null;
+}
+
+export interface RemoteSubmissionDisplay {
+  ingestedLabel: string;
+  statusLabel: string;
+  payoutLabel: string;
+  pendingPipeline: boolean;
+}
+
+export function isSubmissionPipelinePending(submission: RemoteSubmissionLike): boolean {
+  return submission.acceptedCount === null;
+}
+
+export function formatRemoteSubmissionDisplay(submission: RemoteSubmissionLike): RemoteSubmissionDisplay {
+  if (submission.acceptedCount === null) {
+    return {
+      ingestedLabel: `—/${submission.sessionCount}`,
+      statusLabel: "pending data pipeline",
+      payoutLabel: "processing",
+      pendingPipeline: true,
+    };
+  }
+
+  const payoutLabel =
+    submission.totalPayoutCents !== null
+      ? "$" + (submission.totalPayoutCents / 100).toFixed(2)
+      : "pending";
+
+  if (submission.acceptedCount === submission.sessionCount) {
+    return {
+      ingestedLabel: `${submission.acceptedCount}/${submission.sessionCount}`,
+      statusLabel: "ingested successfully",
+      payoutLabel,
+      pendingPipeline: false,
+    };
+  }
+
+  return {
+    ingestedLabel: `${submission.acceptedCount}/${submission.sessionCount}`,
+    statusLabel: "completed",
+    payoutLabel,
+    pendingPipeline: false,
+  };
+}

package/src/submit-worker-state.test.ts

@@ -0,0 +1,61 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { mkdtempSync, readFileSync, writeFileSync } from "fs";
+import { tmpdir } from "os";
+import { join } from "path";
+import {
+  createSubmitWorkerState,
+  listActiveSubmitWorkers,
+  removeSubmitWorkerState,
+  writeSubmitWorkerState,
+} from "./submit-worker-state.js";
+
+describe("submit worker state", () => {
+  const tempDirs: string[] = [];
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it("lists active workers and prunes stale ones", () => {
+    const dir = mkdtempSync(join(tmpdir(), "tracemp-submit-workers-"));
+    tempDirs.push(dir);
+
+    writeSubmitWorkerState("prod", createSubmitWorkerState("prod", 111, "2026-03-21T00:00:00.000Z", 3, 2), { dir });
+    writeSubmitWorkerState("prod", createSubmitWorkerState("prod", 222, "2026-03-21T01:00:00.000Z", 1, 1), { dir });
+
+    vi.spyOn(process, "kill").mockImplementation((pid: number | NodeJS.Signals) => {
+      if (pid === 111) {
+        return true as never;
+      }
+      const err = new Error("missing") as NodeJS.ErrnoException;
+      err.code = "ESRCH";
+      throw err;
+    });
+
+    const workers = listActiveSubmitWorkers("prod", { dir });
+
+    expect(workers).toEqual([
+      createSubmitWorkerState("prod", 111, "2026-03-21T00:00:00.000Z", 3, 2),
+    ]);
+    expect(() => readFileSync(join(dir, "222.json"), "utf-8")).toThrow();
+  });
+
+  it("removes worker state files", () => {
+    const dir = mkdtempSync(join(tmpdir(), "tracemp-submit-workers-"));
+    tempDirs.push(dir);
+
+    writeSubmitWorkerState("prod", createSubmitWorkerState("prod", 333, "2026-03-21T00:00:00.000Z", 2, 1), { dir });
+    removeSubmitWorkerState("prod", 333, { dir });
+
+    expect(() => readFileSync(join(dir, "333.json"), "utf-8")).toThrow();
+  });
+
+  it("prunes invalid state payloads", () => {
+    const dir = mkdtempSync(join(tmpdir(), "tracemp-submit-workers-"));
+    tempDirs.push(dir);
+    writeFileSync(join(dir, "bad.json"), JSON.stringify({ nope: true }), "utf-8");
+
+    expect(listActiveSubmitWorkers("prod", { dir })).toEqual([]);
+    expect(() => readFileSync(join(dir, "bad.json"), "utf-8")).toThrow();
+  });
+});