@tracemarketplace/cli 0.0.15 → 0.0.18

package/src/commands/remove-daemon.ts

@@ -0,0 +1,75 @@
+import { existsSync, readFileSync, unlinkSync } from "fs";
+import chalk from "chalk";
+import { DEFAULT_PROFILE } from "../constants.js";
+import { getDaemonPidPath, resolveProfile } from "../config.js";
+
+interface RemoveDaemonOptions {
+  profile?: string;
+}
+
+function readDaemonPid(profile: string): number | null {
+  const pidPath = getDaemonPidPath(profile);
+  if (!existsSync(pidPath)) return null;
+
+  try {
+    const parsed = Number.parseInt(readFileSync(pidPath, "utf-8").trim(), 10);
+    return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
+  } catch {
+    return null;
+  }
+}
+
+function isProcessRunning(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    return (err as NodeJS.ErrnoException).code === "EPERM";
+  }
+}
+
+function removePidFile(profile: string) {
+  const pidPath = getDaemonPidPath(profile);
+  if (!existsSync(pidPath)) return;
+  try {
+    unlinkSync(pidPath);
+  } catch {}
+}
+
+async function waitForExit(pid: number, timeoutMs = 3000): Promise<boolean> {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (!isProcessRunning(pid)) {
+      return true;
+    }
+    await new Promise((resolve) => setTimeout(resolve, 100));
+  }
+  return !isProcessRunning(pid);
+}
+
+export async function removeDaemonCommand(opts: RemoveDaemonOptions = {}): Promise<void> {
+  const profile = resolveProfile(opts.profile);
+  const pid = readDaemonPid(profile);
+  const profileLabel = profile === DEFAULT_PROFILE ? "default profile" : `profile '${profile}'`;
+
+  if (pid === null) {
+    console.log(chalk.gray(`No tracemp daemon found for ${profileLabel}.`));
+    return;
+  }
+
+  if (!isProcessRunning(pid)) {
+    removePidFile(profile);
+    console.log(chalk.gray(`Removed stale daemon PID file for ${profileLabel}.`));
+    return;
+  }
+
+  process.kill(pid, "SIGTERM");
+
+  if (await waitForExit(pid)) {
+    removePidFile(profile);
+    console.log(chalk.green(`✓ Stopped tracemp daemon for ${profileLabel}`));
+    return;
+  }
+
+  console.log(chalk.yellow(`Sent SIGTERM to tracemp daemon for ${profileLabel} (pid ${pid}), but it is still running.`));
+}

package/src/commands/submit.ts

@@ -23,6 +23,7 @@ interface SubmitOptions {
   session?: string;
   dryRun?: boolean;
   since?: string;
+  sync?: boolean;
 }
 
 interface DiscoveredSession {
@@ -268,33 +269,42 @@ export async function submitCommand(opts: SubmitOptions): Promise<void> {
     return;
   }
 
-  const uploadSpinner = ora(`Submitting ${readyChunkCount} finalized chunk(s) to ${config.profile}...`).start();
+  const uploadSpinner = ora(
+    opts.sync
+      ? `Submitting ${readyChunkCount} finalized chunk(s) to ${config.profile}...`
+      : `Queuing ${readyChunkCount} finalized chunk(s) to ${config.profile}...`
+  ).start();
 
   try {
+    const readySessions = plannedSessions.filter((session) => session.readyChunks > 0);
+    const prefetchedTraces = new Map(readySessions.map((s) => [`${s.source.tool}:${s.source.locator}`, s.trace]));
     const result = await flushTrackedSessions(
       config,
-      plannedSessions.map((session) => session.source),
-      { includeIdleTracked: false }
+      readySessions.map((session) => session.source),
+      { includeIdleTracked: false, prefetchedTraces, sync: opts.sync }
     );
 
     uploadSpinner.stop();
 
-    const failedSessions = result.results.filter((session) => session.error && session.error !== "Empty session");
-
-    console.log(chalk.green("\nSubmission complete!"));
-    console.log(`  Uploaded chunks:   ${chalk.bold(result.uploadedChunks)}`);
-    console.log(`  Duplicate chunks:  ${chalk.gray(result.duplicateChunks)}`);
-    console.log(`  Pending sessions:  ${result.pendingSessions}`);
-    console.log(`  Payout:            ${chalk.green("$" + (result.payoutCents / 100).toFixed(2))}`);
-
-    if (failedSessions.length > 0) {
-      console.log(chalk.yellow(`\n${failedSessions.length} session(s) failed during submit:`));
-      failedSessions.slice(0, 3).forEach((session) => {
-        console.log(chalk.gray(`  ${session.source.label}: ${session.error}`));
-      });
-      if (failedSessions.length > 3) {
-        console.log(chalk.gray(`  ...and ${failedSessions.length - 3} more`));
+    if (opts.sync) {
+      const failedSessions = result.results.filter((session) => session.error && session.error !== "Empty session");
+      console.log(chalk.green("\nSubmission complete!"));
+      console.log(`  Uploaded chunks:   ${chalk.bold(result.uploadedChunks)}`);
+      console.log(`  Duplicate chunks:  ${chalk.gray(result.duplicateChunks)}`);
+      console.log(`  Pending sessions:  ${result.pendingSessions}`);
+      console.log(`  Payout:            ${chalk.green("$" + (result.payoutCents / 100).toFixed(2))}`);
+      if (failedSessions.length > 0) {
+        console.log(chalk.yellow(`\n${failedSessions.length} session(s) failed during submit:`));
+        failedSessions.slice(0, 3).forEach((session) => {
+          console.log(chalk.gray(`  ${session.source.label}: ${session.error}`));
+        });
+        if (failedSessions.length > 3) {
+          console.log(chalk.gray(`  ...and ${failedSessions.length - 3} more`));
+        }
       }
+    } else {
+      console.log(chalk.green(`\nQueued ${readyChunkCount} chunk(s) — processing in background.`));
+      console.log(chalk.gray("  Run again to check confirmation status."));
     }
   } catch (e) {
     uploadSpinner.fail("Submission failed");

package/src/config.ts

@@ -29,6 +29,10 @@ export interface SessionUploadState {
   lastSeenTurnCount: number;
   lastActivityAt: string | null;
   lastFlushedTurnId: string | null;
+  // Async confirmation tracking
+  confirmedChunkIndex: number;         // all chunks below this index are confirmed in DB
+  confirmedOpenChunkStartTurn: number; // openChunkStartTurn to restore if re-submitting
+  unconfirmedSince: string | null;     // ISO timestamp when chunks first went unconfirmed
 }
 
 export interface SubmitState {
@@ -60,6 +64,10 @@ export function getDaemonStatePath(profile?: string): string {
   return join(getConfigDir(), `daemon-state${profileSuffix(profile)}.json`);
 }
 
+export function getDaemonPidPath(profile?: string): string {
+  return join(getConfigDir(), `daemon${profileSuffix(profile)}.pid`);
+}
+
 export function resolveProfile(profile?: string): string {
   if (profile) return normalizeProfile(profile);
   if (process.env.TRACEMP_PROFILE) return normalizeProfile(process.env.TRACEMP_PROFILE);
@@ -261,3 +269,13 @@ function isSessionUploadState(value: unknown): value is SessionUploadState {
     && (value.lastActivityAt === null || typeof value.lastActivityAt === "string")
     && (value.lastFlushedTurnId === null || typeof value.lastFlushedTurnId === "string");
 }
+
+export function migrateSessionUploadState(value: SessionUploadState): SessionUploadState {
+  return {
+    ...value,
+    // Backward compat: existing sessions assume all submitted chunks are confirmed
+    confirmedChunkIndex: value.confirmedChunkIndex ?? value.nextChunkIndex,
+    confirmedOpenChunkStartTurn: value.confirmedOpenChunkStartTurn ?? value.openChunkStartTurn,
+    unconfirmedSince: value.unconfirmedSince ?? null,
+  };
+}

package/src/flush.test.ts

@@ -1,12 +1,15 @@
 import { describe, expect, it } from "vitest";
 import type { NormalizedTrace, Turn } from "@tracemarketplace/shared";
 import type { SessionSource } from "./flush.js";
+import type { SessionUploadState } from "./config.js";
 import {
   collectIdleSessionSources,
   createFreshSessionState,
   migrateLegacySessionState,
   planSessionUploads,
+  verifyUnconfirmedChunks,
 } from "./flush.js";
+import { migrateSessionUploadState } from "./config.js";
 
 function makeTurn(
   turnId: string,
@@ -81,6 +84,20 @@ function makeSource(tool: SessionSource["tool"], locator: string): SessionSource
   return { tool, locator, label: locator };
 }
 
+function makeSessionState(overrides: Partial<SessionUploadState> & Pick<SessionUploadState, "sourceTool" | "sourceSessionId" | "locator">): SessionUploadState {
+  return {
+    nextChunkIndex: 0,
+    openChunkStartTurn: 0,
+    lastSeenTurnCount: 0,
+    lastActivityAt: null,
+    lastFlushedTurnId: null,
+    confirmedChunkIndex: 0,
+    confirmedOpenChunkStartTurn: 0,
+    unconfirmedSince: null,
+    ...overrides,
+  };
+}
+
 describe("planSessionUploads", () => {
   it("flushes a sealed 100k chunk and keeps the tail pending", () => {
     const trace = makeTrace("session-100k", [
@@ -168,10 +185,174 @@ describe("planSessionUploads", () => {
   });
 });
 
+describe("migrateSessionUploadState", () => {
+  it("fills in missing confirmation fields from next/openChunk values", () => {
+    const legacy = {
+      sourceTool: "codex_cli" as const,
+      sourceSessionId: "s1",
+      locator: "/tmp/s1.jsonl",
+      nextChunkIndex: 3,
+      openChunkStartTurn: 10,
+      lastSeenTurnCount: 10,
+      lastActivityAt: "2026-03-21T00:00:00.000Z",
+      lastFlushedTurnId: "a3",
+      // missing: confirmedChunkIndex, confirmedOpenChunkStartTurn, unconfirmedSince
+    } as any;
+
+    const migrated = migrateSessionUploadState(legacy);
+
+    expect(migrated.confirmedChunkIndex).toBe(3);
+    expect(migrated.confirmedOpenChunkStartTurn).toBe(10);
+    expect(migrated.unconfirmedSince).toBeNull();
+  });
+
+  it("does not overwrite existing confirmation fields", () => {
+    const state = makeSessionState({
+      sourceTool: "codex_cli",
+      sourceSessionId: "s2",
+      locator: "/tmp/s2.jsonl",
+      nextChunkIndex: 5,
+      openChunkStartTurn: 20,
+      confirmedChunkIndex: 3,
+      confirmedOpenChunkStartTurn: 12,
+      unconfirmedSince: "2026-03-21T01:00:00.000Z",
+    });
+
+    const migrated = migrateSessionUploadState(state);
+
+    expect(migrated.confirmedChunkIndex).toBe(3);
+    expect(migrated.confirmedOpenChunkStartTurn).toBe(12);
+    expect(migrated.unconfirmedSince).toBe("2026-03-21T01:00:00.000Z");
+  });
+});
+
+describe("verifyUnconfirmedChunks", () => {
+  function makeSubmitState(sessions: Record<string, ReturnType<typeof makeSessionState>>) {
+    return { version: 2 as const, chunks: {}, sessions };
+  }
+
+  function makeMockClient(responses: Record<string, { exists: boolean }>) {
+    return {
+      async get(path: string) {
+        const url = new URL(`http://x${path}`);
+        const tool = url.searchParams.get("source_tool")!;
+        const id = url.searchParams.get("source_session_id")!;
+        const idx = url.searchParams.get("chunk_index")!;
+        const key = `${tool}:${id}:${idx}`;
+        const r = responses[key];
+        if (!r) throw new Error(`Unexpected exists check: ${key}`);
+        return r;
+      },
+    } as any;
+  }
+
+  it("advances confirmedChunkIndex when chunks are confirmed", async () => {
+    const state = makeSubmitState({
+      "codex_cli:sess1": makeSessionState({
+        sourceTool: "codex_cli",
+        sourceSessionId: "sess1",
+        locator: "/tmp/sess1.jsonl",
+        nextChunkIndex: 3,
+        openChunkStartTurn: 12,
+        confirmedChunkIndex: 1,
+        confirmedOpenChunkStartTurn: 4,
+        unconfirmedSince: "2026-03-21T00:00:00.000Z",
+      }),
+    });
+
+    const client = makeMockClient({
+      "codex_cli:sess1:1": { exists: true },
+      "codex_cli:sess1:2": { exists: true },
+    });
+
+    await verifyUnconfirmedChunks(state, client, new Date("2026-03-21T01:00:00.000Z"));
+
+    const s = state.sessions["codex_cli:sess1"]!;
+    expect(s.confirmedChunkIndex).toBe(3);
+    // all confirmed → unconfirmedSince cleared
+    expect(s.unconfirmedSince).toBeNull();
+  });
+
+  it("stops advancing at first missing chunk", async () => {
+    const state = makeSubmitState({
+      "codex_cli:sess2": makeSessionState({
+        sourceTool: "codex_cli",
+        sourceSessionId: "sess2",
+        locator: "/tmp/sess2.jsonl",
+        nextChunkIndex: 3,
+        openChunkStartTurn: 12,
+        confirmedChunkIndex: 1,
+        confirmedOpenChunkStartTurn: 4,
+        unconfirmedSince: "2026-03-21T00:00:00.000Z",
+      }),
+    });
+
+    const client = makeMockClient({
+      "codex_cli:sess2:1": { exists: true },
+      "codex_cli:sess2:2": { exists: false },
+    });
+
+    await verifyUnconfirmedChunks(state, client, new Date("2026-03-21T01:00:00.000Z"));
+
+    const s = state.sessions["codex_cli:sess2"]!;
+    expect(s.confirmedChunkIndex).toBe(2);
+    // still one unconfirmed chunk → unconfirmedSince preserved
+    expect(s.unconfirmedSince).toBe("2026-03-21T00:00:00.000Z");
+  });
+
+  it("resets to confirmed state after 2hr timeout", async () => {
+    const state = makeSubmitState({
+      "codex_cli:sess3": makeSessionState({
+        sourceTool: "codex_cli",
+        sourceSessionId: "sess3",
+        locator: "/tmp/sess3.jsonl",
+        nextChunkIndex: 3,
+        openChunkStartTurn: 12,
+        confirmedChunkIndex: 1,
+        confirmedOpenChunkStartTurn: 4,
+        unconfirmedSince: "2026-03-21T00:00:00.000Z",
+      }),
+    });
+
+    const client = makeMockClient({}); // should not be called
+
+    // now = 2h01m after unconfirmedSince
+    await verifyUnconfirmedChunks(state, client, new Date("2026-03-21T02:01:00.000Z"));
+
+    const s = state.sessions["codex_cli:sess3"]!;
+    // reset back to confirmed baseline so chunks get re-submitted
+    expect(s.nextChunkIndex).toBe(1);
+    expect(s.openChunkStartTurn).toBe(4);
+    expect(s.unconfirmedSince).toBeNull();
+  });
+
+  it("skips sessions that are already fully confirmed", async () => {
+    const state = makeSubmitState({
+      "codex_cli:sess4": makeSessionState({
+        sourceTool: "codex_cli",
+        sourceSessionId: "sess4",
+        locator: "/tmp/sess4.jsonl",
+        nextChunkIndex: 2,
+        openChunkStartTurn: 8,
+        confirmedChunkIndex: 2,
+        confirmedOpenChunkStartTurn: 8,
+        unconfirmedSince: null,
+      }),
+    });
+
+    let callCount = 0;
+    const client = { async get() { callCount++; return { exists: true }; } } as any;
+
+    await verifyUnconfirmedChunks(state, client, new Date("2026-03-21T01:00:00.000Z"));
+
+    expect(callCount).toBe(0);
+  });
+});
+
 describe("collectIdleSessionSources", () => {
   it("returns only tracked sessions with an open tail older than two days", () => {
     const sources = collectIdleSessionSources({
-      "codex_cli:idle": {
+      "codex_cli:idle": makeSessionState({
         sourceTool: "codex_cli",
         sourceSessionId: "idle",
         locator: "/tmp/idle.jsonl",
@@ -180,8 +361,8 @@ describe("collectIdleSessionSources", () => {
         lastSeenTurnCount: 4,
         lastActivityAt: "2026-03-21T00:00:00.000Z",
         lastFlushedTurnId: "a1",
-      },
-      "codex_cli:closed": {
+      }),
+      "codex_cli:closed": makeSessionState({
         sourceTool: "codex_cli",
         sourceSessionId: "closed",
         locator: "/tmp/closed.jsonl",
@@ -190,8 +371,8 @@ describe("collectIdleSessionSources", () => {
         lastSeenTurnCount: 4,
         lastActivityAt: "2026-03-21T00:00:00.000Z",
         lastFlushedTurnId: "a2",
-      },
-      "codex_cli:fresh": {
+      }),
+      "codex_cli:fresh": makeSessionState({
         sourceTool: "codex_cli",
         sourceSessionId: "fresh",
         locator: "/tmp/fresh.jsonl",
@@ -200,7 +381,7 @@ describe("collectIdleSessionSources", () => {
         lastSeenTurnCount: 2,
         lastActivityAt: "2026-03-22T23:59:59.000Z",
         lastFlushedTurnId: null,
-      },
+      }),
     }, new Date("2026-03-23T00:01:00.000Z"));
 
     expect(sources).toEqual([