iteration-cli 0.16.1 → 0.17.1

package/README.md

@@ -1,6 +1,6 @@
 # Iteration CLI (private source)
 
-This package owns the `iteration-cli@0.16.1` product executable. Clerk sign-in,
+This package owns the `iteration-cli@0.17.1` product executable. Clerk sign-in,
 workspace enrollment and automatic expansion, the default Iteration endpoint,
 commands/status UI, history progress, friction summaries and Codex/Cursor billing
 clients live here, not in the public Trace Vault repository.
@@ -38,7 +38,7 @@ From the private repository root after workspace initialization:
 ```sh
 pnpm --filter "iteration-cli..." --recursive --workspace-concurrency=4 run build
 pnpm --filter iteration-cli pack --pack-destination /tmp/iteration-cli-pack
-node packages/cli/smoke.mjs /tmp/iteration-cli-pack/iteration-cli-0.16.1.tgz
+node packages/cli/smoke.mjs /tmp/iteration-cli-pack/iteration-cli-0.17.1.tgz
 ```
 
 The package smoke installs the tarball offline under a disposable home and checks

package/dist/cli.js

@@ -13037,7 +13037,8 @@ function collectorPaths(directory) {
     redactionMarker: path4.join(directory, "redaction.json"),
     lock: path4.join(directory, "collector.lock"),
     historyBackfill: path4.join(directory, "history-backfill.json"),
-    health: path4.join(directory, "health.json")
+    health: path4.join(directory, "health.json"),
+    recentMessages: path4.join(directory, "recent-messages.json")
   };
 }
 async function assertFreshCollectorDirectory(paths, source = "codex") {
@@ -13294,7 +13295,24 @@ async function redactPendingOutbox(paths) {
   await ensurePrivateDirectory(path4.dirname(paths.redactionMarker));
   await writeJsonAtomic(paths.redactionMarker, { version: 1 });
 }
-async function nextBatch(paths, maxBytes) {
+function queuedEventId(line) {
+  const prefix = '{"eventId":"';
+  if (!line.startsWith(prefix)) return null;
+  const end = line.indexOf('"', prefix.length);
+  return end < 0 ? null : line.slice(prefix.length, end);
+}
+async function queuedUserMessageIds(paths, limit2) {
+  const ids = [];
+  if (await outboxSize(paths) === 0) return ids;
+  for await (const line of outboxLines(paths.outbox)) {
+    if (ids.length >= limit2) break;
+    if (!line.includes('"eventType":"user_message"')) continue;
+    const id = queuedEventId(line);
+    if (id !== null) ids.push(id);
+  }
+  return ids;
+}
+async function nextBatch(paths, maxBytes, first = /* @__PURE__ */ new Set()) {
   if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) {
     throw new Error("Maximum upload batch size must be a positive integer.");
   }
@@ -13302,9 +13320,19 @@ async function nextBatch(paths, maxBytes) {
   if (size === 0) return [];
   if (size > MAX_OUTBOX_BYTES)
     throw new Error("Trace outbox exceeds its configured capacity.");
+  const priority = [];
+  const rest = [];
+  for await (const line of outboxLines(paths.outbox)) {
+    const id = first.size === 0 ? null : queuedEventId(line);
+    if (id !== null && first.has(id)) {
+      if (priority.length < MAX_TRACE_EVENTS_PER_BATCH) priority.push(line);
+    } else if (rest.length < MAX_TRACE_EVENTS_PER_BATCH) rest.push(line);
+    if (first.size === 0 ? rest.length === MAX_TRACE_EVENTS_PER_BATCH : priority.length === MAX_TRACE_EVENTS_PER_BATCH)
+      break;
+  }
   const events = [];
   let bytes = 0;
-  for await (const line of outboxLines(paths.outbox)) {
+  for (const line of [...priority, ...rest]) {
     const lineBytes = Buffer.byteLength(line, "utf8") + 1;
     if (events.length >= MAX_TRACE_EVENTS_PER_BATCH || events.length > 0 && bytes + lineBytes > maxBytes) {
       break;
@@ -13317,6 +13345,45 @@ async function nextBatch(paths, maxBytes) {
   }
   return events;
 }
+var recentMessagesSchema = object({
+  version: literal(1),
+  cursors: record(
+    string2(),
+    number2().int().min(0).max(Number.MAX_SAFE_INTEGER)
+  ),
+  // Queued by the pass and not yet acknowledged.
+  queued: array(string2()),
+  sent: number2().int().min(0),
+  scanned: boolean2()
+}).strict();
+async function loadRecentMessages(paths) {
+  let contents;
+  try {
+    contents = await fs3.readFile(paths.recentMessages, "utf8");
+  } catch (error22) {
+    if (error22 instanceof Error && "code" in error22 && error22.code === "ENOENT")
+      return null;
+    throw error22;
+  }
+  let decoded;
+  try {
+    decoded = JSON.parse(contents);
+  } catch (error22) {
+    throw new Error("Collector recent message state is not valid JSON.", {
+      cause: error22
+    });
+  }
+  const parsed = recentMessagesSchema.safeParse(decoded);
+  if (!parsed.success)
+    throw new Error("Collector recent message state has an invalid shape.", {
+      cause: parsed.error
+    });
+  return parsed.data;
+}
+async function saveRecentMessages(state, paths) {
+  await ensurePrivateDirectory(path4.dirname(paths.recentMessages));
+  await writeJsonAtomic(paths.recentMessages, state);
+}
 async function acknowledgeEvents(acknowledgedIds, paths) {
   await rewritePendingEvents(
     paths,
@@ -15353,18 +15420,166 @@ async function scanSourceOnce(config22, paths, signal, policy, accounts) {
     },
     paths
   );
-  const sessions = sessionFiles.filter((file) => file.session);
   return {
     discovered: events.length,
     excludedRecords,
     caughtUp,
-    history: {
-      sessions: sessions.length,
-      readSessions: sessions.filter(
-        (file) => (cursors[file.filename] ?? 0) >= file.size
-      ).length
+    history: historyProgress(sessionFiles, cursors)
+  };
+}
+function historyProgress(files, cursors) {
+  const sessions = files.filter((file) => file.session);
+  return {
+    sessions: sessions.length,
+    readSessions: sessions.filter(
+      (file) => (cursors[file.filename] ?? 0) >= file.size
+    ).length
+  };
+}
+async function savedHistoryProgress(config22, paths) {
+  const files = await sourceFilesNewestFirst(
+    collectorSource(config22),
+    path14.resolve(collectorSourceDirectory(config22))
+  );
+  return historyProgress(files, (await loadState(paths)).cursors);
+}
+async function readRange(filename, start2, length) {
+  const handle = await fs9.open(filename, "r");
+  try {
+    const buffer = Buffer.alloc(length);
+    const result = await handle.read(buffer, 0, length, start2);
+    return buffer.subarray(0, result.bytesRead);
+  } finally {
+    await handle.close();
+  }
+}
+async function messagesInRecords(source, filename, buffer, start2, room, queuedIds) {
+  const events = [];
+  let lineStart = 0;
+  for (let index = 0; index < buffer.length; index += 1) {
+    if (events.length >= room) break;
+    if (buffer[index] !== 10) continue;
+    let lineBytes = buffer.subarray(lineStart, index);
+    if (lineBytes.at(-1) === 13) lineBytes = lineBytes.subarray(0, -1);
+    const byteOffset = start2 + lineStart;
+    lineStart = index + 1;
+    if (lineBytes.length > MAX_RECORD_BYTES) continue;
+    let line;
+    try {
+      line = UTF8_DECODER.decode(lineBytes);
+    } catch {
+      continue;
+    }
+    const { events: parsed } = parseSourceRecord(
+      source,
+      line,
+      filename,
+      byteOffset
+    );
+    for (const record32 of parsed) {
+      if (record32.eventType !== "user_message" || queuedIds.has(record32.eventId))
+        continue;
+      const event3 = await redactTraceEvent(record32);
+      if (Buffer.byteLength(JSON.stringify(event3), "utf8") <= MAX_EVENT_BYTES)
+        events.push(event3);
     }
+  }
+  return { events, consumed: lineStart };
+}
+async function queueRecentMessages(config22, paths, limit2, signal) {
+  if (limit2 === void 0) return void 0;
+  const recent = await loadRecentMessages(paths) ?? {
+    version: 1,
+    cursors: {},
+    queued: [...await queuedUserMessageIds(paths, limit2)],
+    sent: 0,
+    scanned: false
   };
+  if (recent.scanned) return new Set(recent.queued);
+  const source = collectorSource(config22);
+  const mainCursors = (await loadState(paths)).cursors;
+  const files = await sourceFilesNewestFirst(
+    source,
+    path14.resolve(collectorSourceDirectory(config22))
+  );
+  const queuedIds = await pendingEventIds(paths);
+  const cursors = { ...recent.cursors };
+  const queued = [...recent.queued];
+  const events = [];
+  let budget = MAX_SCAN_BYTES_PER_TICK;
+  let reachedEnd = true;
+  for (const file of files) {
+    signal?.throwIfAborted();
+    const room = limit2 - recent.sent - queued.length;
+    const start2 = Math.max(
+      cursors[file.filename] ?? 0,
+      mainCursors[file.filename] ?? 0
+    );
+    if (room <= 0 || start2 >= file.size) continue;
+    const length = Math.min(file.size - start2, MAX_SCAN_BYTES_PER_FILE, budget);
+    const buffer = await readRange(file.filename, start2, length);
+    budget -= buffer.length;
+    const found = await messagesInRecords(
+      source,
+      file.filename,
+      buffer,
+      start2,
+      room,
+      queuedIds
+    );
+    for (const event3 of found.events) {
+      events.push(event3);
+      queued.push(event3.eventId);
+      queuedIds.add(event3.eventId);
+    }
+    const stuck = found.consumed === 0 && length === MAX_SCAN_BYTES_PER_FILE;
+    cursors[file.filename] = stuck ? file.size : start2 + found.consumed;
+    if (buffer.length < file.size - start2 || budget === 0) {
+      reachedEnd = false;
+      if (budget === 0) break;
+    }
+  }
+  await appendEvents(events, paths);
+  await saveRecentMessages(
+    {
+      version: 1,
+      cursors,
+      queued,
+      sent: recent.sent,
+      scanned: recent.sent + queued.length >= limit2 || reachedEnd
+    },
+    paths
+  );
+  return new Set(queued);
+}
+async function settleRecentMessages(paths, limit2) {
+  if (limit2 === void 0) return null;
+  const recent = await loadRecentMessages(paths);
+  if (recent === null) return { sent: 0, complete: false };
+  if (recent.queued.length === 0)
+    return { sent: recent.sent, complete: recent.scanned };
+  const pending = await pendingEventIds(paths);
+  const queued = recent.queued.filter((id) => pending.has(id));
+  const sent = recent.sent + recent.queued.length - queued.length;
+  if (queued.length !== recent.queued.length)
+    await saveRecentMessages(
+      {
+        ...recent,
+        queued,
+        sent,
+        cursors: recent.scanned && queued.length === 0 ? {} : recent.cursors
+      },
+      paths
+    );
+  return { sent, complete: recent.scanned && queued.length === 0 };
+}
+function checkedRecentLimit(config22, limit2) {
+  if (limit2 === void 0) return void 0;
+  if (!Number.isSafeInteger(limit2) || limit2 < 1)
+    throw new Error("The recent message limit must be a positive integer.");
+  if (config22.destination === "vault")
+    throw new Error("Vault collection does not send recent messages first.");
+  return limit2;
 }
 function isConsumedHook(source, filename, policy) {
   if (policy !== void 0) return false;
@@ -15379,12 +15594,12 @@ function retryAfterMs(response) {
   const milliseconds = /^\d+$/.test(header) ? Number(header) * 1e3 : Date.parse(header) - Date.now();
   return Number.isFinite(milliseconds) ? Math.max(1e3, Math.min(milliseconds, 3e5)) : 5e3;
 }
-async function uploadPendingOnce(config22, paths, fetcher = collectorFetch, signal, onAcknowledged, policy) {
+async function uploadPendingOnce(config22, paths, fetcher = collectorFetch, signal, onAcknowledged, policy, first) {
   let uploaded = 0;
   for (let attempt = 0; attempt < MAX_UPLOADS_PER_TICK; attempt += 1) {
     signal?.throwIfAborted();
     await recheckQueuedAccounts(config22, paths, policy);
-    const batch = await nextBatch(paths, MAX_UPLOAD_BYTES);
+    const batch = await nextBatch(paths, MAX_UPLOAD_BYTES, first);
     if (batch.length === 0) return { uploaded, retryAfterMs: null };
     const body = JSON.stringify({ events: batch });
     if (Buffer.byteLength(body, "utf8") > MAX_REQUEST_BYTES) {
@@ -15453,7 +15668,8 @@ async function recheckQueuedAccounts(config22, paths, policy) {
     (event3) => reauthorizeVaultEvent(policy, event3, accounts)
   );
 }
-async function collectorTick(config22, paths, fetcher = collectorFetch, signal, onAcknowledged) {
+async function collectorTick(config22, paths, fetcher = collectorFetch, signal, onAcknowledged, options = {}) {
+  const recentLimit = checkedRecentLimit(config22, options.recentMessages);
   const accounts = await vaultAccountIndex(config22);
   const policy = config22.destination === "vault" ? await fetchVaultPolicy(config22.endpoint, config22.token, fetcher, signal) : void 0;
   if (policy !== void 0) {
@@ -15478,15 +15694,22 @@ async function collectorTick(config22, paths, fetcher = collectorFetch, signal,
     }
   }
   await redactPendingOutbox(paths);
+  const first = await queueRecentMessages(config22, paths, recentLimit, signal);
   let upload = await uploadPendingOnce(
     config22,
     paths,
     fetcher,
     signal,
     onAcknowledged,
-    policy
+    policy,
+    first
   );
-  const scan = await outboxSize(paths) >= SCAN_PAUSE_OUTBOX_BYTES ? { discovered: 0, excludedRecords: 0, caughtUp: false, history: null } : await scanSourceOnce(config22, paths, signal, policy, accounts);
+  const scan = await outboxSize(paths) >= SCAN_PAUSE_OUTBOX_BYTES ? {
+    discovered: 0,
+    excludedRecords: 0,
+    caughtUp: false,
+    history: await savedHistoryProgress(config22, paths)
+  } : await scanSourceOnce(config22, paths, signal, policy, accounts);
   if (policy !== void 0 && scan.caughtUp) {
     const state = await loadState(paths);
     if (state.captureRecheckAt === void 0)
@@ -15502,7 +15725,8 @@ async function collectorTick(config22, paths, fetcher = collectorFetch, signal,
       fetcher,
       signal,
       onAcknowledged,
-      policy
+      policy,
+      first
     );
     upload = { ...next, uploaded: upload.uploaded + next.uploaded };
   }
@@ -15513,6 +15737,7 @@ async function collectorTick(config22, paths, fetcher = collectorFetch, signal,
     outboxBytes: await outboxSize(paths),
     historyCaughtUp: scan.caughtUp,
     history: scan.history,
+    recentMessages: await settleRecentMessages(paths, recentLimit),
     retryAfterMs: upload.retryAfterMs ?? 0
   };
 }
@@ -22385,11 +22610,16 @@ var collectorFrictionSummarySchema = object3({
   weekEpisodes: episodeCountSchema,
   dayEpisodes: episodeCountSchema
 }).strict().refine((summary) => summary.dayEpisodes <= summary.weekEpisodes, "The last day cannot have more friction episodes than the last week.").describe("dayEpisodes must not exceed weekEpisodes.");
+var RECENT_MESSAGE_LIMIT = 400;
 var sessionCountSchema = number4().int().min(0).max(1e6);
 var collectorHistoryProgressSchema = object3({
   sessions: sessionCountSchema,
   readSessions: sessionCountSchema,
-  complete: boolean4()
+  complete: boolean4(),
+  recentMessages: object3({
+    sent: number4().int().min(0).max(RECENT_MESSAGE_LIMIT),
+    complete: boolean4()
+  }).strict().optional().describe("The newest messages sent ahead of the rest of the history; collectors before 0.17.0 omit it.")
 }).strict().refine((progress) => progress.readSessions <= progress.sessions, "Read sessions cannot exceed listed sessions.").refine((progress) => !progress.complete || progress.readSessions === progress.sessions, "A complete import has read every listed session.").describe("readSessions must not exceed sessions; complete requires equal counts.");
 var collectorEnrollmentRequestSchema = object3({
   deviceName: string5().trim().min(1).max(160).describe("Trimmed before length validation."),
@@ -25582,6 +25812,7 @@ async function clearConnection(directory) {
       paths.redactionMarker,
       paths.historyBackfill,
       paths.health,
+      paths.recentMessages,
       ...[
         "cursor-hooks",
         "cursor-hook-failure.json",
@@ -26498,7 +26729,13 @@ var HistoryProgressError = class extends Error {
   }
 };
 function unchanged(left, right) {
-  return right !== null && left.sessions === right.sessions && left.readSessions === right.readSessions && left.complete === right.complete;
+  return right !== null && left.sessions === right.sessions && left.readSessions === right.readSessions && left.complete === right.complete && left.recentMessages?.sent === right.recentMessages?.sent && left.recentMessages?.complete === right.recentMessages?.complete;
+}
+function settled(progress) {
+  return progress.complete && (progress.recentMessages?.complete ?? true);
+}
+function milestone(progress, sent) {
+  return progress.complete || progress.recentMessages?.complete === true && sent?.recentMessages?.complete !== true;
 }
 var HistoryProgressReporter = class {
   #sent = null;
@@ -26506,7 +26743,7 @@ var HistoryProgressReporter = class {
   #completed = false;
   async report(config3, progress, fetcher, signal, nowMs) {
     if (this.#completed || unchanged(progress, this.#sent)) return;
-    if (!progress.complete && nowMs - this.#sentAtMs < REPORT_INTERVAL_MS)
+    if (!milestone(progress, this.#sent) && nowMs - this.#sentAtMs < REPORT_INTERVAL_MS)
       return;
     let response;
     try {
@@ -26539,7 +26776,7 @@ var HistoryProgressReporter = class {
     }
     this.#sent = progress;
     this.#sentAtMs = nowMs;
-    this.#completed = progress.complete;
+    this.#completed = settled(progress);
   }
 };
 
@@ -26648,7 +26885,7 @@ async function runCollector(directory = defaultCollectorDirectory("codex"), expe
     const summariesRefreshedAt = /* @__PURE__ */ new Map();
     let lastAutoCheckAt = 0;
     let lastBillingSyncAt = 0;
-    const historyProgress = new HistoryProgressReporter();
+    const historyProgress2 = new HistoryProgressReporter();
     const runtimeVersion = process.env.ITERATION_RUNTIME_VERSION;
     let lastRuntimeCheckAt = 0;
     let trialSettled = runtimeVersion === void 0;
@@ -26742,15 +26979,15 @@ async function runCollector(directory = defaultCollectorDirectory("codex"), expe
           controller.signal,
           () => {
             acknowledgedAtMs = Date.now();
-          }
+          },
+          { recentMessages: RECENT_MESSAGE_LIMIT }
         );
         billingReady = result.historyCaughtUp && result.outboxBytes === 0;
-        if (result.history !== null) {
-          history = {
-            ...result.history,
-            complete: billingReady && result.history.readSessions === result.history.sessions
-          };
-        }
+        history = {
+          ...result.history,
+          complete: billingReady && result.history.readSessions === result.history.sessions,
+          ...result.recentMessages === null ? {} : { recentMessages: result.recentMessages }
+        };
         await recordTickSuccess(result, acknowledgedAtMs);
       } catch (error3) {
         if (controller.signal.aborted) return null;
@@ -26781,7 +27018,7 @@ async function runCollector(directory = defaultCollectorDirectory("codex"), expe
     const reportHistory = async (history) => {
       if (history !== null) {
         try {
-          await historyProgress.report(
+          await historyProgress2.report(
             config3,
             history,
             collectorFetch,

package/dist/cursor-hook-entry.js

@@ -11541,6 +11541,17 @@ var healthSchema = object({
   }).strict().nullable()
 }).strict();
 var redactionMarkerSchema = object({ version: literal(1) }).strict();
+var recentMessagesSchema = object({
+  version: literal(1),
+  cursors: record(
+    string2(),
+    number2().int().min(0).max(Number.MAX_SAFE_INTEGER)
+  ),
+  // Queued by the pass and not yet acknowledged.
+  queued: array(string2()),
+  sent: number2().int().min(0),
+  scanned: boolean2()
+}).strict();
 var HISTORY_BACKFILL_VERSION = 4;
 var historyBackfillMarkerSchema = object({ version: number2().int().min(1).max(HISTORY_BACKFILL_VERSION) }).strict();
 var repositoryUrls = /* @__PURE__ */ new Map();

package/dist/launcher.js

@@ -11666,6 +11666,17 @@ var healthSchema = object({
   }).strict().nullable()
 }).strict();
 var redactionMarkerSchema = object({ version: literal(1) }).strict();
+var recentMessagesSchema = object({
+  version: literal(1),
+  cursors: record(
+    string2(),
+    number2().int().min(0).max(Number.MAX_SAFE_INTEGER)
+  ),
+  // Queued by the pass and not yet acknowledged.
+  queued: array(string2()),
+  sent: number2().int().min(0),
+  scanned: boolean2()
+}).strict();
 var HISTORY_BACKFILL_VERSION = 4;
 var historyBackfillMarkerSchema = object({ version: number2().int().min(1).max(HISTORY_BACKFILL_VERSION) }).strict();
 var scopeRecordSchema = object({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "iteration-cli",
-  "version": "0.16.1",
+  "version": "0.17.1",
   "type": "module",
   "repository": {
     "type": "git",