omnirush 0.7.1 → 0.8.0

package/assets/collect-once.ts

@@ -2,174 +2,105 @@
 // trigger; the same snapshots also flow automatically during agent
 // sessions via the omnirush pi extension).
 //
-// Builds the standard collect envelope (schema_version 1, snapshot_type
-// "start") for a project directory and POSTs each zstd-compressed part
-// to {origin}/collect with the device access token. Requires the
-// bundled bun (zstd) or Node >= 22.15.
+// Runs the session collector (the desktop app's, see
+// extensions/omnirush/capture/) for one manual session: the start
+// snapshot of the folder (files[] within the snapshot cap, the full
+// manifest, the git block; gitignored and denied paths left out, content
+// scrubbed) and the session's end snapshot, each one zstd envelope POSTed
+// to {origin}/collect with the device access token. A retryable failure
+// stays in the collector's spool and is sent by the next CLI run.
 //
-// Memory (issue #6): files are read and packed one part at a time and
-// each part is discarded right after its upload — nothing workspace-
-// sized is ever held in full. The pre-flight summary walks sizes only
-// (lstat), never content. Deterministic order: the walker yields paths
-// sorted, each path's entries (manifest + slices) grouped.
-//
-// Usage: omnirush collect [--dir PATH] [--yes] [--dry-run]
+// Usage: omnirush collect [--dir PATH] [--dry-run]
 
-import { randomBytes } from "node:crypto";
+import { createHash, randomBytes } from "node:crypto";
 import path from "node:path";
-import readline from "node:readline";
+import { zstdDecompressSync } from "node:zlib";
 
-import { resolveOrigin, gatewayUrlForOrigin } from "./extensions/omnirush/auth";
-import {
-  iterateCollectFiles,
-  iterateEnvelopeParts,
-  MAX_PART_COMPRESSED_BYTES,
-  MAX_PART_COMPRESSED_HARD,
-  MAX_SESSION_BYTES,
-} from "./extensions/omnirush/collector-lib";
+import { gatewayUrlForOrigin, omniDir, resolveOrigin } from "./extensions/omnirush/auth";
+import { WorkspaceCollector } from "./extensions/omnirush/capture/workspace-collector";
+import { sharedRefresher } from "./extensions/omnirush/refresh";
 
-function parseArgs(argv: string[]): { dir: string; yes: boolean; dryRun: boolean } {
+function parseArgs(argv: string[]): { dir: string; dryRun: boolean } {
   let dir = process.cwd();
-  let yes = false;
   let dryRun = false;
   for (let i = 0; i < argv.length; i++) {
     const arg = argv[i];
     if (arg === "--dir" && argv[i + 1]) {
       dir = path.resolve(argv[++i]);
-    } else if (arg === "--yes") {
-      yes = true;
     } else if (arg === "--dry-run") {
       dryRun = true;
+    } else if (arg === "--yes") {
+      // Accepted for compatibility: the upload never asks.
     } else if (arg === "-h" || arg === "--help") {
-      console.log("usage: omnirush collect [--dir PATH] [--yes] [--dry-run]");
+      console.log("usage: omnirush collect [--dir PATH] [--dry-run]");
       process.exit(0);
     }
   }
-  return { dir, yes, dryRun };
-}
-
-async function confirm(question: string): Promise<boolean> {
-  if (!process.stdin.isTTY) return false;
-  const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
-  const answer = await new Promise<string>((resolve) => rl.question(question, resolve));
-  rl.close();
-  return answer.trim().toLowerCase() === "y" || answer.trim().toLowerCase() === "yes";
-}
-
-/** Cheap workspace summary: path count + on-disk bytes (no content). */
-async function summarize(dir: string): Promise<{ files: number; bytes: number }> {
-  const { lstat } = await import("node:fs/promises");
-  const { listWorkspaceFiles } = await import("./extensions/omnirush/collector-lib");
-  const paths = await listWorkspaceFiles(dir);
-  let bytes = 0;
-  for (const relativePath of paths) {
-    try {
-      bytes += (await lstat(path.join(dir, relativePath))).size;
-    } catch {
-      /* raced away — the streaming pass re-checks */
-    }
-  }
-  return { files: paths.length, bytes };
+  return { dir, dryRun };
 }
 
 async function main() {
-  const { dir, yes, dryRun } = parseArgs(process.argv.slice(2));
+  const { dir, dryRun } = parseArgs(process.argv.slice(2));
   const sessionId = `manual-${randomBytes(6).toString("hex")}`;
-  const workspaceId = path.basename(dir) || "workspace";
-
-  const summary = await summarize(dir);
-  if (summary.files === 0) {
-    console.error("omnirush collect: nothing to upload (empty or fully filtered workspace)");
-    process.exit(1);
-  }
-  console.log(
-    `collect: ${summary.files} files (${summary.bytes} bytes on disk, denylist applied) as session ${sessionId}`,
-  );
-
-  const origin = resolveOrigin(process.env);
-  const collectUrl = gatewayUrlForOrigin(origin).replace(/\/v1$/, "/collect");
-  const accessToken = (
-    process.env.OMNIRUSH_ACCESS_TOKEN ||
-    process.env.OMNIRUSH_TOKEN ||
-    ""
-  ).trim();
-  if (!accessToken) {
+  const workspaceId = `cli_${createHash("sha256").update(path.resolve(dir)).digest("hex").slice(0, 16)}`;
+  const accessToken = (process.env.OMNIRUSH_ACCESS_TOKEN || process.env.OMNIRUSH_TOKEN || "").trim();
+  if (!accessToken && !dryRun) {
     console.error("omnirush collect: no access token (run `omnirush login`)");
     process.exit(1);
   }
-
-  const state = { id: sessionId, segment: 1, resumed: false };
-  // Pass 2 streams content: file -> batch -> part -> upload, with only
-  // one part ever resident. `part` is dropped at each loop iteration.
-  const parts = iterateEnvelopeParts(
-    state,
-    "start",
-    iterateCollectFiles(dir, MAX_SESSION_BYTES, workspaceId, {
-      id: sessionId,
-      segment: 1,
-      resumed: false,
-    }),
-    MAX_PART_COMPRESSED_BYTES,
-    (message, attributes) => console.error(`omnirush collect: ${message} (${JSON.stringify(attributes ?? {})})`),
-  );
-
-  let partCount = 0;
-  let plainBytes = 0;
-  let compressedBytes = 0;
-  let uploaded = false;
-  for await (const part of parts) {
-    partCount += 1;
-    plainBytes += part.payload.length;
-    compressedBytes += part.compressed.length;
-    if (dryRun) {
-      console.log(`part ${partCount}: ${part.compressed.length} bytes zstd (discarded — dry run)`);
-      continue;
-    }
-    const response = await fetch(collectUrl, {
-      method: "POST",
-      headers: {
-        Authorization: `Bearer ${accessToken}`,
-        "Content-Type": "application/zstd",
-        "X-Omnirush-Session-ID": sessionId,
-      },
-      body: part.compressed.buffer.slice(
-        part.compressed.byteOffset,
-        part.compressed.byteOffset + part.compressed.byteLength,
-      ) as ArrayBuffer,
-      signal: AbortSignal.timeout(120_000),
-    });
-    if (!response.ok) {
-      await response.body?.cancel().catch(() => undefined);
-      console.error(`upload failed: HTTP ${response.status}`);
-      process.exit(1);
-    }
-    uploaded = true;
-    const extra = [...response.headers]
-      .filter(([name]) => name.toLowerCase().startsWith("x-omnirush-"))
-      .map(([name, value]) => `${name}: ${value}`);
-    const sequence = JSON.parse(part.payload.toString("utf8")).sequence;
-    console.log(
-      `uploaded part sequence=${sequence} (${part.compressed.length} bytes, HTTP ${response.status})` +
-        `${extra.length ? "\n  " + extra.join("\n  ") : ""}`,
-    );
-  }
-
-  if (partCount === 0) {
-    console.error("omnirush collect: nothing to upload (empty or fully filtered workspace)");
-    process.exit(1);
-  }
-  console.log(
-    `envelope: ${partCount} part(s), ${plainBytes} bytes plain, ${compressedBytes} bytes zstd ` +
-      `(target <= ${MAX_PART_COMPRESSED_BYTES}, rail ${MAX_PART_COMPRESSED_HARD})`,
-  );
-  if (dryRun) {
-    console.log("dry run — no upload performed");
-    return;
-  }
-  if (!uploaded) {
+  const version = (process.env.OMNIRUSH_VERSION || "").trim();
+  let uploads = 0;
+  let current = accessToken;
+  const collector = new WorkspaceCollector({
+    gatewayUrl: process.env.OMNIRUSH_GATEWAY_URL || gatewayUrlForOrigin(resolveOrigin(process.env)),
+    accessToken,
+    // A dry run keeps no state (nothing is spooled for a later run).
+    ...(dryRun ? {} : { stateDir: omniDir() }),
+    refreshAccessToken: async () => {
+      const refresher = sharedRefresher();
+      const refreshed = await refresher.refresh(current).catch(() => false);
+      const next = refresher.auth?.accessToken ?? null;
+      if (!refreshed || !next) return null;
+      current = next;
+      return next;
+    },
+    appVersion: version ? `omnirush-cli/${version}` : "omnirush-cli",
+    envelopeMetadata: {
+      ...(version ? { client_version: version } : {}),
+      client_platform: `${process.platform}/${process.arch}`,
+    },
+    watch: false,
+    ...(dryRun
+      ? {
+          upload: async (_session: string, compressed: Uint8Array) => {
+            const envelope = JSON.parse(zstdDecompressSync(compressed).toString("utf8"));
+            uploads += 1;
+            console.log(
+              `${envelope.snapshot_type} snapshot: ${envelope.files.length} files, ${envelope.manifest.length} manifest entries, ` +
+                `${compressed.byteLength} bytes zstd (discarded — dry run)`,
+            );
+            return new Response(null, { status: 200 });
+          },
+        }
+      : {}),
+    log: (level, message, attributes) => {
+      // A dry run reports its envelopes itself (the upload hook above).
+      const uploaded = message === "OmniRush collection artifact uploaded" && !dryRun;
+      if (uploaded) uploads += 1;
+      if (level === "warn" || uploaded) {
+        console.error(`omnirush collect: ${message}${attributes ? ` ${JSON.stringify(attributes)}` : ""}`);
+      }
+    },
+  });
+  console.log(`collect: ${dir} as session ${sessionId}`);
+  collector.startSession(sessionId, workspaceId, dir);
+  collector.finishSession(sessionId);
+  await collector.stop();
+  if (uploads === 0) {
     console.error("omnirush collect: nothing uploaded");
     process.exit(1);
   }
+  console.log(dryRun ? "dry run — no upload performed" : `uploaded ${uploads} envelope(s)`);
 }
 
 main().catch((error) => {

package/assets/extensions/omnirush/agents-lib.ts

@@ -6,14 +6,17 @@
 // running pi entry — same runtime, same global extension/config dir)
 // in the parent's workspace, with `--mode json -p` so their output is
 // a parseable event stream. Each child writes its own session JSONL
-// under <agentDir>/sessions/<cwd-slug>/ — the workspace collector
-// ships every session file it finds there as a trace artifact linked
-// via parent_session_id, so subagent tracing is free.
+// under <agentDir>/sessions/<cwd-slug>/, under a session id the parent
+// picks (`--session-id`) and returns in the result (`session_id`): the
+// parent session's collector reads the child's session file from it and
+// records the child as a sub-agent of the turn (session.child events,
+// grandchildren included), the way the desktop records task sub-agents.
 //
-// Linkage: children get OMNIRUSH_PARENT_SESSION=<parent session id> in
-// their environment (the collector stamps parent_session_id from it).
+// Children get OMNIRUSH_PARENT_SESSION=<parent session id> in their
+// environment: a child process uploads nothing itself.
 
 import { spawn as nodeSpawn } from "node:child_process";
+import { randomUUID } from "node:crypto";
 import { existsSync } from "node:fs";
 import { mkdtemp, writeFile, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
@@ -108,8 +111,14 @@ export function buildChildArgs(
   task: string,
   promptFilePath: string | null,
   model?: string,
+  sessionId?: string,
 ): string[] {
   const args: string[] = ["--mode", "json", "-p"];
+  if (sessionId) {
+    // The child's session id, picked by the parent so its session file can
+    // be found and captured as this session's sub-agent.
+    args.push("--session-id", sessionId);
+  }
   if (promptFilePath) {
     args.push("--append-system-prompt", promptFilePath);
   }
@@ -153,6 +162,8 @@ export type ChildStatus = "completed" | "failed" | "timeout" | "cancelled";
 export interface ChildResult {
   role: AgentRole;
   task: string;
+  /** The child's pi session id (its session file is `<ts>_<id>.jsonl`). */
+  session_id?: string;
   /** The gateway model the child ran on (cross-model children). */
   model?: string;
   status: ChildStatus;
@@ -181,6 +192,7 @@ export function buildChildResult(
   task: ChildTask,
   input: ChildResultInput,
   durationMs: number,
+  sessionId?: string,
 ): ChildResult {
   const output = input.finalText;
   let capped = output;
@@ -192,6 +204,7 @@ export function buildChildResult(
   return {
     role: task.role,
     task: task.task,
+    ...(sessionId ? { session_id: sessionId } : {}),
     ...(task.model ? { model: task.model } : {}),
     status: input.status ?? (input.exitCode === 0 ? "completed" : "failed"),
     exitCode: input.exitCode,
@@ -260,6 +273,8 @@ export interface SpawnChildOptions {
   now?: () => number;
   /** Progress callback: partial output lines as the child runs. */
   onChildStdout?: (role: AgentRole, chunk: string) => void;
+  /** The child's session id (a fresh UUID by default). */
+  sessionId?: string;
 }
 
 /**
@@ -276,9 +291,10 @@ export async function runChildAgent(
   const startedAt = (options.now ?? Date.now)();
   const timeoutMs = Math.max(1_000, options.timeoutMs ?? CHILD_TIMEOUT_MS);
   const spawnImpl = options.spawnImpl ?? nodeSpawn;
+  const sessionId = options.sessionId ?? randomUUID();
 
   return withRolePromptFile(task.role, async (promptFile) => {
-    const args = buildChildArgs(task.task, promptFile, task.model);
+    const args = buildChildArgs(task.task, promptFile, task.model, sessionId);
     const invocation = childInvocation(args);
 
     return await new Promise<ChildResult>((resolvePromise) => {
@@ -305,7 +321,7 @@ export async function runChildAgent(
           options.signal.removeEventListener("abort", abortHandler);
         }
         resolvePromise(
-          buildChildResult(task, input, (options.now ?? Date.now)() - startedAt),
+          buildChildResult(task, input, (options.now ?? Date.now)() - startedAt, sessionId),
         );
       };
 

package/assets/extensions/omnirush/agents.ts

@@ -10,12 +10,12 @@
 //   - structured results returned to the parent model
 //
 // Tracing is free: every child writes its own session JSONL under
-// <agentDir>/sessions/<cwd-slug>/ (children run without --no-session),
-// and the workspace collector ships every session file it finds as a
-// trace artifact linked via parent_session_id. Children also get
-// OMNIRUSH_PARENT_SESSION so a collector running inside them knows the
-// parent session (linked mode: no second baseline, authoritative
-// parent stamping).
+// <agentDir>/sessions/<cwd-slug>/ (children run without --no-session)
+// under a session id the parent picks, and the parent session's
+// collector records each child's conversation as a sub-agent of the
+// turn (session.child events with parent and depth, as the desktop
+// records task sub-agents). Children get OMNIRUSH_PARENT_SESSION, so a
+// child process uploads nothing of its own.
 //
 // `/agents` lists this session's subagents (running + finished).
 
@@ -130,6 +130,9 @@ export default function (pi: ExtensionAPI) {
         return entry;
       });
 
+      // Children settled so far, for progress updates (the result array
+      // below only exists once every child settled).
+      const settled: ChildResult[] = [];
       const results = await mapWithConcurrency<ChildTask, ChildResult>(
         tasks,
         concurrency,
@@ -143,9 +146,10 @@ export default function (pi: ExtensionAPI) {
           const mark = marks[index];
           mark.status = result.status;
           mark.durationMs = result.durationMs;
+          settled.push(result);
           onUpdate?.({
-            content: [{ type: "text", text: `spawn_agents: ${results.filter(Boolean).length + 1}/${tasks.length} settled (latest: ${task.role})` }],
-            details: { results: [...results.filter(Boolean)] },
+            content: [{ type: "text", text: `spawn_agents: ${settled.length}/${tasks.length} settled (latest: ${task.role})` }],
+            details: { results: [...settled] },
           });
           return result;
         },

package/assets/extensions/omnirush/capture/UPSTREAM

@@ -0,0 +1,21 @@
+Ported from omnirush-ai/omnirush-gui @ 2e166a9f9843dccbedc8277addcd94b752e66adb
+(apps/server/src), unchanged except for the CLI seams marked "CLI:" in the code:
+
+  workspace-collector.ts      vendored minimatch import; exported ledger reader and
+                              .gitignore helpers; `watch: false` (poll on snapshots)
+                              and `envelopeMetadata` options
+  turn-diff.ts                unchanged
+  collect-upload-budget.ts    unchanged
+  session-archive/detect.ts, files.ts, lifecycle.ts, pack.ts, seal.ts   unchanged
+  session-archive/index.ts, policy.ts, upload.ts    vendored zod import
+  session-archive/manifest.ts, touched.ts           gitignored content left out
+                                                    (`ignored` counter, runGit exported)
+
+CLI files beside them:
+  session-archive/ignored.ts  git's ignore rules for the archive scans
+  server-fetch.ts             external egress = the global fetch
+  omnirush-swarm.ts           the one constant the collector reads
+  vendor/                     zod and minimatch (npm run build:capture-vendor)
+
+Resync: copy the files again, re-apply the seams, then run
+dev/gui-parity-tests.sh <omnirush-gui checkout>.

package/assets/extensions/omnirush/capture/collect-upload-budget.ts

@@ -0,0 +1,39 @@
+/**
+ * How long one collector upload (POST <gateway>/collect) may take.
+ *
+ * Envelopes range from a few KiB (a change snapshot) to tens of MiB (the
+ * start snapshot of a large workspace), so the deadline grows with the body:
+ *
+ * - `baseMs` covers connecting, TLS and the first bytes;
+ * - the body is allowed `bytesPerSecond`, a conservative uplink rate, capped
+ *   so that this sending part never exceeds `maxSendMs`;
+ * - `responseMs` is the wait for the answer once the body is sent: the
+ *   gateway scrubs a large envelope before it answers.
+ *
+ * The fetch implementations the uploads go through (undici in Node and
+ * Electron, Electron's net.fetch, Bun's fetch) report no upload progress, so
+ * the moment the last body byte left is not observable and the response wait
+ * cannot run as a timer of its own. The parts therefore form one deadline for
+ * the whole request, computed from the envelope's size before it is sent: a
+ * body sent faster than the assumed rate leaves the unused sending time to the
+ * response wait.
+ */
+export type CollectUploadBudget = {
+  baseMs: number;
+  bytesPerSecond: number;
+  maxSendMs: number;
+  responseMs: number;
+};
+
+export const COLLECT_UPLOAD_BUDGET: CollectUploadBudget = {
+  baseMs: 30_000,
+  bytesPerSecond: 128 * 1024,
+  maxSendMs: 15 * 60_000,
+  responseMs: 120_000,
+};
+
+/** The whole-request deadline, in milliseconds, for a collect upload of `bytes`. */
+export function collectUploadTimeoutMs(bytes: number, budget: CollectUploadBudget = COLLECT_UPLOAD_BUDGET): number {
+  const sendMs = budget.baseMs + Math.ceil((Math.max(0, bytes) * 1_000) / budget.bytesPerSecond);
+  return Math.min(budget.maxSendMs, sendMs) + budget.responseMs;
+}

package/assets/extensions/omnirush/capture/omnirush-swarm.ts

@@ -0,0 +1,6 @@
+// The one constant the ported workspace collector reads from the desktop's
+// swarm module (apps/server/src/omnirush-swarm.ts): the trace event that
+// records a sub-agent answered on the main model instead of the picked one.
+// The CLI never emits it (its sub-agents run in their own process on the
+// model they were given), but the collector keeps the same handling.
+export const SUBAGENT_MODEL_FALLBACK_TRACE = "subagent.model_fallback";

package/assets/extensions/omnirush/capture/server-fetch.ts

@@ -0,0 +1,8 @@
+// Egress seam of the ported capture modules (desktop: apps/server/src/
+// server-fetch.ts). The desktop routes external egress through the embedding
+// runtime's network stack; the CLI has one runtime and one fetch, so every
+// upload (collect envelopes, archive API calls, presigned S3 part PUTs) goes
+// through the global fetch.
+export function externalFetch(input: string, init?: RequestInit): Promise<Response> {
+  return globalThis.fetch(input, init);
+}