impel-cli 0.16.4 → 0.17.0

package/src/sessionCollector.js

@@ -0,0 +1,996 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { spawnSync } from "node:child_process";
+import * as zlib from "node:zlib";
+
+import { CONFIG_DIR, loadConfig, redactCredentialText, redactSecretText } from "./config.js";
+
+export const DEFAULT_SESSIONS_URL = "https://sessions.useimpel.com";
+const MAX_HOOK_INPUT_BYTES = 2 * 1024 * 1024;
+const MAX_LEDGER_PAYLOAD_BYTES = 256 * 1024;
+const TRANSCRIPT_CHUNK_BYTES = 512 * 1024;
+const MAX_TRANSCRIPT_CAPTURE_BYTES = 4 * 1024 * 1024;
+const MAX_FLUSH_BATCHES = 8;
+const DEFAULT_MAX_OUTBOX_BYTES = 256 * 1024 * 1024;
+const DEFAULT_MAX_OUTBOX_BATCHES = 2048;
+const MAX_QUARANTINED_BATCHES = 32;
+const MAX_BATCH_FAILURES = 3;
+const REQUEST_TIMEOUT_MS = 30_000;
+const DIRECT_UPLOAD_TIMEOUT_MS = 30_000;
+const SAFE_ID_RE = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/u;
+const TASK_ID_RE = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u;
+
+function stateRoot() {
+  return process.env.IMPEL_SESSIONS_STATE_DIR || path.join(CONFIG_DIR, "sessions");
+}
+
+function sha256(value) {
+  return crypto.createHash("sha256").update(value).digest("hex");
+}
+
+function stableUuid(namespace) {
+  const bytes = crypto.createHash("sha256").update(namespace, "utf8").digest().subarray(0, 16);
+  bytes[6] = (bytes[6] & 0x0f) | 0x50;
+  bytes[8] = (bytes[8] & 0x3f) | 0x80;
+  const hex = bytes.toString("hex");
+  return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
+}
+
+function privateDirectory(directory) {
+  fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
+  try {
+    fs.chmodSync(directory, 0o700);
+  } catch {
+    // Best effort on platforms where chmod is unavailable.
+  }
+}
+
+function atomicWrite(filePath, contents) {
+  privateDirectory(path.dirname(filePath));
+  const temporaryPath = `${filePath}.tmp-${process.pid}-${crypto.randomUUID()}`;
+  try {
+    fs.writeFileSync(temporaryPath, contents, { mode: 0o600 });
+    fs.renameSync(temporaryPath, filePath);
+  } finally {
+    try {
+      fs.rmSync(temporaryPath, { force: true });
+    } catch {
+      // The successful rename already removed the temporary file.
+    }
+  }
+}
+
+function readJson(filePath, fallback = null) {
+  try {
+    return JSON.parse(fs.readFileSync(filePath, "utf8"));
+  } catch {
+    return fallback;
+  }
+}
+
+function positiveIntegerEnvironment(name, fallback) {
+  const raw = process.env[name];
+  if (raw === undefined || raw === "") return fallback;
+  const value = Number(raw);
+  return Number.isSafeInteger(value) && value > 0 ? value : fallback;
+}
+
+function safeSegment(value, fallback) {
+  const normalized = String(value || "")
+    .normalize("NFC")
+    .replace(/[^A-Za-z0-9._:-]+/gu, "-")
+    .replace(/^-+|-+$/gu, "")
+    .slice(0, 96);
+  return normalized || fallback;
+}
+
+function sessionDirectory(tenantId, provider, sessionKey) {
+  return path.join(
+    stateRoot(),
+    "sessions",
+    safeSegment(tenantId, "tenant"),
+    safeSegment(provider, "unknown-provider"),
+    sha256(sessionKey).slice(0, 32),
+  );
+}
+
+function writeError(root, error) {
+  const message = redactSecretText(error?.message || error).slice(0, 2048);
+  atomicWrite(path.join(root, "last-error.json"), `${JSON.stringify({ at: new Date().toISOString(), message }, null, 2)}\n`);
+}
+
+function clearError(root) {
+  try {
+    fs.rmSync(path.join(root, "last-error.json"), { force: true });
+  } catch {
+    // Best effort only.
+  }
+}
+
+function lockOwnerAlive(owner) {
+  if (!Number.isInteger(owner?.pid) || owner.pid < 1) return false;
+  try {
+    process.kill(owner.pid, 0);
+    return true;
+  } catch (error) {
+    return error?.code === "EPERM";
+  }
+}
+
+function acquireLock(root, name, staleMs = 60_000) {
+  const lock = path.join(root, `${name}.lock`);
+  const takeover = `${lock}.takeover`;
+  const token = crypto.randomUUID();
+  privateDirectory(root);
+  const takeoverOwner = readJson(takeover);
+  if (takeoverOwner || fs.existsSync(takeover)) {
+    let age = 0;
+    try { age = Date.now() - fs.statSync(takeover).mtimeMs; } catch { /* Re-check through the create below. */ }
+    if ((takeoverOwner && lockOwnerAlive(takeoverOwner)) || age <= staleMs) return null;
+    try { fs.rmSync(takeover, { force: true }); } catch { return null; }
+  }
+  try {
+    fs.mkdirSync(lock, { mode: 0o700 });
+    if (fs.existsSync(takeover)) {
+      fs.rmSync(lock, { recursive: true, force: true });
+      return null;
+    }
+    atomicWrite(path.join(lock, "owner.json"), `${JSON.stringify({ token, pid: process.pid, at: Date.now() })}\n`);
+    return () => {
+      const current = readJson(path.join(lock, "owner.json"));
+      if (current?.token === token) fs.rmSync(lock, { recursive: true, force: true });
+    };
+  } catch (error) {
+    if (error?.code !== "EEXIST") throw error;
+  }
+  const owner = readJson(path.join(lock, "owner.json"));
+  if (lockOwnerAlive(owner)) return null;
+  let lockAge = 0;
+  try {
+    lockAge = Date.now() - fs.statSync(lock).mtimeMs;
+  } catch (error) {
+    if (error?.code === "ENOENT") return acquireLock(root, name, staleMs);
+    throw error;
+  }
+  const age = owner?.at ? Date.now() - owner.at : lockAge;
+  if (age <= staleMs) return null;
+
+  let takeoverDescriptor;
+  try {
+    takeoverDescriptor = fs.openSync(takeover, "wx", 0o600);
+    fs.writeFileSync(takeoverDescriptor, `${JSON.stringify({ token, pid: process.pid, at: Date.now() })}\n`);
+  } catch (error) {
+    if (takeoverDescriptor !== undefined) fs.closeSync(takeoverDescriptor);
+    if (error?.code === "EEXIST") return null;
+    throw error;
+  }
+  fs.closeSync(takeoverDescriptor);
+  try {
+    const currentOwner = readJson(path.join(lock, "owner.json"));
+    const unchanged = owner?.token ? currentOwner?.token === owner.token : !currentOwner?.token;
+    if (!unchanged || lockOwnerAlive(currentOwner)) return null;
+
+    // Move the stale candidate aside atomically while the takeover file keeps
+    // other acquirers from creating a successor at the original path.
+    const tombstone = `${lock}.stale-${process.pid}-${crypto.randomUUID()}`;
+    try {
+      fs.renameSync(lock, tombstone);
+    } catch (error) {
+      if (error?.code === "ENOENT") return null;
+      throw error;
+    }
+    const movedOwner = readJson(path.join(tombstone, "owner.json"));
+    const sameOwner = owner?.token ? movedOwner?.token === owner.token : !movedOwner?.token;
+    if (!sameOwner || lockOwnerAlive(movedOwner)) {
+      fs.renameSync(tombstone, lock);
+      return null;
+    }
+    fs.rmSync(tombstone, { recursive: true, force: true });
+  } finally {
+    fs.rmSync(takeover, { force: true });
+  }
+  return acquireLock(root, name, staleMs);
+}
+
+function providerRoots(provider) {
+  const roots = [process.env.IMPEL_SESSION_TRANSCRIPT_ROOT];
+  if (provider === "claude_code") roots.push(process.env.CLAUDE_CONFIG_DIR, path.join(os.homedir(), ".claude"));
+  else roots.push(process.env.CODEX_HOME, path.join(os.homedir(), ".codex"));
+  return roots.filter(Boolean).map((candidate) => {
+    const resolved = path.resolve(candidate);
+    try {
+      return fs.realpathSync(resolved);
+    } catch {
+      return resolved;
+    }
+  });
+}
+
+function allowedTranscript(filePath, provider) {
+  if (typeof filePath !== "string" || !path.isAbsolute(filePath)) return null;
+  let stat;
+  let real;
+  try {
+    const linkStat = fs.lstatSync(filePath);
+    if (linkStat.isSymbolicLink() || !linkStat.isFile()) return null;
+    real = fs.realpathSync(filePath);
+    stat = fs.statSync(real);
+  } catch {
+    return null;
+  }
+  const allowed = providerRoots(provider).some((root) => real === root || real.startsWith(`${root}${path.sep}`));
+  return allowed ? { real, stat } : null;
+}
+
+function compactLedgerPayload(input) {
+  const encoded = redactCredentialText(JSON.stringify(input));
+  const sanitized = JSON.parse(encoded);
+  if (Buffer.byteLength(encoded) <= MAX_LEDGER_PAYLOAD_BYTES) return { payload: sanitized, payloadTruncated: false };
+  const keys = [
+    "session_id", "hook_event_name", "cwd", "permission_mode", "model", "source", "reason",
+    "turn_id", "prompt_id", "agent_id", "agent_type", "tool_name", "tool_use_id", "error",
+    "error_details", "last_assistant_message",
+  ];
+  const payload = {};
+  for (const key of keys) {
+    if (!Object.hasOwn(sanitized, key)) continue;
+    const value = sanitized[key];
+    payload[key] = typeof value === "string" && value.length > 131_072
+      ? `${value.slice(0, 131_072)}\n[truncated]`
+      : value;
+  }
+  return { payload, payloadTruncated: true, originalSha256: sha256(encoded) };
+}
+
+function transcriptBatch(descriptor, offset, fileSize, discardUntilNewline = false) {
+  const pieces = [];
+  const digest = crypto.createHash("sha256");
+  let accumulated = 0;
+  let scanOffset = offset;
+  while (scanOffset < fileSize && accumulated < MAX_TRANSCRIPT_CAPTURE_BYTES) {
+    const wanted = Math.min(
+      TRANSCRIPT_CHUNK_BYTES,
+      fileSize - scanOffset,
+      MAX_TRANSCRIPT_CAPTURE_BYTES - accumulated,
+    );
+    const buffer = Buffer.allocUnsafe(wanted);
+    const bytesRead = fs.readSync(descriptor, buffer, 0, wanted, scanOffset);
+    if (bytesRead < 1) break;
+    const chunk = buffer.subarray(0, bytesRead);
+    const newline = pieces.length === 0 && !discardUntilNewline
+      ? chunk.lastIndexOf(0x0a)
+      : chunk.indexOf(0x0a);
+    const consumed = newline >= 0 ? chunk.subarray(0, newline + 1) : chunk;
+    scanOffset += consumed.length;
+    accumulated += consumed.length;
+
+    if (discardUntilNewline) {
+      digest.update(consumed);
+    } else {
+      pieces.push(consumed);
+    }
+
+    if (newline >= 0) {
+      if (discardUntilNewline) {
+        return {
+          payload: null,
+          bytesConsumed: scanOffset - offset,
+          itemCount: 0,
+          discardUntilNewline: false,
+        };
+      }
+      const raw = Buffer.concat(pieces);
+      const redacted = Buffer.from(redactCredentialText(raw.toString("utf8")), "utf8");
+      const itemCount = redacted.reduce((count, byte) => count + (byte === 0x0a ? 1 : 0), 0);
+      return { payload: redacted, bytesConsumed: scanOffset - offset, itemCount, discardUntilNewline: false };
+    }
+  }
+  if (accumulated >= MAX_TRANSCRIPT_CAPTURE_BYTES) {
+    if (discardUntilNewline) {
+      return {
+        payload: null,
+        bytesConsumed: scanOffset - offset,
+        itemCount: 0,
+        discardUntilNewline: true,
+      };
+    }
+    for (const piece of pieces) digest.update(piece);
+    const placeholder = Buffer.from(`${JSON.stringify({
+      type: "impel_oversized_transcript_record",
+      capturedPrefixBytes: accumulated,
+      capturedPrefixSha256: digest.digest("hex"),
+      omitted: true,
+    })}\n`, "utf8");
+    return {
+      payload: placeholder,
+      bytesConsumed: scanOffset - offset,
+      itemCount: 1,
+      discardUntilNewline: true,
+    };
+  }
+  return null;
+}
+
+function spoolBatch(root, sessionMeta, stream, payload, itemCount) {
+  if (!Buffer.isBuffer(payload) || payload.length === 0 || itemCount < 1) return null;
+  const batchId = `b-${Date.now().toString(36)}-${crypto.randomUUID()}`;
+  const pendingRoot = path.join(root, "pending");
+  privateDirectory(pendingRoot);
+  const temporary = fs.mkdtempSync(path.join(pendingRoot, ".tmp-"));
+  const finalDirectory = path.join(pendingRoot, batchId);
+  const manifest = {
+    schemaVersion: 1,
+    batchId,
+    createdAt: new Date().toISOString(),
+    expectedSequence: null,
+    itemCount,
+    rawBytes: payload.length,
+    rawSha256: sha256(payload),
+    session: sessionMeta,
+    stream,
+  };
+  try {
+    fs.writeFileSync(path.join(temporary, "payload"), payload, { mode: 0o600 });
+    fs.writeFileSync(path.join(temporary, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 });
+    fs.renameSync(temporary, finalDirectory);
+  } catch (error) {
+    fs.rmSync(temporary, { recursive: true, force: true });
+    throw error;
+  }
+  return finalDirectory;
+}
+
+function batchDirectoryBytes(directory) {
+  let total = 0;
+  for (const name of ["manifest.json", "payload", "failure.json"]) {
+    try {
+      total += fs.statSync(path.join(directory, name)).size;
+    } catch {
+      // Missing or concurrently moved files do not contribute to the estimate.
+    }
+  }
+  return total;
+}
+
+function recordOutboxDrop(root, droppedBatches, droppedBytes, reason) {
+  if (droppedBatches === 0) return;
+  const noticePath = path.join(root, "outbox-drops.json");
+  const prior = readJson(noticePath, { droppedBatches: 0, droppedBytes: 0 });
+  atomicWrite(noticePath, `${JSON.stringify({
+    droppedBatches: Number(prior.droppedBatches || 0) + droppedBatches,
+    droppedBytes: Number(prior.droppedBytes || 0) + droppedBytes,
+    lastReason: reason,
+    updatedAt: new Date().toISOString(),
+  }, null, 2)}\n`);
+}
+
+function enforceOutboxLimits(root) {
+  const release = acquireLock(root, "flush");
+  if (!release) {
+    return { dropped: 0, pending: pendingDirectories(root).length, bytes: null, deferred: true };
+  }
+  try {
+    const maximumBytes = positiveIntegerEnvironment("IMPEL_SESSIONS_MAX_OUTBOX_BYTES", DEFAULT_MAX_OUTBOX_BYTES);
+    const maximumBatches = positiveIntegerEnvironment("IMPEL_SESSIONS_MAX_OUTBOX_BATCHES", DEFAULT_MAX_OUTBOX_BATCHES);
+    const directories = pendingDirectories(root);
+    let bytes = directories.reduce((total, directory) => total + batchDirectoryBytes(directory), 0);
+    let count = directories.length;
+    const dropped = [];
+    let droppedBytes = 0;
+    for (const directory of directories) {
+      if (count <= maximumBatches && bytes <= maximumBytes) break;
+      const size = batchDirectoryBytes(directory);
+      try {
+        fs.rmSync(directory, { recursive: true, force: true });
+        dropped.push(directory);
+        droppedBytes += size;
+        count -= 1;
+        bytes = Math.max(0, bytes - size);
+      } catch {
+        // The batch disappeared before maintenance acquired its exclusive lock.
+      }
+    }
+    recordOutboxDrop(root, dropped.length, droppedBytes, "local outbox safety limit exceeded; oldest pending batches were evicted");
+    return { dropped: dropped.length, pending: count, bytes };
+  } finally {
+    release();
+  }
+}
+
+function captureLedger(root, sessionMeta, input) {
+  const compact = compactLedgerPayload(input);
+  const event = {
+    schemaVersion: 1,
+    capturedAt: new Date().toISOString(),
+    provider: sessionMeta.provider,
+    surface: sessionMeta.surface,
+    sessionTreeId: sessionMeta.sessionTreeId,
+    event: String(input.hook_event_name || "unknown").slice(0, 128),
+    ...compact,
+  };
+  const payload = Buffer.from(`${JSON.stringify(event)}\n`, "utf8");
+  return spoolBatch(root, sessionMeta, {
+    kind: "hook_ledger",
+    subpath: sessionMeta.surface,
+    format: "ndjson",
+  }, payload, 1);
+}
+
+function captureTranscript(root, sessionMeta, input) {
+  const transcript = allowedTranscript(input.transcript_path, sessionMeta.provider);
+  if (!transcript) return 0;
+  const cursorsRoot = path.join(root, "cursors");
+  privateDirectory(cursorsRoot);
+  const cursorKey = sha256(transcript.real).slice(0, 32);
+  const cursorPath = path.join(cursorsRoot, `${cursorKey}.json`);
+  const prior = readJson(cursorPath, { offset: 0, generation: 0 });
+  let offset = Number.isSafeInteger(prior.offset) && prior.offset >= 0 ? prior.offset : 0;
+  let generation = Number.isSafeInteger(prior.generation) && prior.generation >= 0 ? prior.generation : 0;
+  let discardUntilNewline = prior.discardUntilNewline === true;
+  const identity = `${transcript.stat.dev}:${transcript.stat.ino}`;
+  if ((prior.identity && prior.identity !== identity) || transcript.stat.size < offset) {
+    offset = 0;
+    generation += 1;
+    discardUntilNewline = false;
+  }
+  let captured = 0;
+  let batches = 0;
+  const descriptor = fs.openSync(transcript.real, "r");
+  try {
+    while (offset < transcript.stat.size && captured < MAX_TRANSCRIPT_CAPTURE_BYTES) {
+      const batch = transcriptBatch(descriptor, offset, transcript.stat.size, discardUntilNewline);
+      if (!batch) break;
+      if (batch.payload) {
+        spoolBatch(root, sessionMeta, {
+          kind: "transcript",
+          subpath: `transcripts/${cursorKey}-g${generation}`,
+          format: "ndjson",
+        }, batch.payload, batch.itemCount);
+        batches += 1;
+      }
+      offset += batch.bytesConsumed;
+      captured += batch.bytesConsumed;
+      discardUntilNewline = batch.discardUntilNewline;
+      atomicWrite(cursorPath, `${JSON.stringify({
+        path: transcript.real,
+        identity,
+        offset,
+        generation,
+        ...(discardUntilNewline ? { discardUntilNewline: true } : {}),
+        updatedAt: new Date().toISOString(),
+      }, null, 2)}\n`);
+      if (captured >= MAX_TRANSCRIPT_CAPTURE_BYTES) break;
+    }
+  } finally {
+    fs.closeSync(descriptor);
+  }
+  return batches;
+}
+
+function gitValue(cwd, args) {
+  try {
+    const result = spawnSync("git", args, { cwd, encoding: "utf8", timeout: 1500, stdio: ["ignore", "pipe", "ignore"] });
+    return result.status === 0 ? result.stdout.trim().slice(0, 2048) : "";
+  } catch {
+    return "";
+  }
+}
+
+function repositoryMetadata(cwd) {
+  if (typeof cwd !== "string" || !path.isAbsolute(cwd)) return {};
+  let repositoryUrl = gitValue(cwd, ["config", "--get", "remote.origin.url"]);
+  if (/^[A-Za-z][A-Za-z0-9+.-]*:\/\//u.test(repositoryUrl)) {
+    try {
+      const url = new URL(repositoryUrl);
+      url.username = "";
+      url.password = "";
+      repositoryUrl = url.toString();
+    } catch {
+      repositoryUrl = redactSecretText(repositoryUrl);
+    }
+  } else if (/^[^/@]+@[^:]+:/u.test(repositoryUrl)) {
+    repositoryUrl = repositoryUrl.replace(/^[^@]+@/u, "git@");
+  }
+  return {
+    ...(repositoryUrl ? { repositoryUrl } : {}),
+    repositoryRef: gitValue(cwd, ["symbolic-ref", "--short", "-q", "HEAD"]),
+    repositoryCommit: gitValue(cwd, ["rev-parse", "HEAD"]),
+  };
+}
+
+function sessionsUrl(config) {
+  return String(process.env.IMPEL_SESSIONS_URL || config?.sessionsUrl || DEFAULT_SESSIONS_URL).trim().replace(/\/+$/u, "");
+}
+
+function problemCode(body) {
+  return body?.error?.code || body?.code || "";
+}
+
+function problemMessage(body) {
+  return body?.error?.message || body?.detail || "unknown error";
+}
+
+function requestHeaders(config, tenantId, extra = {}) {
+  const headers = { accept: "application/json", ...extra };
+  const devOrg = process.env.IMPEL_SESSIONS_DEV_ORG_ID;
+  if (devOrg) {
+    headers["x-impel-dev-org-id"] = devOrg;
+    headers["x-impel-dev-user-id"] = process.env.IMPEL_SESSIONS_DEV_USER_ID || "impel-cli-smoke";
+  } else {
+    if (!config?.pat) throw new Error("Impel authentication is unavailable");
+    headers.authorization = `Bearer ${config.pat}`;
+    headers["x-impel-org-id"] = tenantId;
+  }
+  const bypass = process.env.IMPEL_SESSIONS_VERCEL_BYPASS_TOKEN;
+  if (bypass) headers["x-vercel-protection-bypass"] = bypass;
+  return headers;
+}
+
+async function apiRequest(config, tenantId, route, options = {}) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
+  try {
+    const response = await fetch(`${sessionsUrl(config)}${route}`, {
+      ...options,
+      headers: requestHeaders(config, tenantId, options.headers),
+      signal: controller.signal,
+    });
+    const text = await response.text();
+    let body = null;
+    if (text) {
+      try {
+        body = JSON.parse(text);
+      } catch {
+        body = { detail: text.slice(0, 1024) };
+      }
+    }
+    return { response, body };
+  } catch (error) {
+    const method = options.method || "GET";
+    throw new Error(`${method} ${route} failed: ${error?.name === "AbortError" ? `timed out after ${REQUEST_TIMEOUT_MS}ms` : error?.message || error}`);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+function compressZstd(payload) {
+  if (process.env.IMPEL_SESSIONS_DISABLE_DIRECT_UPLOAD === "1") return null;
+  if (typeof zlib.zstdCompressSync !== "function") return null;
+  const level = zlib.constants?.ZSTD_c_compressionLevel;
+  const windowLog = zlib.constants?.ZSTD_c_windowLog;
+  const params = {};
+  if (Number.isInteger(level)) params[level] = 3;
+  if (Number.isInteger(windowLog)) params[windowLog] = 20;
+  return zlib.zstdCompressSync(payload, Object.keys(params).length > 0 ? { params } : undefined);
+}
+
+async function directBlobPut(upload, payload) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), DIRECT_UPLOAD_TIMEOUT_MS);
+  try {
+    const response = await fetch(upload.url, {
+      method: upload.method || "PUT",
+      headers: upload.headers || {},
+      body: payload,
+      signal: controller.signal,
+    });
+    if (!response.ok) {
+      const detail = (await response.text()).slice(0, 1024);
+      throw new Error(`private Blob upload failed with HTTP ${response.status}: ${detail}`);
+    }
+  } catch (error) {
+    throw new Error(`private Blob upload failed: ${error?.name === "AbortError" ? `timed out after ${DIRECT_UPLOAD_TIMEOUT_MS}ms` : error?.message || error}`);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+async function uploadDirect(config, tenantId, streamId, manifest, payload, compressed) {
+  const route = `/v1/streams/${streamId}/batches/${encodeURIComponent(manifest.batchId)}`;
+  const reserved = await apiRequest(config, tenantId, `${route}/upload`, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({
+      expectedSequence: manifest.expectedSequence,
+      itemCount: manifest.itemCount,
+      rawBytes: payload.length,
+      storedBytes: compressed.length,
+      rawSha256: manifest.rawSha256,
+      storedSha256: sha256(compressed),
+    }),
+  });
+  if (reserved.response.ok && reserved.body?.batch) return reserved;
+  if (!reserved.response.ok) return reserved;
+
+  // A prior process may have uploaded the frame and died before recording the
+  // finalize response. Finalize first so that retry never needs blob overwrite.
+  let finalized = await apiRequest(config, tenantId, `${route}/finalize`, { method: "POST" });
+  if (finalized.response.ok) return finalized;
+  if (finalized.response.status !== 409 || problemCode(finalized.body) !== "upload_missing") return finalized;
+
+  await directBlobPut(reserved.body.upload, compressed);
+  finalized = await apiRequest(config, tenantId, `${route}/finalize`, { method: "POST" });
+  return finalized;
+}
+
+async function cancelDirectUpload(config, tenantId, streamId, batchId) {
+  const cancelled = await apiRequest(
+    config,
+    tenantId,
+    `/v1/streams/${streamId}/batches/${encodeURIComponent(batchId)}/upload`,
+    { method: "DELETE" },
+  );
+  if (cancelled.response.ok || [404, 410].includes(cancelled.response.status)) return;
+  throw new Error(`direct upload cancellation failed with HTTP ${cancelled.response.status}: ${problemMessage(cancelled.body)}`);
+}
+
+async function ensureRemoteSession(config, tenantId, manifest) {
+  const id = stableUuid(`impel-session:${tenantId}:${manifest.session.provider}:${manifest.session.sessionTreeId}`);
+  const metadata = repositoryMetadata(manifest.session.cwd);
+  const payload = {
+    id,
+    provider: manifest.session.provider,
+    providerVersion: manifest.session.providerVersion || "",
+    sessionTreeId: manifest.session.sessionTreeId,
+    providerThreadId: manifest.session.providerThreadId,
+    surface: manifest.session.surface,
+    visibility: "user",
+    persistenceState: "mirrored",
+    resumability: "portable_only",
+    completeness: "best_effort",
+    ...metadata,
+  };
+  const created = await apiRequest(config, tenantId, "/v1/sessions", {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify(payload),
+  });
+  if (!created.response.ok && created.response.status !== 409) {
+    throw new Error(`session create failed with HTTP ${created.response.status}: ${problemMessage(created.body)}`);
+  }
+  if (created.response.status === 409) {
+    const existing = await apiRequest(config, tenantId, `/v1/sessions/${id}`);
+    if (!existing.response.ok) {
+      const error = new Error(`session lookup failed with HTTP ${existing.response.status}`);
+      error.permanentBatchFailure = existing.response.status === 404;
+      throw error;
+    }
+    const session = existing.body?.session;
+    if (session?.provider !== payload.provider || session?.sessionTreeId !== payload.sessionTreeId) {
+      const error = new Error("session ID conflict resolved to different provider identity");
+      error.permanentBatchFailure = true;
+      throw error;
+    }
+  }
+  return id;
+}
+
+async function linkTaskBestEffort(config, tenantId, root, sessionID, taskID) {
+  if (!taskID) return;
+  const errorPath = path.join(root, "task-link-error.json");
+  if (Array.isArray(config?.scopes) && !config.scopes.includes("tasks")) {
+    atomicWrite(errorPath, `${JSON.stringify({
+      at: new Date().toISOString(),
+      sessionId: sessionID,
+      taskId: taskID,
+      message: "Task link is pending because the current credential does not have the tasks scope",
+    }, null, 2)}\n`);
+    return;
+  }
+  try {
+    const linked = await apiRequest(
+      config,
+      tenantId,
+      `/v1/sessions/${sessionID}/tasks/${encodeURIComponent(taskID)}`,
+      { method: "PUT" },
+    );
+    if (!linked.response.ok) {
+      throw new Error(`task link failed with HTTP ${linked.response.status}: ${problemMessage(linked.body)}`);
+    }
+    fs.rmSync(errorPath, { force: true });
+  } catch (error) {
+    const message = redactSecretText(error?.message || error).slice(0, 1024);
+    atomicWrite(errorPath, `${JSON.stringify({
+      at: new Date().toISOString(),
+      sessionId: sessionID,
+      taskId: taskID,
+      message,
+    }, null, 2)}\n`);
+  }
+}
+
+async function ensureRemoteStream(config, tenantId, sessionId, stream) {
+  const streamId = stableUuid(`impel-stream:${sessionId}:${stream.kind}:${stream.subpath}:${stream.format}`);
+  const created = await apiRequest(config, tenantId, `/v1/sessions/${sessionId}/streams`, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({ id: streamId, ...stream, requireLease: false }),
+  });
+  if (!created.response.ok && created.response.status !== 409) {
+    throw new Error(`stream create failed with HTTP ${created.response.status}: ${problemMessage(created.body)}`);
+  }
+  const current = await apiRequest(config, tenantId, `/v1/streams/${streamId}`);
+  if (!current.response.ok) throw new Error(`stream lookup failed with HTTP ${current.response.status}`);
+  return current.body.stream;
+}
+
+async function uploadPending(config, tenantId, directory, remote) {
+  const manifestPath = path.join(directory, "manifest.json");
+  const manifest = readJson(manifestPath);
+  if (!manifest || manifest.schemaVersion !== 1) throw new Error(`invalid pending manifest: ${directory}`);
+  const payload = fs.readFileSync(path.join(directory, "payload"));
+  if (payload.length !== manifest.rawBytes || sha256(payload) !== manifest.rawSha256) {
+    throw new Error(`pending payload checksum mismatch: ${directory}`);
+  }
+  const sessionCacheKey = `${manifest.session.provider}:${manifest.session.sessionTreeId}`;
+  let sessionId = remote.sessions.get(sessionCacheKey);
+  if (!sessionId) {
+    sessionId = await ensureRemoteSession(config, tenantId, manifest);
+    remote.sessions.set(sessionCacheKey, sessionId);
+  }
+  const root = path.dirname(path.dirname(directory));
+  await linkTaskBestEffort(config, tenantId, root, sessionId, manifest.session.taskId);
+  const streamCacheKey = `${sessionId}:${manifest.stream.kind}:${manifest.stream.subpath}:${manifest.stream.format}`;
+  let stream = remote.streams.get(streamCacheKey);
+  if (!stream) {
+    stream = await ensureRemoteStream(config, tenantId, sessionId, manifest.stream);
+    remote.streams.set(streamCacheKey, stream);
+  }
+  if (!Number.isSafeInteger(manifest.expectedSequence) || manifest.expectedSequence < 0) {
+    manifest.expectedSequence = stream.headSequence;
+    atomicWrite(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
+  }
+  const send = async () => apiRequest(config, tenantId, `/v1/streams/${stream.id}/batches/${encodeURIComponent(manifest.batchId)}`, {
+    method: "PUT",
+    headers: {
+      "content-type": manifest.stream.format === "ndjson" ? "application/x-ndjson" : "application/octet-stream",
+      "x-impel-expected-sequence": String(manifest.expectedSequence),
+      "x-impel-item-count": String(manifest.itemCount),
+      "x-impel-raw-sha256": manifest.rawSha256,
+    },
+    body: payload,
+  });
+  let compressed = compressZstd(payload);
+  const attemptUpload = async () => {
+    let result = compressed
+      ? await uploadDirect(config, tenantId, stream.id, manifest, payload, compressed)
+      : await send();
+    if (compressed && result.response.status === 409 && problemCode(result.body) === "direct_upload_unavailable") {
+      compressed = null;
+      result = await send();
+    }
+    return result;
+  };
+  let uploaded = await attemptUpload();
+  for (let recovery = 0; !uploaded.response.ok && recovery < 3; recovery += 1) {
+    const code = problemCode(uploaded.body);
+    if (![409, 410].includes(uploaded.response.status) || ![
+      "sequence_conflict",
+      "idempotency_conflict",
+      "upload_expired",
+    ].includes(code)) break;
+    if (compressed) await cancelDirectUpload(config, tenantId, stream.id, manifest.batchId);
+    stream = await ensureRemoteStream(config, tenantId, sessionId, manifest.stream);
+    remote.streams.set(streamCacheKey, stream);
+    manifest.expectedSequence = stream.headSequence;
+    atomicWrite(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
+    uploaded = await attemptUpload();
+  }
+  if (!uploaded.response.ok) {
+    const error = new Error(`batch upload failed with HTTP ${uploaded.response.status}: ${problemMessage(uploaded.body)}`);
+    error.httpStatus = uploaded.response.status;
+    error.problemCode = problemCode(uploaded.body);
+    throw error;
+  }
+  stream.headSequence = uploaded.body?.batch?.endSequence ?? (manifest.expectedSequence + manifest.itemCount);
+  fs.rmSync(directory, { recursive: true, force: true });
+}
+
+function pendingDirectories(root) {
+  const pending = path.join(root, "pending");
+  try {
+    return fs.readdirSync(pending, { withFileTypes: true })
+      .filter((entry) => entry.isDirectory() && !entry.name.startsWith(".tmp-"))
+      .map((entry) => path.join(pending, entry.name))
+      .sort();
+  } catch {
+    return [];
+  }
+}
+
+function directoryCount(root) {
+  try {
+    return fs.readdirSync(root, { withFileTypes: true }).filter((entry) => entry.isDirectory()).length;
+  } catch {
+    return 0;
+  }
+}
+
+function recordBatchFailure(directory, error) {
+  const failurePath = path.join(directory, "failure.json");
+  const prior = readJson(failurePath, { attempts: 0 });
+  const failure = {
+    attempts: Number(prior.attempts || 0) + 1,
+    at: new Date().toISOString(),
+    message: redactSecretText(error?.message || error).slice(0, 1024),
+    ...(Number.isInteger(error?.httpStatus) ? { httpStatus: error.httpStatus } : {}),
+    ...(error?.problemCode ? { problemCode: error.problemCode } : {}),
+  };
+  atomicWrite(failurePath, `${JSON.stringify(failure, null, 2)}\n`);
+  return failure;
+}
+
+function batchFailureIsPermanent(error, failure) {
+  if (error?.permanentBatchFailure === true) return true;
+  if (/^invalid pending manifest:|^pending payload checksum mismatch:/u.test(error?.message || "")) return true;
+  if ([413, 422].includes(error?.httpStatus)) return true;
+  if ([
+    "batch_too_large",
+    "idempotency_conflict",
+    "invalid_batch_identity",
+    "invalid_checksum",
+    "invalid_payload",
+    "invalid_upload_manifest",
+  ].includes(error?.problemCode)) return failure.attempts >= MAX_BATCH_FAILURES;
+  return false;
+}
+
+function quarantineBatch(root, directory, failure) {
+  const failedRoot = path.join(root, "pending-failed");
+  privateDirectory(failedRoot);
+  const destination = path.join(failedRoot, path.basename(directory));
+  try {
+    fs.renameSync(directory, destination);
+  } catch (error) {
+    if (error?.code !== "ENOENT") throw error;
+    return;
+  }
+  atomicWrite(path.join(destination, "failure.json"), `${JSON.stringify({
+    ...failure,
+    quarantinedAt: new Date().toISOString(),
+  }, null, 2)}\n`);
+  const quarantined = fs.readdirSync(failedRoot, { withFileTypes: true })
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => path.join(failedRoot, entry.name))
+    .sort();
+  for (const stale of quarantined.slice(0, Math.max(0, quarantined.length - MAX_QUARANTINED_BATCHES))) {
+    fs.rmSync(stale, { recursive: true, force: true });
+  }
+}
+
+async function flushRoot(root, tenantId, config, limit = MAX_FLUSH_BATCHES) {
+  const release = acquireLock(root, "flush");
+  if (!release) return { flushed: 0, pending: pendingDirectories(root).length, busy: true };
+  let flushed = 0;
+  const remote = { sessions: new Map(), streams: new Map() };
+  try {
+    while (flushed < limit) {
+      const directories = pendingDirectories(root).slice(0, limit - flushed);
+      if (directories.length === 0) break;
+      for (const directory of directories) {
+        try {
+          await uploadPending(config, tenantId, directory, remote);
+          flushed += 1;
+        } catch (error) {
+          const failure = recordBatchFailure(directory, error);
+          if (!batchFailureIsPermanent(error, failure)) throw error;
+          quarantineBatch(root, directory, failure);
+        }
+      }
+    }
+    const pendingLink = readJson(path.join(root, "task-link-error.json"));
+    if (pendingLink?.sessionId && pendingLink?.taskId) {
+      await linkTaskBestEffort(config, tenantId, root, pendingLink.sessionId, pendingLink.taskId);
+    }
+    clearError(root);
+    return { flushed, pending: pendingDirectories(root).length, busy: false };
+  } catch (error) {
+    writeError(root, error);
+    return { flushed, pending: pendingDirectories(root).length, busy: false, error };
+  } finally {
+    release();
+  }
+}
+
+export async function flushSessionOutbox({ tenantId, provider, sessionKey, config = loadConfig() }) {
+  return flushRoot(sessionDirectory(tenantId, provider, sessionKey), tenantId, config);
+}
+
+async function flushOneOtherOutbox({ tenantId, provider, currentRoot, config }) {
+  const providerRoot = path.dirname(currentRoot);
+  let candidates;
+  try {
+    candidates = fs.readdirSync(providerRoot, { withFileTypes: true })
+      .filter((entry) => entry.isDirectory())
+      .map((entry) => path.join(providerRoot, entry.name))
+      .filter((root) => root !== currentRoot && (
+        pendingDirectories(root).length > 0 || fs.existsSync(path.join(root, "task-link-error.json"))
+      ))
+      .sort();
+  } catch {
+    return { flushed: 0, pending: 0 };
+  }
+  if (candidates.length === 0) return { flushed: 0, pending: 0 };
+  return flushRoot(candidates[0], tenantId, config, Math.min(4, MAX_FLUSH_BATCHES));
+}
+
+export async function flushCollectedSession({ tenantId, provider, sessionKey, config = loadConfig() }) {
+  const root = sessionDirectory(tenantId, provider, sessionKey);
+  const current = await flushSessionOutbox({ tenantId, provider, sessionKey, config });
+  if (!current.error && !current.busy) {
+    const other = await flushOneOtherOutbox({ tenantId, provider, currentRoot: root, config });
+    return {
+      ...current,
+      flushed: current.flushed + other.flushed,
+      recoveredOtherSessionBatches: other.flushed,
+    };
+  }
+  return current;
+}
+
+export async function collectSessionHook({ provider, surface, tenantId, input, config = loadConfig(), flush = true }) {
+  if (!tenantId) throw new Error("The hook has no Impel tenant");
+  if (!["claude_code", "codex"].includes(provider)) throw new Error("Unsupported session provider");
+  if (!SAFE_ID_RE.test(surface)) throw new Error("Invalid session surface");
+  const sessionKey = String(input?.session_id || "").trim();
+  if (!sessionKey || sessionKey.length > 512 || /[\r\n\0]/u.test(sessionKey)) throw new Error("Hook input has no valid session_id");
+  const taskId = String(process.env.IMPEL_TASK_ID || "").trim();
+  if (taskId && !TASK_ID_RE.test(taskId)) throw new Error("IMPEL_TASK_ID is invalid");
+  const sessionMeta = {
+    provider,
+    surface,
+    sessionTreeId: sessionKey,
+    providerThreadId: sessionKey,
+    cwd: typeof input.cwd === "string" ? input.cwd : process.cwd(),
+    ...(taskId ? { taskId } : {}),
+  };
+  const root = sessionDirectory(tenantId, provider, sessionKey);
+  privateDirectory(root);
+  captureLedger(root, sessionMeta, input);
+  const releaseCapture = acquireLock(root, "capture", 30_000);
+  if (releaseCapture) {
+    try {
+      captureTranscript(root, sessionMeta, input);
+    } finally {
+      releaseCapture();
+    }
+  }
+  const limits = enforceOutboxLimits(root);
+  const selectedTenant = process.env.IMPEL_SESSIONS_DEV_ORG_ID || config?.tenantId === tenantId;
+  if (flush && config && selectedTenant) return flushCollectedSession({ tenantId, provider, sessionKey, config });
+  return {
+    flushed: 0,
+    pending: pendingDirectories(root).length,
+    queued: true,
+    dropped: limits.dropped,
+    ...(!selectedTenant ? { deferredForTenantSelection: true } : {}),
+  };
+}
+
+export async function readHookInput(stream = process.stdin) {
+  const chunks = [];
+  let bytes = 0;
+  for await (const chunk of stream) {
+    bytes += chunk.length;
+    if (bytes > MAX_HOOK_INPUT_BYTES) throw new Error("Hook input exceeds the 2 MiB safety limit");
+    chunks.push(chunk);
+  }
+  const raw = Buffer.concat(chunks).toString("utf8");
+  const value = JSON.parse(raw);
+  if (!value || Array.isArray(value) || typeof value !== "object") throw new Error("Hook input must be a JSON object");
+  return value;
+}
+
+export function sessionOutboxStatus({ tenantId, provider, sessionKey }) {
+  const root = sessionDirectory(tenantId, provider, sessionKey);
+  return {
+    root,
+    pending: pendingDirectories(root).length,
+    quarantined: directoryCount(path.join(root, "pending-failed")),
+    dropped: readJson(path.join(root, "outbox-drops.json")),
+    taskLinkError: readJson(path.join(root, "task-link-error.json")),
+    lastError: readJson(path.join(root, "last-error.json")),
+  };
+}