@scalequality/cli 0.4.21 → 0.4.22

package/dist/connect.build.json

@@ -1,5 +1,5 @@
 {
   "sdkVersion": "0.3.281",
-  "sha256": "64b584e7f307f8bed9bee151ca7dd57c5fe8e54c8adf6e9fd76ba11dbfba93c1",
-  "sourceCommit": "a4093e270f574ff0830763c574a17585cf269fe6"
+  "sha256": "46fe6b09a8c69a92a0f54cc420a3596c6f13b82177d3aec1d46456ae508a4517",
+  "sourceCommit": "2e2e401c42e98d17e1c0dd8286bdc6d15e44f77b"
 }

package/dist/connect.cjs

@@ -563,10 +563,15 @@ async function writeScrubbedTranscript(source, target) {
 
 // src/application/services/workspaceSandbox/backgroundService.ts
 var import_child_process2 = require("child_process");
-var import_crypto3 = require("crypto");
+var import_crypto4 = require("crypto");
 var import_fs5 = require("fs");
 var import_path6 = require("path");
 
+// src/application/services/workspaceSandbox/HttpSessionTransport.ts
+var import_crypto2 = require("crypto");
+var import_util2 = require("util");
+var import_zlib = require("zlib");
+
 // src/application/services/workspaceSandbox/reasoning.ts
 var REASONING_LEVELS = ["off", "low", "medium", "high", "xhigh", "max"];
 function isReasoningLevel(value) {
@@ -729,6 +734,15 @@ function serializeCheckpoints(map) {
   }
   return Object.keys(repos).length ? JSON.stringify({ version: CHECKPOINT_MAP_VERSION, repos }) : "";
 }
+var CHECKPOINT_INLINE_MAX_BYTES = 6 * 1024 * 1024;
+var CHECKPOINT_PART_BYTES = 4 * 1024 * 1024;
+var CHECKPOINT_MAX_PARTS = 32;
+function checkpointPartsOf(raw) {
+  const p = raw;
+  if (!p || typeof p !== "object" || typeof p.sha256 !== "string" || !/^[0-9a-f]{64}$/.test(p.sha256)) return null;
+  if (!Number.isInteger(p.count) || p.count < 1 || p.count > CHECKPOINT_MAX_PARTS || !Number.isInteger(p.bytes) || p.bytes < 1) return null;
+  return { sha256: p.sha256, count: p.count, bytes: p.bytes };
+}
 
 // src/application/services/workspaceSandbox/workspaceGit.ts
 var run = (0, import_util.promisify)(import_child_process.execFile);
@@ -856,11 +870,36 @@ async function computeDiff(root, base, caps = DIFF_CAPS, scope) {
 async function worktreeTreeId(root, scope) {
   return withWorktreeIndex(root, async (env) => (await git(["write-tree"], { cwd: root, env })).trim(), scope);
 }
-async function checkpointPatch(root, base, scope) {
+var CHECKPOINT_KEPT_HEADER = "ScaleQuality-Kept:";
+var CHECKPOINT_KEPT_TEXT = { bytes: 1024 * 1024, lines: 2e4 };
+async function checkpointPatch(root, base, scope, opts = {}) {
   return withWorktreeIndex(root, async (env) => {
-    const patch = await git(["diff", "--cached", "--no-renames", "--binary", base, ...scopeSpec(scope)], { cwd: root, env, maxBuffer: 512 * 1024 * 1024 });
-    return patch ? `${CHECKPOINT_BASE_HEADER} ${base}
-${patch}` : "";
+    const head = `${CHECKPOINT_BASE_HEADER} ${base}
+`;
+    const whole = async () => {
+      const patch2 = await git(["diff", "--cached", "--no-renames", "--binary", base, ...scopeSpec(scope)], { cwd: root, env, maxBuffer: 512 * 1024 * 1024 });
+      return patch2 ? head + patch2 : "";
+    };
+    if (!opts.keepInFolder) return whole();
+    const changes = await listChangesWithEnv(root, base, env, scope);
+    const kept = [];
+    const carried = [];
+    for (const c of changes) {
+      const size = c.binary || c.status === "deleted" ? 0 : (await (0, import_promises2.stat)((0, import_path2.join)(root, c.path)).catch(() => null))?.size ?? 0;
+      const large = size > CHECKPOINT_KEPT_TEXT.bytes || c.additions + c.deletions > CHECKPOINT_KEPT_TEXT.lines;
+      if (c.binary || large) kept.push(c);
+      else carried.push(c.path);
+    }
+    if (!kept.length) return whole();
+    let patch = "";
+    for (let i = 0; i < carried.length; i += 200) {
+      patch += await git(
+        ["diff", "--cached", "--no-renames", "--binary", base, "--", ...carried.slice(i, i + 200)],
+        { cwd: root, env: { ...env, GIT_LITERAL_PATHSPECS: "1" }, maxBuffer: 512 * 1024 * 1024 }
+      );
+    }
+    return head + kept.map((c) => `${CHECKPOINT_KEPT_HEADER} ${c.status} ${JSON.stringify(c.path)}
+`).join("") + patch;
   }, scope);
 }
 var PR_FILE_MAX_BYTES = 2 * 1024 * 1024;
@@ -1931,6 +1970,8 @@ var TransportError = class extends Error {
 };
 
 // src/application/services/workspaceSandbox/HttpSessionTransport.ts
+var gzipAsync = (0, import_util2.promisify)(import_zlib.gzip);
+var gunzipAsync = (0, import_util2.promisify)(import_zlib.gunzip);
 var GONE = [401, 403, 404, 410];
 var HttpSessionTransport = class {
   constructor(opts) {
@@ -1946,6 +1987,8 @@ var HttpSessionTransport = class {
   gone;
   async bootstrap() {
     const raw = await this.get(`/bootstrap?features=${ENGINE_FEATURES.join(",")}`, this.opts.requestTimeoutMs ?? 3e4);
+    const parts = checkpointPartsOf(raw?.checkpointParts);
+    if (parts && typeof raw?.checkpointPatch !== "string") raw.checkpointPatch = await this.checkpointFromParts(parts);
     return toSessionBootstrap(raw);
   }
   async pollCommands(waitSeconds, signal) {
@@ -1972,7 +2015,31 @@ var HttpSessionTransport = class {
     return this.post("/pull-request", req, this.opts.toolTimeoutMs ?? 18e4);
   }
   async checkpoint(req) {
-    await this.post("/checkpoint", req, this.opts.requestTimeoutMs ?? 6e4);
+    const timeout = this.opts.requestTimeoutMs ?? 6e4;
+    if (Buffer.byteLength(JSON.stringify(req), "utf8") <= CHECKPOINT_INLINE_MAX_BYTES) {
+      await this.post("/checkpoint", req, timeout);
+      return;
+    }
+    const packed = await gzipAsync(Buffer.from(req.patch, "utf8"));
+    const sha256 = (0, import_crypto2.createHash)("sha256").update(req.patch, "utf8").digest("hex");
+    const count = Math.ceil(packed.length / CHECKPOINT_PART_BYTES);
+    if (count > CHECKPOINT_MAX_PARTS) throw new TransportError(`checkpoint of ${packed.length} compressed bytes is over the limit`, 413, false);
+    for (let index = 0; index < count; index++) {
+      const data = packed.subarray(index * CHECKPOINT_PART_BYTES, (index + 1) * CHECKPOINT_PART_BYTES).toString("base64");
+      await this.post("/checkpoint/parts", { sha256, index, count, data }, timeout);
+    }
+    await this.post("/checkpoint", { parts: { sha256, count, bytes: packed.length }, sdkSessionId: req.sdkSessionId }, timeout);
+  }
+  async checkpointFromParts(parts) {
+    const chunks = [];
+    for (let index = 0; index < parts.count; index++) {
+      const r = await this.get(`/checkpoint/parts/${index}`, this.opts.requestTimeoutMs ?? 6e4);
+      if (r?.sha256 !== parts.sha256 || typeof r.data !== "string") throw new TransportError("checkpoint part does not match", 502, false);
+      chunks.push(Buffer.from(r.data, "base64"));
+    }
+    const patch = (await gunzipAsync(Buffer.concat(chunks)).catch(() => Buffer.alloc(0))).toString("utf8");
+    if ((0, import_crypto2.createHash)("sha256").update(patch, "utf8").digest("hex") !== parts.sha256) throw new TransportError("checkpoint parts do not match", 502, false);
+    return patch;
   }
   async openRepository(repoFullName) {
     const raw = await this.post("/repositories/open", { repoFullName }, this.opts.requestTimeoutMs ?? 3e4);
@@ -2277,7 +2344,7 @@ var import_os3 = require("os");
 var import_path5 = require("path");
 
 // src/application/services/workspaceSandbox/localTracking.ts
-var import_crypto2 = require("crypto");
+var import_crypto3 = require("crypto");
 var import_fs4 = require("fs");
 var import_promises5 = require("fs/promises");
 var import_os2 = require("os");
@@ -2305,7 +2372,7 @@ function gitTracker(root, base, prefix) {
     },
     snapshot: () => worktreeTreeId(root, scope),
     restore: (tree) => restoreTree(root, tree, scope),
-    checkpoint: () => checkpointPatch(root, base, scope),
+    checkpoint: (opts) => checkpointPatch(root, base, scope, opts),
     prFiles: (published) => filesForPullRequest(root, base, published, scope),
     treeId: () => worktreeTreeId(root, scope)
   };
@@ -2389,7 +2456,7 @@ async function mapLimit(items, n, fn) {
 async function repoFingerprint(root) {
   const head = (await git(["rev-parse", "-q", "--verify", "HEAD"], { cwd: root }).catch(() => "")).trim();
   const status = await git(["--no-optional-locks", "status", "--porcelain", "-z", "--untracked-files=all"], { cwd: root, timeoutMs: 6e4 }).catch(() => "?");
-  const h = (0, import_crypto2.createHash)("sha256").update(head).update("\0").update(status);
+  const h = (0, import_crypto3.createHash)("sha256").update(head).update("\0").update(status);
   const recs = status.split("\0");
   let n = 0;
   for (let i = 0; i < recs.length && n < 2e3; i++) {
@@ -2561,10 +2628,10 @@ var LooseFileTracker = class {
     }
     await restoreTree(this.o.root, target, this.scope());
   }
-  async checkpoint() {
+  async checkpoint(opts) {
     await this.chain;
     if (!this.entries.length) return "";
-    return checkpointPatch(this.o.root, await this.baseTree(), this.scope());
+    return checkpointPatch(this.o.root, await this.baseTree(), this.scope(), opts);
   }
   async prFiles(published) {
     await this.chain;
@@ -3530,7 +3597,7 @@ function serviceSlug(api) {
   } catch {
   }
   const slug = host.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 40) || "custom";
-  return `${slug}-${(0, import_crypto3.createHash)("sha256").update(api).digest("hex").slice(0, 6)}`;
+  return `${slug}-${(0, import_crypto4.createHash)("sha256").update(api).digest("hex").slice(0, 6)}`;
 }
 function serviceNames(api) {
   const slug = serviceSlug(api);
@@ -3669,7 +3736,7 @@ function installStableCopy(o) {
   const bundle = (0, import_path6.join)(o.packageRoot, "dist", "connect.cjs");
   const bin = (dir2) => (0, import_path6.join)(dir2, "node_modules", "@scalequality", "cli", "bin", "scalequality.mjs");
   if (!(0, import_fs5.existsSync)(bundle)) throw new ServiceError("NOT_AN_INSTALLED_CLI", "This installation of @scalequality/cli is incomplete (dist/connect.cjs is missing).");
-  const sha = (0, import_crypto3.createHash)("sha256").update((0, import_fs5.readFileSync)(bundle)).digest("hex");
+  const sha = (0, import_crypto4.createHash)("sha256").update((0, import_fs5.readFileSync)(bundle)).digest("hex");
   const base = (0, import_path6.join)(sqHome(o.home), "cli");
   const dir = (0, import_path6.join)(base, pkg.version);
   if (inside2(dir, o.packageRoot)) return { dir, bin: bin(dir), version: pkg.version, reused: true };
@@ -4817,7 +4884,7 @@ async function addFolder(credentials2, api, path, home) {
 var import_child_process3 = require("child_process");
 var import_promises14 = require("fs/promises");
 var import_fs9 = require("fs");
-var import_crypto7 = require("crypto");
+var import_crypto8 = require("crypto");
 var import_path15 = require("path");
 
 // src/application/services/execution/LanguageAdapter.ts
@@ -5261,7 +5328,7 @@ function limitInEngineText(text3) {
 }
 
 // src/application/services/workspaceSandbox/scalequalityTools.ts
-var import_crypto4 = require("crypto");
+var import_crypto5 = require("crypto");
 
 // node_modules/zod/v3/external.js
 var external_exports = {};
@@ -9381,7 +9448,7 @@ function buildScaleQualityServer(sdk, host) {
   const tools = REMOTE_TOOLS.map(
     (spec) => sdk.tool(spec.name, spec.description, spec.shape, async (args) => {
       const a = { ...args ?? {} };
-      if (spec.idempotent && typeof a.idempotencyKey !== "string") a.idempotencyKey = (0, import_crypto4.randomUUID)();
+      if (spec.idempotent && typeof a.idempotencyKey !== "string") a.idempotencyKey = (0, import_crypto5.randomUUID)();
       return callScaleQualityTool(host, spec.name, a);
     })
   );
@@ -9956,7 +10023,7 @@ function blockText(content) {
 }
 
 // src/application/services/workspaceSandbox/inputChannel.ts
-var import_crypto5 = require("crypto");
+var import_crypto6 = require("crypto");
 var InputChannel = class {
   buffer = [];
   waiting = null;
@@ -10014,7 +10081,7 @@ var InputChannel = class {
 var UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
 function messageUuid(id) {
   if (UUID.test(id)) return id.toLowerCase();
-  const h = (0, import_crypto5.createHash)("sha256").update(`sq-message:${id}`).digest("hex");
+  const h = (0, import_crypto6.createHash)("sha256").update(`sq-message:${id}`).digest("hex");
   return `${h.slice(0, 8)}-${h.slice(8, 12)}-5${h.slice(13, 16)}-${(parseInt(h[16], 16) & 3 | 8).toString(16)}${h.slice(17, 20)}-${h.slice(20, 32)}`;
 }
 function userMessage(text3, images, uuid2, priority) {
@@ -10346,11 +10413,11 @@ async function detectTestCommandFromManifests(dir) {
 }
 
 // src/application/services/workspaceSandbox/transcript.ts
-var import_crypto6 = require("crypto");
+var import_crypto7 = require("crypto");
 var import_fs8 = require("fs");
 var import_promises11 = require("fs/promises");
 var import_path12 = require("path");
-var import_zlib = require("zlib");
+var import_zlib2 = require("zlib");
 var TRANSCRIPT_CAPS = { maxRawBytes: 64 * 1024 * 1024, maxCompressedBytes: 4 * 1024 * 1024 };
 function engineProjectDir(cwd) {
   return cwd.replace(/[^a-zA-Z0-9]/g, "-");
@@ -10389,10 +10456,10 @@ async function packTranscript(configDir, sessionId, redactor, caps = TRANSCRIPT_
   }
   const text3 = lines2.length ? `${lines2.join("\n")}
 ` : "";
-  const zipped = (0, import_zlib.gzipSync)(Buffer.from(text3, "utf8"), { level: 9 });
+  const zipped = (0, import_zlib2.gzipSync)(Buffer.from(text3, "utf8"), { level: 9 });
   if (zipped.length > caps.maxCompressedBytes) return { tooLarge: true, bytes: zipped.length };
   return {
-    payload: { sdkSessionId: sessionId, encoding: "gzip-base64", data: zipped.toString("base64"), bytes: Buffer.byteLength(text3), sha256: (0, import_crypto6.createHash)("sha256").update(text3).digest("hex") },
+    payload: { sdkSessionId: sessionId, encoding: "gzip-base64", data: zipped.toString("base64"), bytes: Buffer.byteLength(text3), sha256: (0, import_crypto7.createHash)("sha256").update(text3).digest("hex") },
     secretsRemoved: removed
   };
 }
@@ -10401,11 +10468,11 @@ async function restoreTranscript(configDir, cwd, t) {
   if (await findTranscript(configDir, t.sdkSessionId)) return true;
   let text3;
   try {
-    text3 = (0, import_zlib.gunzipSync)(Buffer.from(t.data, "base64"), { maxOutputLength: TRANSCRIPT_CAPS.maxRawBytes });
+    text3 = (0, import_zlib2.gunzipSync)(Buffer.from(t.data, "base64"), { maxOutputLength: TRANSCRIPT_CAPS.maxRawBytes });
   } catch {
     return false;
   }
-  if (t.sha256 && (0, import_crypto6.createHash)("sha256").update(text3).digest("hex") !== t.sha256) return false;
+  if (t.sha256 && (0, import_crypto7.createHash)("sha256").update(text3).digest("hex") !== t.sha256) return false;
   const dir = (0, import_path12.join)(configDir, "projects", engineProjectDir(cwd));
   await (0, import_promises11.mkdir)(dir, { recursive: true, mode: 448 });
   const target = (0, import_path12.join)(dir, `${t.sdkSessionId}.jsonl`);
@@ -12014,7 +12081,7 @@ ${String(item.content)}` : String(item.content);
     if (files === "unavailable") this.error("REWIND_FAILED", { turnId });
     this.emit({ type: "rewind", data: { turnId, files, conversation, ...untouched.length ? { untouched } : {} } });
     await this.diffNow();
-    await this.saveCheckpoint().catch(() => void 0);
+    await this.saveCheckpoint().catch((e) => this.deps.log.warn("checkpoint failed", { error: this.redactor.text(e.message) }));
     this.setState("READY", "REWOUND");
     await this.sink.flush();
   }
@@ -12575,7 +12642,7 @@ ${text3}`;
       this.turnWatch = null;
       this.previousTrouble = stopped || !watch ? null : watch.errored ? "follows a request that ended with an error" : watch.lastExit !== null && watch.lastExit !== 0 ? "follows a failed verification" : null;
       await this.diffNow();
-      await this.saveCheckpoint().catch(() => void 0);
+      await this.saveCheckpoint().catch((e) => this.deps.log.warn("checkpoint failed", { error: this.redactor.text(e.message) }));
       this.setState("READY", stopped ? "STOPPED" : void 0);
       await this.sink.flush();
     }
@@ -12960,9 +13027,18 @@ ${text3}`;
    * The checkpoint: every open repository's change against its base, plus the
    * saved changes of repositories not open in this run, as one map. In the
    * cloud workspace the engine's transcript goes with it (v4), so a resumed
-   * session continues the same conversation.
+   * session continues the same conversation. On the person's computer the
+   * folder keeps its binary and very large files, which the map only names.
+   * The transcript is kept even when the change could not be.
    */
   async saveCheckpoint() {
+    try {
+      await this.saveChange();
+    } finally {
+      await this.saveTranscript().catch((e) => this.deps.log.warn("transcript not kept", { error: e.message }));
+    }
+  }
+  async saveChange() {
     const map = new Map(this.pendingCheckpoints);
     for (const repo2 of this.repos.values()) {
       const key2 = repo2.repoFullName ?? repo2.localId ?? LOCAL_FOLDER_KEY;
@@ -12970,7 +13046,7 @@ ${text3}`;
         if (repo2.unsaved) map.set(key2, repo2.unsaved);
         continue;
       }
-      const patch2 = await repo2.tracker.checkpoint();
+      const patch2 = await repo2.tracker.checkpoint(this.local ? { keepInFolder: true } : void 0);
       if (patch2) map.set(key2, patch2);
       else if (repo2.unsaved) map.set(key2, repo2.unsaved);
       else map.delete(key2);
@@ -12982,7 +13058,6 @@ ${patch}`;
       await this.deps.transport.checkpoint({ patch, sdkSessionId: this.sdkSessionId });
       this.lastCheckpoint = key;
     }
-    await this.saveTranscript().catch((e) => this.deps.log.warn("transcript not kept", { error: e.message }));
   }
   /** The engine conversation, without secrets, kept by the API (only when it changed). */
   async saveTranscript() {
@@ -13027,8 +13102,7 @@ ${patch}`;
     if (running) await Promise.race([running, new Promise((r) => setTimeout(r, 2e4))]);
     if (opts.checkpoint) {
       await this.saveCheckpoint().catch((e) => {
-        this.deps.log.warn("checkpoint failed", { error: e.message });
-        this.error("CHECKPOINT_FAILED");
+        this.deps.log.warn("checkpoint failed", { error: this.redactor.text(e.message) });
       });
     }
     if (reason === "shutdown" && this.state && this.state !== "FAILED") this.setState("PAUSED", pausedCode);
@@ -13060,7 +13134,7 @@ ${patch}`;
       this.pendingCheckpoints.delete(repo2.localId);
       repo2.dormant = !repo2.unsaved;
     }
-    const stateDir = this.deps.stateDir ?? (0, import_path15.join)(this.deps.scratch, `folder-${(0, import_crypto7.createHash)("sha256").update(local.root).digest("hex").slice(0, 16)}`);
+    const stateDir = this.deps.stateDir ?? (0, import_path15.join)(this.deps.scratch, `folder-${(0, import_crypto8.createHash)("sha256").update(local.root).digest("hex").slice(0, 16)}`);
     const tracker = new LooseFileTracker({ root: local.root, stateDir, excluded: () => local.repos.map((r) => r.root) });
     const tracked = local.gitAvailable && await tracker.init().then(() => true, (e) => {
       this.deps.log.warn("loose files not tracked", { error: e.message });
@@ -13677,7 +13751,7 @@ function buildQueryOptions(o) {
 }
 function spawnKey(i) {
   const { orgMcpServers, ...data } = i;
-  return (0, import_crypto7.createHash)("sha256").update(JSON.stringify({ ...data, orgMcpServers: Object.keys(orgMcpServers).sort() })).digest("hex");
+  return (0, import_crypto8.createHash)("sha256").update(JSON.stringify({ ...data, orgMcpServers: Object.keys(orgMcpServers).sort() })).digest("hex");
 }
 function timedGate(gate, took) {
   const timed = async (run2) => {
@@ -13703,7 +13777,7 @@ async function hasLocalTranscript(configDir, sessionId) {
 }
 
 // src/application/services/workspaceSandbox/selfUpdate.ts
-var import_crypto8 = require("crypto");
+var import_crypto9 = require("crypto");
 var import_fs10 = require("fs");
 var import_path16 = require("path");
 var REGISTRY_LATEST = "https://registry.npmjs.org/@scalequality%2fcli/latest";
@@ -13886,7 +13960,7 @@ async function installVersion(o) {
     if (pkg?.name !== "@scalequality/cli" || pkg?.version !== o.version) throw new UpdateError("WRONG_PACKAGE", `npm installed ${String(pkg?.name)}@${String(pkg?.version)} instead of ${o.version}.`);
     const build = readJson2((0, import_path16.join)(root, "dist", "connect.build.json"));
     const bundle = (0, import_path16.join)(root, "dist", "connect.cjs");
-    const sha = (0, import_fs10.existsSync)(bundle) ? (0, import_crypto8.createHash)("sha256").update((0, import_fs10.readFileSync)(bundle)).digest("hex") : null;
+    const sha = (0, import_fs10.existsSync)(bundle) ? (0, import_crypto9.createHash)("sha256").update((0, import_fs10.readFileSync)(bundle)).digest("hex") : null;
     if (!sha || typeof build?.sha256 !== "string" || build.sha256 !== sha) throw new UpdateError("BUNDLE_MISMATCH", `The engine of ${o.version} does not match the sha256 it declares.`);
     let copy;
     try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scalequality/cli",
-  "version": "0.4.21",
+  "version": "0.4.22",
   "description": "ScaleQuality CLI. Connect your computer to the ScaleQuality AI Workspace (`scalequality login`), run its coding engine in any folder of your computer, and import your Claude Code and Codex conversations.",
   "type": "module",
   "bin": {