drop2run 0.10.0 → 0.11.0

package/dist/index.js

@@ -711,7 +711,9 @@ async function prepareDeploy(siteId, files, options, signal) {
   }));
   const prepared = await request(
     `sites/${encodeURIComponent(siteId)}/deploys/prepare`,
-    { files: manifest },
+    // `batches` asks for batch permits; an API that predates them ignores the field, and one that
+    // knows them answers with single permits until they are switched on.
+    { files: manifest, batches: true },
     options,
     signal
   );
@@ -800,6 +802,47 @@ async function problemToError(response) {
   const message = typeof problem.detail === "string" ? problem.detail : typeof problem.title === "string" ? problem.title : `The server responded with ${response.status}.`;
   return new DeployError(code, message, problem);
 }
+function batchLayout(members) {
+  return members.map((member) => `${member.sha256} ${member.size}
+`).join("");
+}
+function isBatch(target) {
+  return "files" in target;
+}
+function parseBatchResult(body) {
+  try {
+    const parsed = JSON.parse(body);
+    if (parsed === null || typeof parsed !== "object") return null;
+    const { stored, present, failed } = parsed;
+    if (typeof stored !== "number" || typeof present !== "number" || !Array.isArray(failed))
+      return null;
+    return { stored, present, failed };
+  } catch {
+    return null;
+  }
+}
+async function mapInOrder(items, limit, fn, signal) {
+  const results = new Array(items.length);
+  let next = 0;
+  let failure2 = null;
+  const lane = async () => {
+    for (; ; ) {
+      if (failure2 !== null) return;
+      const index = next++;
+      if (index >= items.length) return;
+      try {
+        results[index] = await fn(items[index], index);
+      } catch (error) {
+        failure2 ??= { error };
+        return;
+      }
+    }
+  };
+  const lanes = Array.from({ length: Math.min(Math.max(1, limit), items.length) }, lane);
+  await Promise.all(lanes);
+  if (failure2 !== null) throw failure2.error;
+  return results;
+}
 const FILE_TYPES = [
   // Documents. The three extensions that make a folder a documents site.
   {
@@ -1283,13 +1326,46 @@ async function sha256Hex(bytes) {
   const digest = await crypto.subtle.digest("SHA-256", bytes);
   return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
 }
+const HASH_CONCURRENCY = 4;
+const HASH_BYTES_IN_FLIGHT = 16 * 1024 * 1024;
 async function hashAll(files, onProgress, signal) {
-  const hashed = [];
-  for (const [index, file] of files.entries()) {
-    hashed.push({ ...file, sha256: await sha256Hex(await readForHash(file)) });
-    onProgress?.(index + 1, files.length);
-  }
-  return hashed;
+  const budget = byteBudget(HASH_BYTES_IN_FLIGHT);
+  let done = 0;
+  return mapInOrder(
+    files,
+    HASH_CONCURRENCY,
+    async (file) => {
+      const size = sizeOf(file.bytes);
+      await budget.acquire(size);
+      try {
+        signal?.throwIfAborted();
+        const hashed = { ...file, sha256: await sha256Hex(await readForHash(file)) };
+        done += 1;
+        onProgress?.(done, files.length);
+        return hashed;
+      } finally {
+        budget.release(size);
+      }
+    }
+  );
+}
+function byteBudget(limit) {
+  let held = 0;
+  let waiting = [];
+  return {
+    async acquire(size) {
+      while (held > 0 && held + size > limit) {
+        await new Promise((resolve2) => waiting.push(resolve2));
+      }
+      held += size;
+    },
+    release(size) {
+      held -= size;
+      const woken = waiting;
+      waiting = [];
+      for (const wake of woken) wake();
+    }
+  };
 }
 async function readForHash(file) {
   try {
@@ -1318,10 +1394,13 @@ const IGNORED_DIRECTORIES = [
   ".codex",
   ".gemini"
 ];
+function isEnvironmentFile(name) {
+  return name === ".env" || name === ".envrc" || name.startsWith(".env.");
+}
 function shouldIgnore(path) {
   const segments = path.split("/");
   const name = segments.at(-1) ?? "";
-  if (name === "" || IGNORED_EXACT.has(name)) return true;
+  if (name === "" || IGNORED_EXACT.has(name) || isEnvironmentFile(name)) return true;
   if (name.startsWith("._")) return true;
   return segments.some((segment) => IGNORED_DIRECTORIES.includes(segment));
 }
@@ -1456,9 +1535,7 @@ async function readSiteName(files) {
   }
 }
 function decodeHead(bytes) {
-  return new TextDecoder("utf-8", { fatal: false }).decode(
-    bytes.subarray(0, HEAD_BYTES)
-  );
+  return new TextDecoder("utf-8", { fatal: false }).decode(bytes.subarray(0, HEAD_BYTES));
 }
 function nameFromHtml(html) {
   const document = parse(html);
@@ -1483,9 +1560,7 @@ function parse(html) {
   }
 }
 function matchTag(html, tag) {
-  const match = new RegExp(`<${tag}[^>]*>([\\s\\S]*?)</${tag}\\s*>`, "i").exec(
-    html
-  );
+  const match = new RegExp(`<${tag}[^>]*>([\\s\\S]*?)</${tag}\\s*>`, "i").exec(html);
   return match?.[1] === void 0 ? null : decodeEntities(match[1]);
 }
 function matchOgSiteName(html) {
@@ -1586,6 +1661,7 @@ function trimSuffix(title) {
   return title;
 }
 const DEFAULT_CONCURRENCY = 8;
+const MAX_BATCH_CONCURRENCY = 64;
 const DEFAULT_BYTES_IN_FLIGHT = 64 * 1024 * 1024;
 const DEFAULT_RETRIES = 3;
 const BACKOFF_MS = [1e3, 2e3, 4e3];
@@ -1602,14 +1678,31 @@ async function uploadAll(targets, uploadUrl, files, onProgress, signal, deps = {
   const put = deps.put ?? defaultTransport(deps.fetch);
   const delay = deps.delay ?? defaultDelay;
   const concurrency = deps.concurrency ?? DEFAULT_CONCURRENCY;
+  const batchConcurrency = Math.max(
+    1,
+    Math.min(MAX_BATCH_CONCURRENCY, Math.floor(deps.batchConcurrency ?? concurrency))
+  );
   const retries = deps.retries ?? DEFAULT_RETRIES;
   const stallMs = deps.stallMs ?? DEFAULT_STALL_MS;
   const now = deps.now ?? Date.now;
   const maxBytesInFlight = deps.maxBytesInFlight ?? DEFAULT_BYTES_IN_FLIGHT;
   const byPath = new Map(files.map((file) => [file.path, file]));
+  const fileCount = targets.reduce(
+    (sum, target) => sum + (isBatch(target) ? target.files.length : 1),
+    0
+  );
   let nextIndex = 0;
   let done = 0;
   let failed = false;
+  let firstFailure;
+  const local = new AbortController();
+  const inner = local.signal;
+  const fail = (error) => {
+    if (failed) return;
+    failed = true;
+    firstFailure = error;
+    local.abort();
+  };
   let reserved = 0;
   let waiting = [];
   const acquire = async (size) => {
@@ -1624,17 +1717,32 @@ async function uploadAll(targets, uploadUrl, files, onProgress, signal, deps = {
     waiting = [];
     for (const wake of woken) wake();
   };
+  const running = { single: 0, batch: 0 };
+  const limit = { single: concurrency, batch: batchConcurrency };
+  let slotWaiting = [];
+  const takeSlot = async (kind) => {
+    while (running[kind] >= limit[kind]) {
+      await new Promise((resolve2) => slotWaiting.push(resolve2));
+    }
+    running[kind] += 1;
+  };
+  const freeSlot = (kind) => {
+    running[kind] -= 1;
+    const woken = slotWaiting;
+    slotWaiting = [];
+    for (const wake of woken) wake();
+  };
   let landedBytes = 0;
   const inFlight = /* @__PURE__ */ new Map();
   let lastReport = Number.NEGATIVE_INFINITY;
   const report = (path) => {
-    if (onProgress === void 0) return;
+    if (onProgress === void 0 || failed) return;
     const at = now();
     if (path === void 0 && at - lastReport < PROGRESS_INTERVAL_MS) return;
     lastReport = at;
     let bytes = landedBytes;
     for (const sent of inFlight.values()) bytes += sent;
-    onProgress(done, targets.length, bytes, path);
+    onProgress(done, fileCount, bytes, path);
   };
   const worker = async () => {
     for (; ; ) {
@@ -1642,25 +1750,49 @@ async function uploadAll(targets, uploadUrl, files, onProgress, signal, deps = {
       const index = nextIndex++;
       const target = targets[index];
       if (target === void 0) return;
-      const file = byPath.get(target.path);
-      if (file === void 0) {
-        failed = true;
-        throw new DeployError(
-          ClientErrorCode.UploadFailed,
-          `The server asked for ${target.path}, which is not in the manifest.`,
-          { path: target.path }
-        );
+      inner.throwIfAborted();
+      const refs = isBatch(target) ? target.files : [target];
+      const members = [];
+      for (const ref of refs) {
+        const file = byPath.get(ref.path);
+        if (file === void 0) {
+          const missing = new DeployError(
+            ClientErrorCode.UploadFailed,
+            `The server asked for ${ref.path}, which is not in the manifest.`,
+            { path: ref.path }
+          );
+          fail(missing);
+          throw missing;
+        }
+        members.push(file);
       }
-      const size = sizeOf(file.bytes);
-      await acquire(size);
+      const layout = isBatch(target) ? new TextEncoder().encode(
+        batchLayout(
+          target.files.map((ref, i) => ({
+            sha256: ref.sha256,
+            size: sizeOf(members[i].bytes)
+          }))
+        )
+      ) : null;
+      const size = members.reduce((sum, file) => sum + sizeOf(file.bytes), layout?.length ?? 0);
+      const copied = isBatch(target) ? members.reduce(
+        (sum, file) => sum + (file.bytes instanceof Blob ? 0 : sizeOf(file.bytes)),
+        0
+      ) : 0;
+      const reservation = size + copied;
+      const first = members[0].path;
+      const label = members.length === 1 ? first : `${first} and ${members.length - 1} more`;
+      const kind = isBatch(target) ? "batch" : "single";
+      await takeSlot(kind);
+      await acquire(reservation);
       try {
         if (failed) return;
-        signal?.throwIfAborted();
-        await uploadOne(target, file, {
+        inner.throwIfAborted();
+        const context = {
           put,
           delay,
           retries,
-          signal,
+          signal: inner,
           uploadUrl,
           stallMs,
           onSent: (sent) => {
@@ -1669,26 +1801,34 @@ async function uploadAll(targets, uploadUrl, files, onProgress, signal, deps = {
           },
           onRetry: (attempt, stalled) => {
             inFlight.set(index, 0);
-            onRetry?.(target.path, attempt, retries + 1, stalled);
+            onRetry?.(label, attempt, retries + 1, stalled);
           }
-        });
+        };
+        if (isBatch(target) && layout !== null)
+          await uploadBatchOne(target, members, layout, context);
+        else await uploadOne(target, members[0], context);
       } catch (error) {
-        failed = true;
+        fail(error);
         throw error;
       } finally {
-        release(size);
+        release(reservation);
+        freeSlot(kind);
       }
       inFlight.delete(index);
-      done += 1;
+      done += members.length;
       landedBytes += size;
-      report(file.path);
+      report(members.at(-1).path);
     }
   };
-  const workers = Array.from({ length: Math.min(concurrency, targets.length) }, worker);
+  const workers = Array.from(
+    { length: Math.min(Math.max(concurrency, batchConcurrency), targets.length) },
+    worker
+  );
   try {
     await Promise.all(workers);
   } catch (error) {
-    throw asDeployError(error);
+    throw asDeployError(firstFailure ?? error);
+  } finally {
   }
 }
 async function uploadOne(target, file, context) {
@@ -1701,7 +1841,7 @@ async function uploadOne(target, file, context) {
       context.signal?.throwIfAborted();
     }
     try {
-      const status = await context.put({
+      const answer = await context.put({
         url: context.uploadUrl,
         token: target.token,
         body: untyped(file.bytes),
@@ -1709,6 +1849,7 @@ async function uploadOne(target, file, context) {
         onSent: context.onSent,
         stallMs: context.stallMs
       });
+      const status = typeof answer === "number" ? answer : answer.status;
       if (status >= 200 && status < 300) return;
       if (status === 401 || status === 403 || status === 429) {
         throw new DeployError(
@@ -1737,6 +1878,85 @@ async function uploadOne(target, file, context) {
     cause: describeError(lastError)
   });
 }
+async function uploadBatchOne(batch, members, layout, context) {
+  const first = members[0].path;
+  const label = members.length === 1 ? first : `${first} and ${members.length - 1} more`;
+  let lastError;
+  for (let attempt = 0; attempt <= context.retries; attempt++) {
+    context.signal?.throwIfAborted();
+    if (attempt > 0) {
+      context.onRetry(attempt + 1, lastError instanceof StalledError);
+      await context.delay(BACKOFF_MS[attempt - 1] ?? BACKOFF_MS.at(-1) ?? 1e3);
+      context.signal?.throwIfAborted();
+    }
+    try {
+      const body = new Blob([layout, ...members.map((file) => file.bytes)]);
+      const answer = await context.put({
+        url: context.uploadUrl,
+        token: batch.token,
+        body,
+        signal: context.signal,
+        onSent: context.onSent,
+        stallMs: context.stallMs,
+        wantsBody: true
+      });
+      const status = typeof answer === "number" ? answer : answer.status;
+      if (status >= 200 && status < 300) {
+        const result = typeof answer === "number" ? null : parseBatchResult(answer.body);
+        if (result === null) {
+          lastError = new Error(
+            "The upload service answered with something that is not a batch result."
+          );
+          continue;
+        }
+        if (result.failed.length === 0) return;
+        const changed = result.failed.find((failure2) => failure2.reason === "hash_mismatch");
+        if (changed !== void 0) {
+          const path = members[changed.i]?.path ?? first;
+          throw new DeployError(
+            ClientErrorCode.ReadFailed,
+            `${path} changed after it was read, so it no longer matches what was checked. Drop the folder again.`,
+            { path }
+          );
+        }
+        lastError = new Error(
+          `${result.failed.length} of ${members.length} files were not stored.`
+        );
+        continue;
+      }
+      if ([400, 401, 403, 411, 413, 429].includes(status)) {
+        throw new DeployError(ClientErrorCode.UploadFailed, refusalMessage(status, label), {
+          path: first,
+          status
+        });
+      }
+      lastError = new Error(`HTTP ${status}`);
+    } catch (error) {
+      if (error instanceof DeployError) throw error;
+      if (isAbort(error)) throw cancelled();
+      lastError = error;
+    }
+  }
+  const message = lastError instanceof StalledError ? `Upload of ${label} stopped moving and did not recover after ${context.retries + 1} attempts. Check your connection and try again.` : `Could not upload ${label} after ${context.retries + 1} attempts.`;
+  throw new DeployError(ClientErrorCode.UploadFailed, message, {
+    path: first,
+    cause: describeError(lastError)
+  });
+}
+function refusalMessage(status, label) {
+  switch (status) {
+    case 401:
+      return `Upload of ${label} was refused because its upload window expired. Try deploying again.`;
+    case 429:
+      return `Upload of ${label} was refused: this deploy has used every upload it was given. Try deploying again.`;
+    case 400:
+    case 411:
+    case 413:
+      return `Upload of ${label} was refused because it no longer matches what was checked — a file may have changed since you dropped it. Drop the folder again.`;
+    default:
+      return `Upload of ${label} was refused by the upload service. Try deploying again.`;
+  }
+}
 function untyped(content) {
   if (ArrayBuffer.isView(content) || content.type === "") return content;
   return content.slice(0, content.size);
@@ -1747,7 +1967,8 @@ function defaultTransport(doFetch) {
   return Xhr === void 0 ? fetchTransport(globalFetch()) : xhrTransport(Xhr);
 }
 function fetchTransport(doFetch) {
-  return async ({ url, token, body, signal }) => {
+  return async (request2) => {
+    const { url, token, body, signal } = request2;
     const response = await doFetch(url, {
       method: "PUT",
       // The view, not a copy. Wrapping it in `new Blob([body.slice().buffer])` made two extra
@@ -1762,11 +1983,11 @@ function fetchTransport(doFetch) {
       headers: { Authorization: `Bearer ${token}` },
       ...signal ? { signal } : {}
     });
-    return response.status;
+    return request2.wantsBody === true ? { status: response.status, body: await response.text() } : response.status;
   };
 }
 function xhrTransport(Xhr) {
-  return ({ url, token, body, signal, onSent, stallMs }) => new Promise((resolve2, reject) => {
+  return ({ url, token, body, signal, onSent, stallMs, wantsBody }) => new Promise((resolve2, reject) => {
     const xhr = new Xhr();
     let settled = false;
     let timer;
@@ -1800,7 +2021,11 @@ function xhrTransport(Xhr) {
       onSent(event.loaded);
     };
     xhr.onprogress = arm;
-    xhr.onload = () => settle(() => resolve2(xhr.status));
+    xhr.onload = () => settle(
+      () => resolve2(
+        wantsBody === true ? { status: xhr.status, body: xhr.responseText ?? "" } : xhr.status
+      )
+    );
     xhr.onerror = () => settle(() => reject(new TypeError("Network error during upload.")));
     arm();
     xhr.send(body);
@@ -1865,11 +2090,16 @@ async function deploy(source, siteId, onProgress, signal, options = {}) {
       return;
     }
     const sizes = new Map(hashed.map((file) => [file.path, sizeOf(file.bytes)]));
-    const totalBytes = prepared.upload.reduce(
-      (sum, target) => sum + (sizes.get(target.path) ?? 0),
-      0
-    );
-    if (prepared.upload.length > 0) {
+    const batches = prepared.batches ?? [];
+    const targets = [...prepared.upload, ...batches];
+    const totalBytes = prepared.upload.reduce((sum, target) => sum + (sizes.get(target.path) ?? 0), 0) + batches.reduce((sum, batch) => {
+      const members = batch.files.map((ref) => ({
+        sha256: ref.sha256,
+        size: sizes.get(ref.path) ?? 0
+      }));
+      return sum + batchLayout(members).length + members.reduce((n, member) => n + member.size, 0);
+    }, 0);
+    if (targets.length > 0) {
       onProgress({
         type: "uploading",
         done: prepared.reused,
@@ -1880,7 +2110,7 @@ async function deploy(source, siteId, onProgress, signal, options = {}) {
       });
     }
     await uploadAll(
-      prepared.upload,
+      targets,
       prepared.uploadUrl,
       hashed,
       (done, _total, bytes, path) => onProgress({
@@ -1893,7 +2123,9 @@ async function deploy(source, siteId, onProgress, signal, options = {}) {
         ...path === void 0 ? {} : { path }
       }),
       signal,
-      options.upload,
+      // The server's number unless the caller set one: it is tuned on the server so a deploy of many
+      // small files speeds up without a CLI or MCP release.
+      typeof prepared.batchConcurrency === "number" && options.upload?.batchConcurrency === void 0 ? { ...options.upload, batchConcurrency: prepared.batchConcurrency } : options.upload,
       (path, attempt, attempts, stalled) => onProgress({ type: "retrying", path, attempt, attempts, stalled })
     );
     onProgress({ type: "completing", reused: prepared.reused });
@@ -3058,7 +3290,7 @@ async function run(commandLine) {
   const positional = [...positionals(argv), ...literal];
   const [command, ...rest] = positional;
   if (argv.includes("--version")) {
-    const { version } = await import("./package-CGI7Yt81.js").then(
+    const { version } = await import("./package-B79z0E4v.js").then(
       (module) => module.default
     );
     return { text: version, json: { version }, code: 0 };

package/dist/{package-CGI7Yt81.js → package-B79z0E4v.js}

@@ -1,5 +1,5 @@
 const name = "drop2run";
-const version = "0.10.0";
+const version = "0.11.0";
 const description = "Publish a page, a folder of documents or a built site to Drop2Run from the command line.";
 const license = "MIT";
 const type = "module";

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "drop2run",
-	"version": "0.10.0",
+	"version": "0.11.0",
 	"description": "Publish a page, a folder of documents or a built site to Drop2Run from the command line.",
 	"license": "MIT",
 	"type": "module",