bulletin-deploy 0.7.14 → 0.7.15-rc.1

package/README.md

@@ -115,6 +115,33 @@ The runtime uses a 24-hour cache at `${XDG_CACHE_HOME:-~/.cache}/bulletin-deploy
 - `pool accounts`: derived Bulletin uploader accounts used to spread nonce and authorization load.
 - `PoP`: Proof of Personhood, which some `.dot` names require before registration.
 
+## Incremental Upload
+
+After the first deploy of a `.dot` domain, every subsequent deploy automatically reuses chunks already stored on Bulletin instead of re-uploading them. There is no flag to enable; it just runs.
+
+How it works:
+
+1. The previous deploy embeds a manifest at `.bulletin-deploy/manifest.json` inside the deployed content (file classification, block ordering, chunk metadata).
+2. The new deploy fetches the previous contenthash from DotNS, then fetches that manifest via the Bulletin IPFS gateway.
+3. The new build's CAR is sliced into chunks; each chunk's CID is HEAD-probed against the gateway. Chunks already present are skipped.
+4. Only the chunks that actually changed (typically: the manifest itself and any modified content) are uploaded.
+
+The summary line at the end of a deploy shows the savings:
+
+```
+Cache:
+  Manifest:    embedded (1 attempt)
+  Probed:      18 chunks  →  15 cached, 2 to upload, 1 probe-failed
+  Recycled:    3 CIDs found on-chain that weren't in the previous manifest
+  Saved:       ~52s and 14.3 MB upload
+```
+
+CI runners benefit identically — no `actions/cache` wiring required, because the manifest travels with the deployed content rather than living on the runner's disk.
+
+**Encrypted deploys** (`--password`) bypass the incremental path. Encryption produces non-deterministic CAR bytes per run, so chunk-level dedup can't apply.
+
+**Force a full re-upload** by deleting `.bulletin-deploy/` from your build output (or changing one byte in any non-volatile file) before deploying. The classifier treats `.bulletin-deploy/` paths as volatile, so removing the manifest forces the next deploy to fall back to the heuristic classifier.
+
 ## Domain Rules
 
 DotNS classifies labels on-chain and may require a specific Proof of Personhood level before registration.

package/bin/bulletin-deploy

@@ -31,6 +31,9 @@ for (let i = 0; i < args.length; i++) {
   else if (args[i] === "--name") { flags.name = args[++i]; }
   else if (args[i] === "--description") { flags.description = args[++i]; }
   else if (args[i] === "--gh-pages-mirror") { flags.ghPagesMirror = true; }
+  else if (args[i] === "--allow-large-deploy") { flags.allowLargeDeploy = true; }
+  else if (args[i] === "--reproducible") { flags.reproducibleSource = "commit"; }
+  else if (args[i].startsWith("--reproducible=")) { flags.reproducibleSource = args[i].slice("--reproducible=".length); }
   else if (args[i] === "--version" || args[i] === "-V") { flags.version = true; }
   else if (args[i] === "--help" || args[i] === "-h") { flags.help = true; }
   else { positional.push(args[i]); }
@@ -255,6 +258,8 @@ try {
     ghPagesMirror: flags.ghPagesMirror,
     name: flags.name,
     description: flags.description,
+    allowLargeDeploy: flags.allowLargeDeploy,
+    reproducibleSource: flags.reproducibleSource,
   });
 
   const output = process.env.GITHUB_OUTPUT;

package/dist/bug-report.js

@@ -9,10 +9,10 @@ import {
   offerBugReport,
   scrubSecrets,
   setDeployContext
-} from "./chunk-3RNYFSU7.js";
-import "./chunk-6YJ46BN2.js";
-import "./chunk-4AP5ZFNV.js";
-import "./chunk-N7YE5ZN3.js";
+} from "./chunk-QEXYZ5SQ.js";
+import "./chunk-4KWVGBB2.js";
+import "./chunk-545X4HLT.js";
+import "./chunk-NZUB2BVL.js";
 import "./chunk-QGM4M3NI.js";
 export {
   buildCliFlagsSummary,

package/dist/{chunk-6YJ46BN2.js → chunk-4KWVGBB2.js}

@@ -1,6 +1,6 @@
 import {
   VERSION
-} from "./chunk-4AP5ZFNV.js";
+} from "./chunk-545X4HLT.js";
 
 // src/version-check.ts
 import { execSync, execFileSync } from "child_process";

package/dist/{chunk-4AP5ZFNV.js → chunk-545X4HLT.js}

@@ -1,7 +1,7 @@
 import {
   package_default,
   writeRunState
-} from "./chunk-N7YE5ZN3.js";
+} from "./chunk-NZUB2BVL.js";
 
 // src/memory-report.ts
 import * as fs2 from "fs";

package/dist/chunk-5MRZ3V4A.js

@@ -0,0 +1,171 @@
+import {
+  MANIFEST_DIR,
+  MANIFEST_FILENAME,
+  parseManifest
+} from "./chunk-S7EM5VMW.js";
+
+// src/manifest-fetch.ts
+import { CarReader } from "@ipld/car/reader";
+import * as dagPB from "@ipld/dag-pb";
+import { CID } from "multiformats/cid";
+var DEFAULT_GATEWAY = "https://paseo-ipfs.polkadot.io";
+var DEFAULT_TIMEOUT_MS = 5e3;
+var RANGE_TIERS = [
+  "bytes=0-4095",
+  "bytes=0-65535",
+  "bytes=0-1048575",
+  void 0
+  // full body
+];
+async function fetchAcrossTiers(url, budget, start) {
+  let lastReason = "unknown";
+  let attempts = 0;
+  let bytesDownloaded = 0;
+  for (let tier = 0; tier < RANGE_TIERS.length; tier++) {
+    if (Date.now() - start > budget) {
+      return { outcome: "retryable", reason: `budget exceeded: ${lastReason}`, attempts, bytesDownloaded };
+    }
+    attempts++;
+    const headers = {};
+    if (RANGE_TIERS[tier] !== void 0) headers.Range = RANGE_TIERS[tier];
+    let res;
+    try {
+      const ctrl = new AbortController();
+      const remaining = budget - (Date.now() - start);
+      const timer = setTimeout(() => ctrl.abort(), Math.max(100, remaining));
+      try {
+        res = await fetch(url, { headers, signal: ctrl.signal });
+      } finally {
+        clearTimeout(timer);
+      }
+    } catch (e) {
+      lastReason = `network error: ${e?.message ?? e}`;
+      continue;
+    }
+    if (res.status === 404) {
+      return { outcome: "404", attempts, bytesDownloaded };
+    }
+    if (res.status !== 200 && res.status !== 206) {
+      lastReason = `gateway HTTP ${res.status}`;
+      continue;
+    }
+    const isFullBody = res.status === 200;
+    let carBytes;
+    try {
+      const buf = await res.arrayBuffer();
+      carBytes = new Uint8Array(buf);
+      bytesDownloaded = carBytes.length;
+    } catch (e) {
+      if (isFullBody) {
+        return { outcome: "parse_error", reason: `body read error: ${e?.message ?? e}`, attempts, bytesDownloaded };
+      }
+      lastReason = `body read error: ${e?.message ?? e}`;
+      continue;
+    }
+    let manifestBytes;
+    try {
+      manifestBytes = await extractManifestFromCar(carBytes);
+    } catch (e) {
+      const msg = String(e?.message ?? e);
+      if (isFullBody) {
+        return { outcome: "parse_error", reason: `CAR parse error: ${msg}`, attempts, bytesDownloaded };
+      }
+      lastReason = `truncated at tier ${tier}: ${msg}`;
+      continue;
+    }
+    if (!manifestBytes) {
+      if (isFullBody) {
+        return { outcome: "parse_error", reason: "no .bulletin-deploy/manifest.json in deployed DAG", attempts, bytesDownloaded };
+      }
+      lastReason = `manifest not in slice tier ${tier}`;
+      continue;
+    }
+    const text = new TextDecoder().decode(manifestBytes);
+    const parsed = parseManifest(text);
+    if (parsed.ok) {
+      return { outcome: "success", manifest: parsed.manifest, attempts, bytesDownloaded };
+    }
+    return { outcome: "parse_error", reason: parsed.error, attempts, bytesDownloaded };
+  }
+  return { outcome: "retryable", reason: `tiers exhausted: ${lastReason}`, attempts, bytesDownloaded };
+}
+async function fetchPreviousManifest(prevContenthash, options = {}) {
+  if (prevContenthash === null) return { source: "none" };
+  const gatewayList = (options.gateways ?? (options.gateway ? [options.gateway] : [DEFAULT_GATEWAY])).map((g) => g.replace(/\/$/, ""));
+  const budget = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const start = Date.now();
+  let lastReason = "unknown";
+  let totalAttempts = 0;
+  let bytesDownloaded = 0;
+  for (const gateway of gatewayList) {
+    const url = `${gateway}/ipfs/${prevContenthash}`;
+    const tierResult = await fetchAcrossTiers(url, budget, start);
+    if (tierResult.outcome === "success") {
+      return { source: "embedded", manifest: tierResult.manifest, attempts: totalAttempts + tierResult.attempts, bytesDownloaded: bytesDownloaded + tierResult.bytesDownloaded };
+    }
+    if (tierResult.outcome === "404" || tierResult.outcome === "parse_error") {
+      return {
+        source: "heuristic_fallback",
+        reason: tierResult.outcome === "404" ? "gateway 404" : tierResult.reason,
+        attempts: totalAttempts + tierResult.attempts,
+        bytesDownloaded: bytesDownloaded + tierResult.bytesDownloaded
+      };
+    }
+    lastReason = tierResult.reason;
+    totalAttempts += tierResult.attempts;
+    bytesDownloaded += tierResult.bytesDownloaded;
+  }
+  return { source: "heuristic_fallback", reason: `all gateways exhausted: ${lastReason}`, attempts: totalAttempts, bytesDownloaded };
+}
+async function extractManifestFromCar(carBytes) {
+  const reader = await CarReader.fromBytes(carBytes);
+  const roots = await reader.getRoots();
+  if (roots.length === 0) return null;
+  const blocks = /* @__PURE__ */ new Map();
+  for await (const { cid, bytes } of reader.blocks()) {
+    blocks.set(cid.toString(), bytes);
+  }
+  const rootBytes = blocks.get(roots[0].toString());
+  if (!rootBytes) return null;
+  const rootNode = dagPB.decode(rootBytes);
+  const bdLink = (rootNode.Links ?? []).find((l) => l.Name === MANIFEST_DIR);
+  if (!bdLink) return null;
+  const bdBytes = blocks.get(bdLink.Hash.toString());
+  if (!bdBytes) return null;
+  const bdNode = dagPB.decode(bdBytes);
+  const manLink = (bdNode.Links ?? []).find((l) => l.Name === MANIFEST_FILENAME);
+  if (!manLink) return null;
+  const manCidStr = manLink.Hash.toString();
+  const manCid = CID.parse(manCidStr);
+  const manBytes = blocks.get(manCidStr);
+  if (!manBytes) return null;
+  if (manCid.code === 85) {
+    return manBytes;
+  }
+  if (manCid.code === 112) {
+    const node = dagPB.decode(manBytes);
+    const parts = [];
+    let total = 0;
+    for (const link of node.Links ?? []) {
+      const leafBytes = blocks.get(link.Hash.toString());
+      if (!leafBytes) return null;
+      parts.push(leafBytes);
+      total += leafBytes.length;
+    }
+    const out = new Uint8Array(total);
+    let pos = 0;
+    for (const part of parts) {
+      out.set(part, pos);
+      pos += part.length;
+    }
+    return out;
+  }
+  return null;
+}
+
+export {
+  DEFAULT_GATEWAY,
+  DEFAULT_TIMEOUT_MS,
+  fetchPreviousManifest,
+  extractManifestFromCar
+};

package/dist/{chunk-LM5U3XDV.js → chunk-5SODMVIF.js}

@@ -5,7 +5,7 @@ import {
   captureWarning,
   setDeployAttribute,
   withSpan
-} from "./chunk-4AP5ZFNV.js";
+} from "./chunk-545X4HLT.js";
 
 // src/dotns.ts
 import { spawn } from "child_process";

package/dist/chunk-C2TS5MER.js

@@ -0,0 +1,64 @@
+// src/chunker.ts
+var CHUNK_SIZE_TARGET = 1024 * 1024;
+var CHUNK_SIZE_MAX = 2 * 1024 * 1024 - 1024;
+function concat(parts) {
+  let total = 0;
+  for (const p of parts) total += p.length;
+  const out = new Uint8Array(total);
+  let off = 0;
+  for (const p of parts) {
+    out.set(p, off);
+    off += p.length;
+  }
+  return out;
+}
+function packSection(files) {
+  const chunks = [];
+  let buffer = [];
+  let bufferLen = 0;
+  const flush = () => {
+    if (bufferLen === 0) return;
+    chunks.push(concat(buffer));
+    buffer = [];
+    bufferLen = 0;
+  };
+  for (const file of files) {
+    const fileBytes = file.blocks.reduce((s, b) => s + b.length, 0);
+    if (fileBytes === 0) continue;
+    if (fileBytes > CHUNK_SIZE_TARGET) {
+      flush();
+      if (fileBytes > CHUNK_SIZE_MAX) {
+        for (const block of file.blocks) {
+          if (bufferLen + block.length > CHUNK_SIZE_MAX) {
+            flush();
+          }
+          buffer.push(block);
+          bufferLen += block.length;
+        }
+        flush();
+      } else {
+        for (const block of file.blocks) {
+          buffer.push(block);
+          bufferLen += block.length;
+          flush();
+        }
+      }
+    } else {
+      if (bufferLen + fileBytes > CHUNK_SIZE_TARGET) {
+        flush();
+      }
+      for (const block of file.blocks) {
+        buffer.push(block);
+        bufferLen += block.length;
+      }
+    }
+  }
+  flush();
+  return chunks;
+}
+
+export {
+  CHUNK_SIZE_TARGET,
+  CHUNK_SIZE_MAX,
+  packSection
+};

package/dist/chunk-KOSF5FDO.js

@@ -0,0 +1,49 @@
+import {
+  MANIFEST_DIR,
+  MANIFEST_PATH
+} from "./chunk-S7EM5VMW.js";
+
+// src/manifest-embed.ts
+import * as fs from "fs";
+import * as path from "path";
+function writeAtomic(filePath, body) {
+  const tmp = `${filePath}.tmp`;
+  fs.writeFileSync(tmp, body);
+  fs.renameSync(tmp, filePath);
+}
+function ensureDir(dirPath) {
+  fs.mkdirSync(dirPath, { recursive: true });
+}
+function writeEmbeddedManifestPlaceholder(buildDir, data) {
+  ensureDir(path.join(buildDir, MANIFEST_DIR));
+  const payload = {
+    version: data.version,
+    previous_contenthash: data.previousContenthash,
+    deployed_at: data.deployedAt,
+    framework: data.framework,
+    files: {},
+    stableBlockOrder: [],
+    blocks: [],
+    chunks: {}
+  };
+  writeAtomic(path.join(buildDir, MANIFEST_PATH), JSON.stringify(payload, null, 2));
+}
+function finaliseEmbeddedManifest(buildDir, data) {
+  ensureDir(path.join(buildDir, MANIFEST_DIR));
+  const payload = {
+    version: data.version,
+    previous_contenthash: data.previousContenthash,
+    deployed_at: data.deployedAt,
+    framework: data.framework,
+    files: data.files,
+    stableBlockOrder: data.stableBlockOrder,
+    blocks: data.blocks,
+    chunks: data.chunks
+  };
+  writeAtomic(path.join(buildDir, MANIFEST_PATH), JSON.stringify(payload, null, 2));
+}
+
+export {
+  writeEmbeddedManifestPlaceholder,
+  finaliseEmbeddedManifest
+};

package/dist/chunk-MJTQOXBC.js

@@ -0,0 +1,140 @@
+// src/incremental-stats.ts
+var SECONDS_PER_PROBE_SKIP = 3.5;
+function countByReason(probe, reason) {
+  return probe.filter((r) => r.present === null && r.failureReason === reason).length;
+}
+function computeStats(input) {
+  const present = input.probeResults.filter((r) => r.present === true);
+  const failed = input.probeResults.filter((r) => r.present === null);
+  const presentInPrev = present.filter((r) => input.prevChunks[r.cid] != null);
+  const recycled = present.filter((r) => input.prevChunks[r.cid] == null);
+  return {
+    manifestSource: input.manifestSource,
+    manifestFetchAttempts: input.manifestFetchAttempts,
+    manifestBytes: input.manifestBytes ?? 0,
+    framework: input.framework,
+    filesTotal: input.filesTotal,
+    filesStable: input.filesStable,
+    filesVolatile: input.filesVolatile,
+    probedTotal: input.probeResults.length,
+    probePresent: present.length,
+    probeAbsent: input.probeResults.filter((r) => r.present === false).length,
+    probeFailed: failed.length,
+    probeFailedRpc: countByReason(input.probeResults, "rpc_error"),
+    probeFailedDecode: countByReason(input.probeResults, "decode_error"),
+    probeFailedMetadata: countByReason(input.probeResults, "metadata_error"),
+    recycledCids: recycled.length,
+    retentionPeriodBlocks: input.retentionPeriodBlocks,
+    bytesSkipped: input.bytesSkipped,
+    bytesUploaded: input.bytesUploaded,
+    chunksTotal: input.chunksTotal,
+    chunksUploaded: input.chunksUploaded,
+    chunksSkipped: input.chunksSkipped,
+    carBytes: input.carBytes,
+    section0Bytes: input.sectionSizes.section0,
+    section1Bytes: input.sectionSizes.section1,
+    section2Bytes: input.sectionSizes.section2,
+    estimatedSecondsSaved: Math.round(SECONDS_PER_PROBE_SKIP * presentInPrev.length),
+    tier2VerifiedCount: input.tier2VerifiedCount,
+    tier2InconclusiveCount: input.tier2InconclusiveCount,
+    tier2FallbackCount: input.tier2FallbackCount
+  };
+}
+function telemetryAttributes(s) {
+  const hitRate = s.filesTotal === 0 ? 0 : s.filesStable / s.filesTotal;
+  return {
+    "deploy.cache.manifest_source": s.manifestSource,
+    "deploy.cache.manifest_fetch_attempts": String(s.manifestFetchAttempts),
+    "deploy.cache.manifest_bytes": String(s.manifestBytes),
+    "deploy.cache.framework": s.framework ?? "",
+    "deploy.cache.hit_rate": String(Math.round(hitRate * 1e3) / 1e3),
+    "deploy.cache.files_total": String(s.filesTotal),
+    "deploy.cache.files_stable": String(s.filesStable),
+    "deploy.cache.files_volatile": String(s.filesVolatile),
+    "deploy.cache.probed_total": String(s.probedTotal),
+    "deploy.cache.probe_present": String(s.probePresent),
+    "deploy.cache.probe_absent": String(s.probeAbsent),
+    "deploy.cache.probe_failed": String(s.probeFailed),
+    "deploy.cache.probe_failed_rpc": String(s.probeFailedRpc),
+    "deploy.cache.probe_failed_decode": String(s.probeFailedDecode),
+    "deploy.cache.probe_failed_metadata": String(s.probeFailedMetadata),
+    "deploy.cache.recycled_cids": String(s.recycledCids),
+    "deploy.cache.retention_period_blocks": String(s.retentionPeriodBlocks),
+    "deploy.cache.chunks_total": String(s.chunksTotal),
+    "deploy.cache.chunks_uploaded": String(s.chunksUploaded),
+    "deploy.cache.chunks_skipped": String(s.chunksSkipped),
+    "deploy.cache.bytes_skipped": String(s.bytesSkipped),
+    "deploy.cache.bytes_uploaded": String(s.bytesUploaded),
+    "deploy.cache.car_bytes": String(s.carBytes),
+    "deploy.cache.section0_bytes": String(s.section0Bytes),
+    "deploy.cache.section1_bytes": String(s.section1Bytes),
+    "deploy.cache.section2_bytes": String(s.section2Bytes),
+    "deploy.cache.estimated_seconds_saved": String(s.estimatedSecondsSaved),
+    "deploy.cache.tier2_fallback": String(s.tier2FallbackCount),
+    "deploy.cache.tier2_verified": String(s.tier2VerifiedCount),
+    "deploy.cache.tier2_inconclusive": String(s.tier2InconclusiveCount)
+  };
+}
+function fmtMb(bytes) {
+  return (bytes / 1e6).toFixed(1);
+}
+function fmtKb(bytes) {
+  return (bytes / 1e3).toFixed(1);
+}
+function renderSummary(s) {
+  const lines = [];
+  if (s.manifestSource === "heuristic_fallback") {
+    lines.push(`  \u26A0 Previous manifest fetch failed after ${s.manifestFetchAttempts} attempts (gateway timeout).`);
+    lines.push(`    Using heuristic classification \u2014 hit rate may be lower this run.`);
+    lines.push(`    Subsequent deploys recover automatically.`);
+    lines.push("");
+  }
+  lines.push(`Cache:`);
+  if (s.manifestSource === "none") {
+    lines.push(`  Manifest:      first deploy (no previous manifest)`);
+  } else if (s.manifestSource === "embedded") {
+    const attemptsStr = `${s.manifestFetchAttempts} attempt${s.manifestFetchAttempts === 1 ? "" : "s"}`;
+    const sizeStr = s.manifestBytes > 0 ? `, ${fmtKb(s.manifestBytes)} KB Range hit` : "";
+    lines.push(`  Manifest:      embedded (${attemptsStr}${sizeStr})`);
+  } else {
+    lines.push(`  Manifest:      heuristic_fallback (${s.manifestFetchAttempts} attempts)`);
+  }
+  if (s.filesTotal > 0 && s.manifestSource !== "none") {
+    const pct = s.filesTotal === 0 ? 0 : Math.round(s.filesStable / s.filesTotal * 100);
+    lines.push(`  Files:         ${s.filesStable} unchanged, ${s.filesVolatile} changed (${pct} % stable)`);
+  }
+  if (s.probedTotal > 0) {
+    let probeFailedStr = "";
+    if (s.probeFailed > 0) {
+      const reasons = [];
+      if (s.probeFailedRpc > 0) reasons.push("rpc_error");
+      if (s.probeFailedDecode > 0) reasons.push("decode_error");
+      if (s.probeFailedMetadata > 0) reasons.push("metadata_error");
+      probeFailedStr = `, ${s.probeFailed} probe-failed (${reasons.join(", ")})`;
+    }
+    lines.push(`  Probed:        ${s.probedTotal} chunks  \u2192  ${s.probePresent} cached, ${s.probeAbsent} to upload${probeFailedStr}`);
+  }
+  if (s.recycledCids > 0) {
+    lines.push(`  Recycled:      ${s.recycledCids} CIDs found on-chain that weren't in the previous manifest`);
+  }
+  if (s.tier2FallbackCount > 0) {
+    const inconclusiveStr = s.tier2InconclusiveCount > 0 ? `, ${s.tier2InconclusiveCount} inconclusive` : "";
+    lines.push(`  Verify:        ${s.tier2VerifiedCount}/${s.tier2FallbackCount} via-fallback chunks confirmed on chain${inconclusiveStr}`);
+  }
+  lines.push(`  CAR sections:  manifest ${fmtKb(s.section0Bytes)} KB \xB7 stable ${fmtMb(s.section1Bytes)} MB \xB7 volatile ${fmtMb(s.section2Bytes)} MB`);
+  if (s.chunksUploaded > 0 && s.bytesSkipped > 0) {
+    lines.push(`  Upload:        ${fmtMb(s.bytesUploaded)} MB across ${s.chunksUploaded} chunks (vs ${fmtMb(s.carBytes)} MB if full deploy)`);
+  } else if (s.chunksUploaded > 0) {
+    lines.push(`  Upload:        ${fmtMb(s.bytesUploaded)} MB across ${s.chunksUploaded} chunks`);
+  }
+  if (s.estimatedSecondsSaved > 0 || s.bytesSkipped > 0) {
+    lines.push(`  Saved:         ~${s.estimatedSecondsSaved} s and ${fmtMb(s.bytesSkipped)} MB`);
+  }
+  return lines.join("\n");
+}
+
+export {
+  computeStats,
+  telemetryAttributes,
+  renderSummary
+};