@hiveku-apps/sync 0.3.6 → 0.4.0

package/src/projects.mjs

@@ -1,15 +1,16 @@
 /**
  * `hiveku-sync` project commands — pull a Hiveku coder project into
- * ./projects/<slug>/, show diff vs remote, and commit edits back via MCP.
+ * ./projects/<slug>/, show diff vs remote, and (with `commit --github`) push
+ * edits to a connected GitHub branch via MCP.
  *
- * No native git clone. The MCP server handles GitHub commits via the
- * account's installed GitHub App — same code path as the v3 editor sidebar.
+ * No native git clone. Versions of Your site and of branches live in
+ * versions.mjs; the paged download pull-project uses lives in pullTree.mjs.
  */
 
 import { createHash } from 'node:crypto';
 import { existsSync } from 'node:fs';
-import { mkdir, readdir, readFile, stat } from 'node:fs/promises';
-import { dirname, join, relative, sep } from 'node:path';
+import { readdir, readFile, stat } from 'node:fs/promises';
+import { join, relative, sep } from 'node:path';
 
 import { HivekuMcpClient } from './mcp.mjs';
 import {
@@ -20,6 +21,9 @@ import {
   readJson,
 } from './util.mjs';
 import { isAccountMemoryPath } from './account-memory.mjs';
+import { downloadProjectTree } from './pullTree.mjs';
+import { readPulledFiles, RUN_HINTS_FILE, setAsideStaleFiles, shownPath, writePulledFiles } from './pullRecord.mjs';
+import { branchFlag, branchLabel } from './versions.mjs';
 // NOTE: push.mjs imports helpers from this module; the cycle is safe because
 // both sides reference the imports only inside functions (never at module eval).
 import { computePushPlan } from './push.mjs';
@@ -100,25 +104,6 @@ export async function clientFromCwd({ profileOverride } = {}) {
   return { client, baseDir, baseUrl, profile };
 }
 
-async function listAllRemoteFiles(client, projectId) {
-  const all = [];
-  let page = 1;
-  const limit = 500;
-  while (true) {
-    const result = await client.callToolJson('project_files_list', {
-      project_id: projectId,
-      page,
-      limit,
-    });
-    const batch = result?.data || [];
-    all.push(...batch);
-    const totalPages = result?.pagination?.total_pages ?? 1;
-    if (page >= totalPages || batch.length === 0) break;
-    page += 1;
-  }
-  return all;
-}
-
 /**
  * Every local file push, commit and status consider. The pulled account memory
  * (hiveku-data/account/) is left out wherever it sits: it is a read-only copy
@@ -135,8 +120,7 @@ export async function walkLocalFiles(rootDir) {
       return;
     }
     for (const entry of entries) {
-      if (entry.name.startsWith('.')) continue;
-      if (IGNORE_DIRS.has(entry.name)) continue;
+      if (walkSkipsName(entry.name)) continue;
       const full = join(dir, entry.name);
       if (isAccountMemoryPath(relative(rootDir, full).split(sep).join('/'))) continue;
       if (entry.isDirectory()) {
@@ -150,6 +134,24 @@ export async function walkLocalFiles(rootDir) {
   return out;
 }
 
+/** A file or folder name walkLocalFiles never enters: dot entries and build/tool folders. */
+function walkSkipsName(name) {
+  return name.startsWith('.') || IGNORE_DIRS.has(name);
+}
+
+/**
+ * Could walkLocalFiles ever list this project path? A path it cannot list
+ * (a dot file such as .gitignore, anything under node_modules/ or .next/, the
+ * pulled account memory) always reads as missing locally, so it must never be
+ * treated as deleted here.
+ */
+export function localWalkCanList(projectPath) {
+  const rel = String(projectPath || '').replace(/^\/+/, '');
+  if (!rel) return false;
+  if (rel.split('/').some((segment) => !segment || walkSkipsName(segment))) return false;
+  return !isAccountMemoryPath(rel);
+}
+
 async function buildLocalManifest(rootDir) {
   const files = await walkLocalFiles(rootDir);
   const manifest = [];
@@ -376,6 +378,8 @@ export async function runProjectsList({ profile, client: injectedClient } = {})
 
 export async function runPullProject({ slugOrId, profile, branch, client: injectedClient, baseDir: baseDirOverride }) {
   if (!slugOrId) throw new Error('Provide a project slug or UUID');
+  // null = Your site (main); a bare --branch with no name is refused.
+  const pullBranch = branchFlag(branch) ?? null;
   const { client, baseDir } = await resolveClient({ client: injectedClient, profile, baseDir: baseDirOverride });
   const project = await resolveProject(client, slugOrId);
   // An external site has no file tree: the pull would write nothing and stamp
@@ -387,47 +391,45 @@ export async function runPullProject({ slugOrId, profile, branch, client: inject
 
   console.log(`Pulling ${project.name || project.slug || projectId} (type=${project.project_type || '?'})`);
   console.log(`  Project ID: ${projectId}`);
+  console.log(`  ${pullBranch ? `Branch: ${pullBranch}` : 'Your site'}`);
   console.log(`  Destination: ${dest}`);
 
-  const files = await listAllRemoteFiles(client, projectId);
-  if (files.length === 0) {
+  // What the previous pull wrote here, so files the new tree lacks can be
+  // taken out of the folder (pullRecord.mjs).
+  const previous = await readPulledFiles(baseDir, localSlug);
+
+  // Paged, so a site of any size comes down (a site over 150 MB cannot be
+  // sent in one answer). See pullTree.mjs.
+  const tree = await downloadProjectTree({ client, projectId, branch: pullBranch, dest });
+  const written = tree.written;
+  const skipped = tree.skipped.length;
+  if (written === 0 && skipped === 0) {
     console.log('  No files to pull (project is empty).');
   }
 
-  let written = 0;
-  let skipped = 0;
-  for (const fileMeta of files) {
-    const result = await client.callToolJson('project_file_get', {
-      project_id: projectId,
-      file_path: fileMeta.file_path,
-    });
-    const fileBody = result?.file || result?.data?.file || result?.data;
-    if (!fileBody || typeof fileBody.file_content !== 'string') {
-      skipped += 1;
-      continue;
-    }
-    let contentBuf;
-    if (fileBody.file_content.startsWith('__BINARY_BASE64__')) {
-      contentBuf = Buffer.from(
-        fileBody.file_content.substring('__BINARY_BASE64__'.length),
-        'base64',
-      );
-    } else {
-      contentBuf = Buffer.from(fileBody.file_content, 'utf8');
-    }
-    const outPath = join(dest, fileMeta.file_path);
-    await mkdir(dirname(outPath), { recursive: true });
-    await writeFileAtomic(outPath, contentBuf);
-    written += 1;
-  }
+  // The download is complete: move out what the previous pull wrote and this
+  // tree lacks (another branch's files, files deleted on Hiveku), so a push
+  // cannot send them back.
+  const localFiles = (await walkLocalFiles(dest)).map((abs) => relative(dest, abs).split(sep).join('/'));
+  const tidy = await setAsideStaleFiles({
+    baseDir,
+    slug: localSlug,
+    dest,
+    projectId,
+    previous,
+    treePaths: [...(tree.paths || []), ...tree.skipped],
+    localFiles,
+  });
+  await writePulledFiles(baseDir, localSlug, { branch: pullBranch, files: tree.paths || [] });
 
-  // Drop a RUN.md keyed off project_type so the user knows how to start it.
+  // Run hints keyed off project_type, inside the folder's .hiveku/ so a push
+  // never sends them into the site.
   const runMd = [
     `# ${project.name || project.slug || projectId}`,
     '',
     `Project type: \`${project.project_type || 'unknown'}\``,
     `Hiveku project id: \`${projectId}\``,
-    branch ? `Tracking branch: \`${branch}\`` : '',
+    pullBranch ? `Tracking branch: \`${pullBranch}\`` : '',
     '',
     '## Run locally',
     '',
@@ -439,15 +441,18 @@ export async function runPullProject({ slugOrId, profile, branch, client: inject
     '',
     "This folder is NOT a git clone — it's a snapshot. `push` writes your edits to the",
     'DEPLOY SOURCE (what `deploy` builds from), lane-routing images/fonts/video to the',
-    'S3/CDN asset store and everything else to versioned code files.',
+    'S3/CDN asset store and everything else to the project files, then saves them as',
+    `one version of ${pullBranch ? `branch \`${pullBranch}\`` : 'Your site'}, named in plain language for the site owner.`,
     '',
     'From this folder:',
     '',
     '```bash',
     'hiveku-sync pull-env                          # project secrets → .env.local',
     'hiveku-sync status                            # diff local vs the deploy source',
-    'hiveku-sync push                              # save changes back',
+    'hiveku-sync push -m "Updated the pricing section on the Home page"',
     'hiveku-sync push --deploy development --wait  # ...then deploy and wait',
+    'hiveku-sync versions                          # the version history',
+    'hiveku-sync rollback <version id>             # what going back would change (dry run)',
     '```',
     '',
     'Or ask Claude in this workspace to push/deploy for you.',
@@ -455,7 +460,7 @@ export async function runPullProject({ slugOrId, profile, branch, client: inject
   ]
     .filter((line) => line !== undefined)
     .join('\n');
-  await writeFileAtomic(join(dest, 'RUN.md'), runMd);
+  await writeFileAtomic(join(dest, RUN_HINTS_FILE), runMd);
 
   // Persist project metadata so status/push know which project this folder maps to.
   const manifest = await readProjectsManifest(baseDir);
@@ -465,14 +470,46 @@ export async function runPullProject({ slugOrId, profile, branch, client: inject
     name: project.name || project.slug || localSlug,
     project_type: project.project_type || null,
     external_platform: project.external_platform || null,
-    branch: branch || null,
+    branch: pullBranch,
+    // What the pull came from: the version it was at, and on a branch the
+    // working-tree fingerprint. push compares it with the one
+    // project_files_status reports, to see whether someone else saved on the
+    // branch since (pushGuards.mjs), and records the tree its own push left.
+    head_commit_id: tree.head_commit_id ?? null,
+    working_tree_etag: tree.working_tree_etag ?? null,
     last_pull_at: new Date().toISOString(),
   };
   await writeProjectsManifest(baseDir, manifest);
 
-  console.log(`  Wrote ${written} file${written !== 1 ? 's' : ''}${skipped ? ` (${skipped} skipped — no content)` : ''}`);
-  console.log(`  Run hints in ${join(dest, 'RUN.md')}`);
-  return { dest, written, skipped, projectId, slug: localSlug };
+  console.log(`  Wrote ${written} file${written !== 1 ? 's' : ''}${skipped ? ` (${skipped} skipped: no content or an unsafe path)` : ''}`);
+  if (tidy.setAside.length) {
+    const n = tidy.setAside.length;
+    console.log(
+      `  Moved ${n} file${n !== 1 ? 's' : ''} that ${n !== 1 ? 'are' : 'is'} not on ${branchLabel(pullBranch)} out of this folder ` +
+        `(an earlier pull wrote ${n !== 1 ? 'them' : 'it'}): they are in ${shownPath(baseDir, tidy.setAsideDir)}/`,
+    );
+  }
+  if (tidy.notOnTarget.length) {
+    const n = tidy.notOnTarget.length;
+    console.log(
+      `  ${n} file${n !== 1 ? 's' : ''} here ${n !== 1 ? 'are' : 'is'} not on ${branchLabel(pullBranch)} and a push would add ` +
+        `${n !== 1 ? 'them' : 'it'}: ${tidy.notOnTarget.slice(0, 10).join(', ')}${n > 10 ? ` and ${n - 10} more` : ''}. ` +
+        'Keep the ones that are your own new work; move the others out of the folder before you push.',
+    );
+  }
+  console.log(`  Run hints in ${join(dest, RUN_HINTS_FILE)}`);
+  return {
+    dest,
+    written,
+    skipped,
+    projectId,
+    slug: localSlug,
+    branch: pullBranch,
+    head_commit_id: tree.head_commit_id ?? null,
+    working_tree_etag: tree.working_tree_etag ?? null,
+    set_aside: tidy.setAside,
+    not_on_target: tidy.notOnTarget,
+  };
 }
 
 export async function findProjectFolder({ slug, baseDir }) {
@@ -507,7 +544,7 @@ function summarizeDiff(diff) {
   };
 }
 
-export async function runProjectStatus({ slug, profile, json = false, client: injectedClient, baseDir: baseDirOverride }) {
+export async function runProjectStatus({ slug, profile, json = false, branch, client: injectedClient, baseDir: baseDirOverride }) {
   // Resolve baseDir as the account folder. If we're inside ./projects/<x>/,
   // walk up two levels. Otherwise treat cwd as the account folder.
   const cwd = process.cwd();
@@ -525,12 +562,16 @@ export async function runProjectStatus({ slug, profile, json = false, client: in
 
   // Deploy-source-accurate + asset-lane-aware — the SAME diff `push` runs, so
   // status never flags unchanged images as only_local (code goes through
-  // project_files_status; assets through assets_list).
-  const { diff, assetAll, assetToUpload, assetSkipped, plan } = await computePushPlan({
+  // project_files_status; assets through assets_list). A folder pulled from a
+  // branch diffs against that branch's working tree.
+  const flagged = branchFlag(branch);
+  const statusBranch = flagged !== undefined ? flagged : (info.branch && info.branch !== 'main' ? info.branch : null);
+  const { diff, remoteOnly, notMirrored, assetAll, assetToUpload, assetSkipped, plan } = await computePushPlan({
     client,
     projectId: info.project_id,
     folder,
     includeDeletes: true,
+    branch: statusBranch,
   });
 
   if (json) {
@@ -540,6 +581,11 @@ export async function runProjectStatus({ slug, profile, json = false, client: in
           slug: resolvedSlug,
           project_id: info.project_id,
           code: diff,
+          // only_remote split: what push --include-deletes would delete, and
+          // what this folder never holds (dot files, build folders, files
+          // Hiveku manages), which is never a deletion.
+          remote_only_deletable: remoteOnly,
+          remote_only_not_mirrored: notMirrored,
           assets: {
             to_upload: assetToUpload.map((f) => f.path),
             unchanged: assetSkipped,
@@ -556,10 +602,14 @@ export async function runProjectStatus({ slug, profile, json = false, client: in
 
   const changed = diff.changed?.length || 0;
   const onlyLocal = diff.only_local?.length || 0;
-  const onlyRemote = diff.only_remote?.length || 0;
+  const onlyRemote = remoteOnly.length;
   const unchanged = diff.same_count || 0;
   console.log(`Project: ${resolvedSlug}  (${info.project_id})`);
-  console.log('Compared against the DEPLOY SOURCE (builder_code_versions + asset store).\n');
+  console.log(
+    statusBranch
+      ? `Compared against branch "${statusBranch}" (its working tree) and the shared image store.\n`
+      : 'Compared against Your site (what a deploy ships) and the shared image store.\n',
+  );
   console.log('Code:');
   console.log(`  changed:     ${changed}`);
   console.log(`  added:       ${onlyLocal}`);
@@ -579,10 +629,17 @@ export async function runProjectStatus({ slug, profile, json = false, client: in
   printList(diff.changed, 'M', 'Changed code:');
   printList(diff.only_local, '+', 'Added code (will be created on push):');
   printList(
-    diff.only_remote,
+    remoteOnly,
     '-',
     'Remote-only code (deleted locally or never pulled — push --include-deletes to delete):',
+    (path) => path,
   );
+  if (notMirrored.length) {
+    console.log(
+      `\n${notMirrored.length} more file${notMirrored.length !== 1 ? 's are' : ' is'} only on Hiveku because this folder never holds ` +
+        'them (hidden files such as .gitignore, build folders, files Hiveku manages). A push never deletes them.',
+    );
+  }
   printList(assetToUpload, 'A', 'Assets to upload (new or changed):');
   return { diff, plan };
 }

package/src/pullRecord.mjs

@@ -0,0 +1,98 @@
+/**
+ * What pull-project wrote into a project folder, so the next pull can take
+ * away what the new tree no longer has.
+ *
+ * A pull writes every file of Your site (or of a branch) into
+ * projects/<slug>/ and never removed anything. Once `--branch` downloads the
+ * branch's real tree, switching a folder between Your site and a branch left
+ * the other tree's files behind, and the next push (which targets the pulled
+ * branch) sent them as new files and saved them as a version: a branch-only
+ * page ended up on Your site. A file deleted on Hiveku since the last pull
+ * came back the same way.
+ *
+ * So each pull records the paths it wrote (.hiveku/pulled/<slug>.json in the
+ * account folder), and the next pull moves the recorded files the new tree
+ * lacks out of the project folder, into .hiveku/set-aside/<slug>/<time>/ in
+ * the account folder: never deleted, and outside everything push reads. A
+ * file the pull did not write (your own new file) is never moved; the pull
+ * lists it instead, because a push would add it. The run hints hiveku-sync
+ * used to write as RUN.md in the project folder (they were pushed into the
+ * site) now live in the project folder's .hiveku/RUN.md; an old generated
+ * RUN.md is set aside like a stale file.
+ */
+
+import { existsSync } from 'node:fs';
+import { mkdir, readFile, rename } from 'node:fs/promises';
+import { dirname, join, relative, sep } from 'node:path';
+
+import { readJson, writeFileAtomic } from './util.mjs';
+import { localPathFor } from './pullTree.mjs';
+
+const PULLED_DIR = '.hiveku/pulled';
+const SET_ASIDE_DIR = '.hiveku/set-aside';
+/** Where pull-project writes its run hints inside the project folder (never pushed: push skips dot folders). */
+export const RUN_HINTS_FILE = '.hiveku/RUN.md';
+
+/** A project path as the local walk spells it (no leading slash). */
+export function recordPath(projectPath) {
+  return String(projectPath || '').replace(/^\/+/, '');
+}
+
+/** The paths the last pull wrote into projects/<slug>/, or null when no pull recorded them (an older hiveku-sync). */
+export async function readPulledFiles(baseDir, slug) {
+  const data = await readJson(join(baseDir, PULLED_DIR, `${slug}.json`));
+  return data && Array.isArray(data.files) ? data : null;
+}
+
+export async function writePulledFiles(baseDir, slug, { branch, files }) {
+  const record = { branch: branch || null, pulled_at: new Date().toISOString(), files: [...new Set(files.map(recordPath))].sort() };
+  await writeFileAtomic(join(baseDir, PULLED_DIR, `${slug}.json`), JSON.stringify(record, null, 2));
+  return record;
+}
+
+/** Is this the RUN.md an older hiveku-sync generated for this project (not the site's own)? */
+async function isGeneratedRunMd(dest, projectId) {
+  try {
+    const text = await readFile(join(dest, 'RUN.md'), 'utf8');
+    return text.includes(`Hiveku project id: \`${projectId}\``) && /## Ship changes back/.test(text);
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * After a complete download: move the files the previous pull wrote that the
+ * new tree lacks out of the folder, and name the other local files the tree
+ * lacks. `localFiles` are the folder's files as push sees them (relative
+ * paths). Returns { setAside: [paths], setAsideDir, notOnTarget: [paths] }.
+ */
+export async function setAsideStaleFiles({ baseDir, slug, dest, projectId, previous, treePaths, localFiles }) {
+  const inTree = new Set([...treePaths].map(recordPath));
+  const wrotePreviously = new Set((previous?.files || []).map(recordPath));
+  const extra = localFiles.filter((p) => !inTree.has(p));
+  const stale = [];
+  const notOnTarget = [];
+  for (const path of extra) {
+    if (wrotePreviously.has(path) || (path === 'RUN.md' && (await isGeneratedRunMd(dest, projectId)))) stale.push(path);
+    else notOnTarget.push(path);
+  }
+  if (stale.length === 0) return { setAside: [], setAsideDir: null, notOnTarget };
+  const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const setAsideDir = join(baseDir, SET_ASIDE_DIR, slug, stamp);
+  const moved = [];
+  for (const path of stale) {
+    const from = localPathFor(dest, path);
+    const to = localPathFor(setAsideDir, path);
+    if (!from || !to || !existsSync(from)) continue;
+    await mkdir(dirname(to), { recursive: true });
+    await rename(from, to);
+    moved.push(path);
+  }
+  return { setAside: moved, setAsideDir: moved.length ? setAsideDir : null, notOnTarget };
+}
+
+/** A folder path for a person, relative to the account folder when it is inside it. */
+export function shownPath(baseDir, abs) {
+  const rel = relative(baseDir, abs);
+  return rel && !rel.startsWith('..') ? rel.split(sep).join('/') : abs;
+}

package/src/pullTree.mjs

@@ -0,0 +1,211 @@
+/**
+ * Download a project's files (Your site, or a branch's working tree) into a
+ * local folder, one page at a time, so a site of any size can be pulled.
+ *
+ * First choice: project_vcs_checkout with `limit` (and `cursor` after the
+ * first page). Paths come in byte order; a page also stops at 16 MB and a
+ * single bigger file comes alone, so no file is ever cut short. The answer
+ * carries `head_commit_id` and, on a branch, `working_tree_etag` (the tree the
+ * page came from: when it changes between pages someone saved on the branch
+ * mid-download, and the download starts over). An unpaged answer (no
+ * `next_cursor` key: a builder or MCP server from before paging) is the whole
+ * tree in one answer, which is complete too.
+ *
+ * Fallback: project_files_bulk_get paged by `cursor` (it never refuses a large
+ * site). Used when checkout cannot page here: a Your-site checkout over 150 MB
+ * answers 413 content_too_large to a caller whose `limit` was not forwarded,
+ * and an older server may not have paging at all. Files over its per-file cap
+ * come back empty with `truncated: true` and are fetched one by one with
+ * project_file_get.
+ *
+ * Builder contract: hiveku_builder feature/versions-core
+ * src/app/api/olympus/builder/projects/[projectId]/vcs/checkout/route.ts and
+ * .../files-bulk/route.ts.
+ */
+
+import { mkdir } from 'node:fs/promises';
+import { dirname, resolve, sep } from 'node:path';
+
+import { writeFileAtomic } from './util.mjs';
+import { branchLabel, readToolError } from './versions.mjs';
+
+export const CHECKOUT_PAGE_FILES = 500;
+const BULK_MAX_FILE_BYTES = 10 * 1024 * 1024;
+const BULK_MAX_TOTAL_BYTES = 32 * 1024 * 1024;
+const MAX_RESTARTS = 3;
+const BINARY_SENTINEL = '__BINARY_BASE64__';
+
+/** Answers that mean the project or branch is not there: another route will not help. */
+const FINAL_CODES = new Set(['branch_missing', 'invalid_branch', 'project_missing']);
+
+/** The bytes to write for one file of an answer, or null when it carries no content. */
+export function fileBytes(file) {
+  const content = file && (typeof file.content === 'string' ? file.content : typeof file.file_content === 'string' ? file.file_content : null);
+  if (content === null) return null;
+  if (content.startsWith(BINARY_SENTINEL)) return Buffer.from(content.slice(BINARY_SENTINEL.length), 'base64');
+  if (file.encoding === 'base64') return Buffer.from(content, 'base64');
+  return Buffer.from(content, 'utf8');
+}
+
+/** Where a project path lands under `dest`, or null when it would land outside it. */
+export function localPathFor(dest, projectPath) {
+  const rel = String(projectPath || '').replace(/^\/+/, '');
+  if (!rel) return null;
+  const root = resolve(dest);
+  const abs = resolve(root, rel);
+  return abs.startsWith(root + sep) ? abs : null;
+}
+
+class Writer {
+  constructor(dest) {
+    this.dest = dest;
+    this.written = new Set();
+    this.skipped = [];
+  }
+
+  async write(projectPath, bytes) {
+    const abs = localPathFor(this.dest, projectPath);
+    if (!abs || bytes === null) {
+      this.skipped.push(projectPath);
+      return;
+    }
+    await mkdir(dirname(abs), { recursive: true });
+    await writeFileAtomic(abs, bytes);
+    this.written.add(projectPath);
+  }
+}
+
+class BranchMovedError extends Error {}
+
+/** Said when Hiveku's answer cannot be used; a pull that stops leaves every local file in place. */
+const INCOMPLETE_LIST = 'Hiveku sent an incomplete file list. Nothing in this folder was removed; try the pull again.';
+
+class IncompleteListError extends Error {
+  constructor() {
+    super(INCOMPLETE_LIST);
+  }
+}
+
+function friendlyError(error, branch) {
+  if (error.code === 'branch_missing') {
+    return new Error(`There is no ${branchLabel(branch)} on this project. Leave out --branch to pull Your site.`);
+  }
+  if (error.code === 'invalid_branch') return new Error(`"${branch}" is not a valid branch name.`);
+  // A route's own error text is written for people; the client's raw
+  // "Tool <name> errored: {...}" wrapper is not.
+  if (error.text && !/^Tool \S+ errored:/.test(error.text)) return new Error(error.text);
+  return new Error(`Hiveku could not send the files${error.status ? ` (it answered ${error.status})` : ''}. Nothing in this folder was removed; try the pull again.`);
+}
+
+async function viaCheckout({ client, projectId, branch, writer, log }) {
+  let cursor = null;
+  let etag;
+  let head = null;
+  let total = null;
+  let page = 0;
+  for (;;) {
+    const args = { project_id: projectId, limit: CHECKOUT_PAGE_FILES };
+    if (branch) args.branch = branch;
+    if (cursor) args.cursor = cursor;
+    const data = (await client.callToolJson('project_vcs_checkout', args))?.data;
+    if (!data || !Array.isArray(data.files)) throw new IncompleteListError();
+    page += 1;
+    if (page === 1) {
+      etag = data.working_tree_etag ?? null;
+      head = data.head_commit_id ?? null;
+    } else if (branch && (data.working_tree_etag ?? null) !== etag) {
+      throw new BranchMovedError('the branch changed between pages');
+    }
+    for (const file of data.files) await writer.write(file.path, fileBytes(file));
+    if (typeof data.total_files === 'number') total = data.total_files;
+    const paged = Object.prototype.hasOwnProperty.call(data, 'next_cursor');
+    if (!paged || !data.next_cursor) break;
+    if (data.next_cursor === cursor) throw new IncompleteListError();
+    cursor = data.next_cursor;
+    log(`  files ${writer.written.size}${total !== null ? `/${total}` : ''}`);
+  }
+  return { head_commit_id: head, working_tree_etag: etag ?? null, lane: 'checkout' };
+}
+
+async function viaFilesBulk({ client, projectId, branch, writer, log }) {
+  let cursor = null;
+  let etag;
+  let page = 0;
+  const truncated = [];
+  for (;;) {
+    const args = {
+      project_id: projectId,
+      include_assets: true,
+      max_file_bytes: BULK_MAX_FILE_BYTES,
+      max_total_bytes: BULK_MAX_TOTAL_BYTES,
+    };
+    if (branch) args.branch = branch;
+    if (cursor) args.cursor = cursor;
+    const data = (await client.callToolJson('project_files_bulk_get', args))?.data;
+    if (!data || !Array.isArray(data.files)) throw new IncompleteListError();
+    page += 1;
+    if (page === 1) etag = data.working_tree_etag ?? null;
+    else if (branch && (data.working_tree_etag ?? null) !== etag) throw new BranchMovedError('the branch changed between pages');
+    for (const file of data.files) {
+      if (file.truncated) truncated.push(file.path);
+      else await writer.write(file.path, fileBytes(file));
+    }
+    if (!data.partial || !data.next_cursor) break;
+    if (data.next_cursor === cursor) throw new IncompleteListError();
+    cursor = data.next_cursor;
+    log(`  files ${writer.written.size}`);
+  }
+  // Files over the per-file cap, one by one. One that cannot be read (for
+  // example a stored copy that is gone) is reported as skipped; the rest of
+  // the pull goes on.
+  for (const path of truncated) {
+    if (writer.written.has(path)) continue;
+    const args = { project_id: projectId, file_path: path };
+    if (branch) args.branch = branch;
+    let body;
+    try {
+      body = await client.callToolJson('project_file_get', args);
+    } catch {
+      writer.skipped.push(path);
+      continue;
+    }
+    await writer.write(path, fileBytes(body?.data || body?.file || null));
+  }
+  return { head_commit_id: null, working_tree_etag: etag ?? null, lane: 'files-bulk' };
+}
+
+/**
+ * Pull every file of Your site (branch null) or of `branch` into `dest`.
+ * Returns { written, paths: [written paths], skipped: [paths], head_commit_id,
+ * working_tree_etag, lane }. Throws, having removed nothing, when the tree
+ * could not be read completely.
+ */
+export async function downloadProjectTree({ client, projectId, branch = null, dest, log = console.log }) {
+  let lane = viaCheckout;
+  for (let attempt = 0; ; attempt++) {
+    const writer = new Writer(dest);
+    try {
+      const result = await lane({ client, projectId, branch, writer, log });
+      return { ...result, written: writer.written.size, paths: [...writer.written], skipped: writer.skipped };
+    } catch (err) {
+      if (err instanceof BranchMovedError) {
+        if (attempt >= MAX_RESTARTS) {
+          throw new Error(`${branchLabel(branch)} kept changing while it was downloading. Try again in a moment.`);
+        }
+        log(`  ${branchLabel(branch)} changed while downloading; starting over.`);
+        continue;
+      }
+      if (err instanceof IncompleteListError && lane !== viaCheckout) throw err;
+      const error = readToolError(err);
+      if (FINAL_CODES.has(error.code) || error.status === 401 || error.status === 403) throw friendlyError(error, branch);
+      if (lane === viaCheckout) {
+        log(error.code === 'content_too_large'
+          ? '  This site is too large to send in one answer here; downloading it in pages instead.'
+          : '  Downloading through the file list instead.');
+        lane = viaFilesBulk;
+        continue;
+      }
+      throw friendlyError(error, branch);
+    }
+  }
+}