@hiveku-apps/sync 0.3.6 → 0.5.0

package/src/pull.mjs

@@ -1,87 +1,118 @@
 /**
- * `hiveku-sync pull` — fetches memory/rules/skills via the MCP memory_list tool
- * and writes them as local .md files under the current folder.
+ * `hiveku-sync pull` — fetches the account's memory, rules, skills,
+ * shortcuts, specialists and profiles with ONE memory_list call and writes
+ * each entry as a local .md file, filed by its owner (knowledge-files.mjs):
+ *
+ *   rules/sales/<name>.md, skills/seo/<name>.md, memory/helpdesk/<name>.md, ...
+ *   <type folder>/shared/<name>.md for an entry no agent owns
+ *   <type folder>/<owner>/not-followed/<name>.md for an entry filed under an
+ *   owner that no agent reads (the Memory page hides it)
+ *
+ * The owner is the `owner` memory_list returns (the builder's one owner rule),
+ * or the same rule from memory-owner.mjs when an older builder does not return
+ * it. The chief of staff's own `_account:*` rows and Voice and pronunciation
+ * are written where the Memory page shows them, read-only.
+ *
+ * It also refreshes the folder layout and memory rules in CLAUDE.md, the part
+ * between hiveku-sync's markers (claude-md.mjs).
  *
  * The account memory (domains `account` and `account-suggestions`) is not a
  * knowledge entry: it is classified explicitly, never written under memory/,
  * and pulled once from account_memory_get into a read-only
  * hiveku-data/account/ACCOUNT_MEMORY.md (see account-memory.mjs).
  *
+ * A file an earlier pull wrote for an entry this pull files somewhere else
+ * (an older hiveku-sync wrote flat <type>/<name>.md files), or no longer
+ * mirrors, is moved to .hiveku/set-aside/knowledge/<time>/: never deleted.
+ * A file whose entry is gone from Hiveku stays where it is and is listed as
+ * removed. When the listing fails, nothing is moved or reported.
+ *
  * Safety: refuses to overwrite when the .env API key no longer matches the
  * manifest's recorded key preview (prevents pulling Account A's data into a
  * folder previously synced for Account B — the 2026-05-11 cross-tenant bug
  * class).
  */
 
-import { join } from 'node:path';
+import { existsSync } from 'node:fs';
+import { mkdir, readFile, rename } from 'node:fs/promises';
+import { dirname, join, resolve, sep } from 'node:path';
 
 import { HivekuMcpClient } from './mcp.mjs';
-import {
-  loadDotenv,
-  readEnv,
-  safeSlug,
-  writeFileAtomic,
-  readJson,
-  toFrontmatter,
-  extractDepartmentTag,
-  TYPE_TO_FOLDER,
-  SUPPORTED_TYPES,
-} from './util.mjs';
+import { loadDotenv, readEnv, writeFileAtomic, readJson, SUPPORTED_TYPES } from './util.mjs';
 import {
   ACCOUNT_MEMORY_FILE,
-  classifyMemoryEntry,
   ACCOUNT_MEMORY_TOOL,
   normalizeAccountMemory,
   renderAccountMemoryFile,
   resolveAccountId,
   writeReadOnly,
 } from './account-memory.mjs';
+import { planKnowledgeFiles } from './knowledge-files.mjs';
+import { refreshClaudeMd } from './claude-md.mjs';
 
 const MANIFEST_PATH = '.hiveku/manifest.json';
+export const KNOWLEDGE_SET_ASIDE_DIR = '.hiveku/set-aside/knowledge';
 
 function keyPreview(apiKey) {
   return apiKey.slice(0, 10);
 }
 
-async function fetchType(client, type) {
-  const result = await client.callToolJson('memory_list', { type });
+/** `rel` under `root`, or null when it would land outside it (a manifest path is data, not trusted). */
+function insidePath(root, rel) {
+  const base = resolve(root);
+  const abs = resolve(base, String(rel || '').replace(/^\/+/, ''));
+  return abs.startsWith(base + sep) ? abs : null;
+}
+
+/** Every account-level entry, of every type, in one call. */
+async function fetchAll(client) {
+  const result = await client.callToolJson('memory_list', {});
   if (!result?.data || !Array.isArray(result.data)) {
-    throw new Error(`memory_list(${type}) returned unexpected shape`);
+    throw new Error('memory_list returned unexpected shape');
   }
   return result.data;
 }
 
-function entryFilename(entry) {
-  const name = entry.name || entry.domain.replace(/^_[^:]+:/, '');
-  return `${safeSlug(name)}.md`;
-}
-
-function renderEntry(entry, type) {
-  const body = entry.content ?? '';
-  const department = extractDepartmentTag(body);
-  const frontmatter = toFrontmatter({
-    id: entry.id,
-    name: entry.name,
-    type,
-    domain: entry.domain,
-    department,
-    project_id: entry.project_id,
-    version: entry.version,
-    updated_at: entry.updated_at,
-  });
-  return `${frontmatter}${body}`;
-}
-
 /**
  * Write hiveku-data/account/ACCOUNT_MEMORY.md from account_memory_get, read-only.
  * Returns { file, version, suggestions, accountId }, or throws (the caller
- * records the error and keeps the previous copy).
+ * records the error and keeps the previous copy). Pass `accountId` when it is
+ * already known (null included) to skip looking it up again.
  */
-export async function pullAccountMemory({ client, baseDir, fetchedAt = new Date().toISOString() }) {
+export async function pullAccountMemory({ client, baseDir, fetchedAt = new Date().toISOString(), accountId }) {
   const memory = normalizeAccountMemory(await client.callToolJson(ACCOUNT_MEMORY_TOOL, {}));
-  const accountId = await resolveAccountId(client);
-  await writeReadOnly(join(baseDir, ACCOUNT_MEMORY_FILE), renderAccountMemoryFile({ memory, accountId, fetchedAt }));
-  return { file: ACCOUNT_MEMORY_FILE, version: memory.version, suggestions: memory.suggestions.length, accountId };
+  const id = accountId === undefined ? await resolveAccountId(client) : accountId;
+  await writeReadOnly(join(baseDir, ACCOUNT_MEMORY_FILE), renderAccountMemoryFile({ memory, accountId: id, fetchedAt }));
+  return { file: ACCOUNT_MEMORY_FILE, version: memory.version, suggestions: memory.suggestions.length, accountId: id };
+}
+
+/** The `domain` an earlier pull wrote into a file's front matter, or null. */
+async function domainInFile(path) {
+  try {
+    const text = await readFile(path, 'utf8');
+    const fm = text.match(/^---\n([\s\S]*?)\n---\n/);
+    const line = fm?.[1].match(/^domain: "((?:[^"\\]|\\.)*)"$/m);
+    return line ? line[1].replace(/\\"/g, '"') : null;
+  } catch {
+    return null;
+  }
+}
+
+/** Move files an earlier pull wrote out of the tree, into one timestamped folder. Returns the moved paths. */
+async function setAside(baseDir, relPaths) {
+  if (relPaths.length === 0) return { moved: [], dir: null };
+  const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const dir = join(baseDir, KNOWLEDGE_SET_ASIDE_DIR, stamp);
+  const moved = [];
+  for (const rel of relPaths) {
+    const from = insidePath(baseDir, rel);
+    const to = insidePath(dir, rel);
+    if (!from || !to || !existsSync(from)) continue;
+    await mkdir(dirname(to), { recursive: true });
+    await rename(from, to);
+    moved.push(rel);
+  }
+  return { moved, dir: moved.length ? dir : null };
 }
 
 export async function runPull({ baseDir = process.cwd(), profile, client: injectedClient } = {}) {
@@ -117,35 +148,70 @@ export async function runPull({ baseDir = process.cwd(), profile, client: inject
   const client = injectedClient || new HivekuMcpClient({ baseUrl, apiKey, profile: activeProfile });
   if (!injectedClient) await client.initialize();
 
-  const summary = { total: 0, by_type: {}, errors: [], account_memory: null, account_rows_skipped: 0 };
+  const summary = {
+    total: 0,
+    by_type: {},
+    // Entries an agent reads (or every agent shares), by owner folder.
+    by_owner: {},
+    // Entries filed under <type>/<owner>/not-followed/: no agent reads them.
+    not_followed: 0,
+    errors: [],
+    account_memory: null,
+    account_rows_skipped: 0,
+    not_mirrored: { system: 0, hidden: 0 },
+    set_aside: [],
+    set_aside_dir: null,
+    claude_md: null,
+  };
   const writtenFiles = new Set();
+  const fileDomains = {};
+  const readOnlyFiles = new Set();
+  const previousFiles = Array.isArray(manifest?.files) ? manifest.files : [];
+  const previousDomains = manifest?.file_domains && typeof manifest.file_domains === 'object' ? manifest.file_domains : {};
+
+  // The account id gives the read-only copies their Memory page link.
+  const accountId = await resolveAccountId(client);
+
+  let plan = null;
+  try {
+    plan = planKnowledgeFiles(await fetchAll(client), { accountId });
+  } catch (err) {
+    summary.errors.push({ type: 'memory_list', message: err.message });
+  }
 
-  for (const type of SUPPORTED_TYPES) {
-    const folder = TYPE_TO_FOLDER[type];
-    let entries;
-    try {
-      entries = await fetchType(client, type);
-    } catch (err) {
-      summary.errors.push({ type, message: err.message });
-      continue;
+  if (plan) {
+    for (const type of SUPPORTED_TYPES) summary.by_type[type] = 0;
+    summary.account_rows_skipped = plan.skipped.account;
+    summary.not_mirrored = { system: plan.skipped.system, hidden: plan.skipped.hidden };
+    for (const file of plan.files) {
+      const abs = join(baseDir, file.rel);
+      if (file.readOnly) {
+        await writeReadOnly(abs, file.body);
+        readOnlyFiles.add(file.rel);
+      } else {
+        await writeFileAtomic(abs, file.body);
+      }
+      writtenFiles.add(file.rel);
+      fileDomains[file.rel] = file.domain;
+      summary.by_type[file.type] = (summary.by_type[file.type] ?? 0) + 1;
+      if (file.notFollowed) {
+        summary.not_followed += 1;
+      } else {
+        const owner = file.owner ?? 'shared';
+        summary.by_owner[owner] = (summary.by_owner[owner] ?? 0) + 1;
+      }
+      summary.total += 1;
     }
-    // The two account-memory domains are not knowledge entries: an older
-    // server still lists them, and they would land as memory/account.md and
-    // memory/account-suggestions.md (raw JSON Lines) looking editable. They
-    // are written once, read-only, from account_memory_get below.
-    const knowledge = [];
-    for (const entry of entries) {
-      if (classifyMemoryEntry(entry, folder).kind === 'knowledge') knowledge.push(entry);
-      else summary.account_rows_skipped += 1;
+  } else {
+    // Nothing was listed, so nothing can be called moved or removed: keep
+    // what the last pull recorded, as it was.
+    for (const rel of previousFiles) {
+      if (rel === ACCOUNT_MEMORY_FILE) continue;
+      writtenFiles.add(rel);
+      if (typeof previousDomains[rel] === 'string') fileDomains[rel] = previousDomains[rel];
     }
-    summary.by_type[type] = knowledge.length;
-    summary.total += knowledge.length;
-
-    for (const entry of knowledge) {
-      const filename = entryFilename(entry);
-      const fullPath = join(baseDir, folder, filename);
-      await writeFileAtomic(fullPath, renderEntry(entry, type));
-      writtenFiles.add(`${folder}/${filename}`);
+    for (const rel of Array.isArray(manifest?.read_only) ? manifest.read_only : []) {
+      if (rel !== ACCOUNT_MEMORY_FILE && writtenFiles.has(rel)) readOnlyFiles.add(rel);
     }
   }
 
@@ -153,18 +219,37 @@ export async function runPull({ baseDir = process.cwd(), profile, client: inject
   // A failure keeps the previous copy (and its manifest line), so it is not
   // reported as removed upstream.
   try {
-    summary.account_memory = await pullAccountMemory({ client, baseDir });
+    summary.account_memory = await pullAccountMemory({ client, baseDir, accountId });
     writtenFiles.add(ACCOUNT_MEMORY_FILE);
   } catch (err) {
     summary.errors.push({ type: 'account memory', message: err.message });
-    if ((manifest?.files ?? []).includes(ACCOUNT_MEMORY_FILE)) writtenFiles.add(ACCOUNT_MEMORY_FILE);
+    if (previousFiles.includes(ACCOUNT_MEMORY_FILE)) writtenFiles.add(ACCOUNT_MEMORY_FILE);
   }
 
-  // v1: don't delete stale files; just record them in the manifest so the
-  // user can prune manually if they want. Avoids losing local edits.
-  const previouslySeen = new Set(manifest?.files ?? []);
-  const removedRemotely = [...previouslySeen].filter((f) => !writtenFiles.has(f));
+  // Files the last pull wrote that this one did not. When Hiveku still lists
+  // the entry, its copy is now somewhere else (or no longer mirrored): the
+  // old copy is set aside. Otherwise the entry is gone upstream: the file is
+  // left on disk and reported, never deleted.
+  const notWritten = previousFiles.filter((rel) => !writtenFiles.has(rel));
+  const outOfPlace = [];
+  const removedRemotely = [];
+  for (const rel of notWritten) {
+    const path = insidePath(baseDir, rel);
+    // Already gone from disk (you deleted it): nothing to move, nothing to prune.
+    if (!path || !existsSync(path)) continue;
+    const domain = typeof previousDomains[rel] === 'string' ? previousDomains[rel] : await domainInFile(path);
+    if (plan && domain && plan.listed.has(domain)) outOfPlace.push(rel);
+    else removedRemotely.push(rel);
+  }
+  const tidy = await setAside(baseDir, outOfPlace);
+  summary.set_aside = tidy.moved;
+  summary.set_aside_dir = tidy.dir;
 
+  const sortedDomains = Object.fromEntries(Object.entries(fileDomains).sort(([a], [b]) => (a < b ? -1 : 1)));
+  // Which entry each file holds, when this pull (or the last) knew it. A failed
+  // listing over a folder an older release pulled knows none: leave it out,
+  // so readers (the Codex rules digest) keep reading the files on disk.
+  const knowsDomains = Boolean(plan) || (manifest?.file_domains && typeof manifest.file_domains === 'object');
   await writeFileAtomic(
     manifestPath,
     JSON.stringify(
@@ -173,10 +258,13 @@ export async function runPull({ baseDir = process.cwd(), profile, client: inject
         base_url: baseUrl,
         profile: activeProfile,
         last_sync_at: new Date().toISOString(),
+        // Filed by owner: <type folder>/<owner>/<name>.md (shared/ for no owner).
+        layout: 'by-owner',
         counts: summary.by_type,
         files: [...writtenFiles].sort(),
-        // Pulled for reading only: edited on the dashboard, never uploaded.
-        read_only: [ACCOUNT_MEMORY_FILE],
+        ...(knowsDomains ? { file_domains: sortedDomains } : {}),
+        // Pulled for reading only: changed on the Memory page, never uploaded.
+        read_only: [ACCOUNT_MEMORY_FILE, ...[...readOnlyFiles].sort()],
         removed_remotely_since_last_sync: removedRemotely,
       },
       null,
@@ -184,5 +272,21 @@ export async function runPull({ baseDir = process.cwd(), profile, client: inject
     ),
   );
 
-  return { ...summary, removed_remotely: removedRemotely, profile: activeProfile, baseUrl };
+  // The folder layout and memory rules in CLAUDE.md, which only `init` wrote
+  // before: refreshed, so a folder an older release made stops teaching the
+  // old ones (claude-md.mjs). A failure here is reported, not a failed pull.
+  try {
+    summary.claude_md = await refreshClaudeMd(baseDir);
+  } catch (err) {
+    summary.claude_md = { status: 'failed', message: err.message };
+  }
+
+  let codexLane = false;
+  try {
+    codexLane = (await readFile(join(baseDir, 'AGENTS.md'), 'utf8')).includes('<!-- hiveku:begin -->');
+  } catch {
+    /* no Codex lane */
+  }
+
+  return { ...summary, removed_remotely: removedRemotely, profile: activeProfile, baseUrl, codex_lane: codexLane };
 }

package/src/pullRecord.mjs

@@ -0,0 +1,98 @@
+/**
+ * What pull-project wrote into a project folder, so the next pull can take
+ * away what the new tree no longer has.
+ *
+ * A pull writes every file of Your site (or of a branch) into
+ * projects/<slug>/ and never removed anything. Once `--branch` downloads the
+ * branch's real tree, switching a folder between Your site and a branch left
+ * the other tree's files behind, and the next push (which targets the pulled
+ * branch) sent them as new files and saved them as a version: a branch-only
+ * page ended up on Your site. A file deleted on Hiveku since the last pull
+ * came back the same way.
+ *
+ * So each pull records the paths it wrote (.hiveku/pulled/<slug>.json in the
+ * account folder), and the next pull moves the recorded files the new tree
+ * lacks out of the project folder, into .hiveku/set-aside/<slug>/<time>/ in
+ * the account folder: never deleted, and outside everything push reads. A
+ * file the pull did not write (your own new file) is never moved; the pull
+ * lists it instead, because a push would add it. The run hints hiveku-sync
+ * used to write as RUN.md in the project folder (they were pushed into the
+ * site) now live in the project folder's .hiveku/RUN.md; an old generated
+ * RUN.md is set aside like a stale file.
+ */
+
+import { existsSync } from 'node:fs';
+import { mkdir, readFile, rename } from 'node:fs/promises';
+import { dirname, join, relative, sep } from 'node:path';
+
+import { readJson, writeFileAtomic } from './util.mjs';
+import { localPathFor } from './pullTree.mjs';
+
+const PULLED_DIR = '.hiveku/pulled';
+const SET_ASIDE_DIR = '.hiveku/set-aside';
+/** Where pull-project writes its run hints inside the project folder (never pushed: push skips dot folders). */
+export const RUN_HINTS_FILE = '.hiveku/RUN.md';
+
+/** A project path as the local walk spells it (no leading slash). */
+export function recordPath(projectPath) {
+  return String(projectPath || '').replace(/^\/+/, '');
+}
+
+/** The paths the last pull wrote into projects/<slug>/, or null when no pull recorded them (an older hiveku-sync). */
+export async function readPulledFiles(baseDir, slug) {
+  const data = await readJson(join(baseDir, PULLED_DIR, `${slug}.json`));
+  return data && Array.isArray(data.files) ? data : null;
+}
+
+export async function writePulledFiles(baseDir, slug, { branch, files }) {
+  const record = { branch: branch || null, pulled_at: new Date().toISOString(), files: [...new Set(files.map(recordPath))].sort() };
+  await writeFileAtomic(join(baseDir, PULLED_DIR, `${slug}.json`), JSON.stringify(record, null, 2));
+  return record;
+}
+
+/** Is this the RUN.md an older hiveku-sync generated for this project (not the site's own)? */
+async function isGeneratedRunMd(dest, projectId) {
+  try {
+    const text = await readFile(join(dest, 'RUN.md'), 'utf8');
+    return text.includes(`Hiveku project id: \`${projectId}\``) && /## Ship changes back/.test(text);
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * After a complete download: move the files the previous pull wrote that the
+ * new tree lacks out of the folder, and name the other local files the tree
+ * lacks. `localFiles` are the folder's files as push sees them (relative
+ * paths). Returns { setAside: [paths], setAsideDir, notOnTarget: [paths] }.
+ */
+export async function setAsideStaleFiles({ baseDir, slug, dest, projectId, previous, treePaths, localFiles }) {
+  const inTree = new Set([...treePaths].map(recordPath));
+  const wrotePreviously = new Set((previous?.files || []).map(recordPath));
+  const extra = localFiles.filter((p) => !inTree.has(p));
+  const stale = [];
+  const notOnTarget = [];
+  for (const path of extra) {
+    if (wrotePreviously.has(path) || (path === 'RUN.md' && (await isGeneratedRunMd(dest, projectId)))) stale.push(path);
+    else notOnTarget.push(path);
+  }
+  if (stale.length === 0) return { setAside: [], setAsideDir: null, notOnTarget };
+  const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const setAsideDir = join(baseDir, SET_ASIDE_DIR, slug, stamp);
+  const moved = [];
+  for (const path of stale) {
+    const from = localPathFor(dest, path);
+    const to = localPathFor(setAsideDir, path);
+    if (!from || !to || !existsSync(from)) continue;
+    await mkdir(dirname(to), { recursive: true });
+    await rename(from, to);
+    moved.push(path);
+  }
+  return { setAside: moved, setAsideDir: moved.length ? setAsideDir : null, notOnTarget };
+}
+
+/** A folder path for a person, relative to the account folder when it is inside it. */
+export function shownPath(baseDir, abs) {
+  const rel = relative(baseDir, abs);
+  return rel && !rel.startsWith('..') ? rel.split(sep).join('/') : abs;
+}

package/src/pullTree.mjs

@@ -0,0 +1,211 @@
+/**
+ * Download a project's files (Your site, or a branch's working tree) into a
+ * local folder, one page at a time, so a site of any size can be pulled.
+ *
+ * First choice: project_vcs_checkout with `limit` (and `cursor` after the
+ * first page). Paths come in byte order; a page also stops at 16 MB and a
+ * single bigger file comes alone, so no file is ever cut short. The answer
+ * carries `head_commit_id` and, on a branch, `working_tree_etag` (the tree the
+ * page came from: when it changes between pages someone saved on the branch
+ * mid-download, and the download starts over). An unpaged answer (no
+ * `next_cursor` key: a builder or MCP server from before paging) is the whole
+ * tree in one answer, which is complete too.
+ *
+ * Fallback: project_files_bulk_get paged by `cursor` (it never refuses a large
+ * site). Used when checkout cannot page here: a Your-site checkout over 150 MB
+ * answers 413 content_too_large to a caller whose `limit` was not forwarded,
+ * and an older server may not have paging at all. Files over its per-file cap
+ * come back empty with `truncated: true` and are fetched one by one with
+ * project_file_get.
+ *
+ * Builder contract: hiveku_builder feature/versions-core
+ * src/app/api/olympus/builder/projects/[projectId]/vcs/checkout/route.ts and
+ * .../files-bulk/route.ts.
+ */
+
+import { mkdir } from 'node:fs/promises';
+import { dirname, resolve, sep } from 'node:path';
+
+import { writeFileAtomic } from './util.mjs';
+import { branchLabel, readToolError } from './versions.mjs';
+
+export const CHECKOUT_PAGE_FILES = 500;
+const BULK_MAX_FILE_BYTES = 10 * 1024 * 1024;
+const BULK_MAX_TOTAL_BYTES = 32 * 1024 * 1024;
+const MAX_RESTARTS = 3;
+const BINARY_SENTINEL = '__BINARY_BASE64__';
+
+/** Answers that mean the project or branch is not there: another route will not help. */
+const FINAL_CODES = new Set(['branch_missing', 'invalid_branch', 'project_missing']);
+
+/** The bytes to write for one file of an answer, or null when it carries no content. */
+export function fileBytes(file) {
+  const content = file && (typeof file.content === 'string' ? file.content : typeof file.file_content === 'string' ? file.file_content : null);
+  if (content === null) return null;
+  if (content.startsWith(BINARY_SENTINEL)) return Buffer.from(content.slice(BINARY_SENTINEL.length), 'base64');
+  if (file.encoding === 'base64') return Buffer.from(content, 'base64');
+  return Buffer.from(content, 'utf8');
+}
+
+/** Where a project path lands under `dest`, or null when it would land outside it. */
+export function localPathFor(dest, projectPath) {
+  const rel = String(projectPath || '').replace(/^\/+/, '');
+  if (!rel) return null;
+  const root = resolve(dest);
+  const abs = resolve(root, rel);
+  return abs.startsWith(root + sep) ? abs : null;
+}
+
+class Writer {
+  constructor(dest) {
+    this.dest = dest;
+    this.written = new Set();
+    this.skipped = [];
+  }
+
+  async write(projectPath, bytes) {
+    const abs = localPathFor(this.dest, projectPath);
+    if (!abs || bytes === null) {
+      this.skipped.push(projectPath);
+      return;
+    }
+    await mkdir(dirname(abs), { recursive: true });
+    await writeFileAtomic(abs, bytes);
+    this.written.add(projectPath);
+  }
+}
+
+class BranchMovedError extends Error {}
+
+/** Said when Hiveku's answer cannot be used; a pull that stops leaves every local file in place. */
+const INCOMPLETE_LIST = 'Hiveku sent an incomplete file list. Nothing in this folder was removed; try the pull again.';
+
+class IncompleteListError extends Error {
+  constructor() {
+    super(INCOMPLETE_LIST);
+  }
+}
+
+function friendlyError(error, branch) {
+  if (error.code === 'branch_missing') {
+    return new Error(`There is no ${branchLabel(branch)} on this project. Leave out --branch to pull Your site.`);
+  }
+  if (error.code === 'invalid_branch') return new Error(`"${branch}" is not a valid branch name.`);
+  // A route's own error text is written for people; the client's raw
+  // "Tool <name> errored: {...}" wrapper is not.
+  if (error.text && !/^Tool \S+ errored:/.test(error.text)) return new Error(error.text);
+  return new Error(`Hiveku could not send the files${error.status ? ` (it answered ${error.status})` : ''}. Nothing in this folder was removed; try the pull again.`);
+}
+
+async function viaCheckout({ client, projectId, branch, writer, log }) {
+  let cursor = null;
+  let etag;
+  let head = null;
+  let total = null;
+  let page = 0;
+  for (;;) {
+    const args = { project_id: projectId, limit: CHECKOUT_PAGE_FILES };
+    if (branch) args.branch = branch;
+    if (cursor) args.cursor = cursor;
+    const data = (await client.callToolJson('project_vcs_checkout', args))?.data;
+    if (!data || !Array.isArray(data.files)) throw new IncompleteListError();
+    page += 1;
+    if (page === 1) {
+      etag = data.working_tree_etag ?? null;
+      head = data.head_commit_id ?? null;
+    } else if (branch && (data.working_tree_etag ?? null) !== etag) {
+      throw new BranchMovedError('the branch changed between pages');
+    }
+    for (const file of data.files) await writer.write(file.path, fileBytes(file));
+    if (typeof data.total_files === 'number') total = data.total_files;
+    const paged = Object.prototype.hasOwnProperty.call(data, 'next_cursor');
+    if (!paged || !data.next_cursor) break;
+    if (data.next_cursor === cursor) throw new IncompleteListError();
+    cursor = data.next_cursor;
+    log(`  files ${writer.written.size}${total !== null ? `/${total}` : ''}`);
+  }
+  return { head_commit_id: head, working_tree_etag: etag ?? null, lane: 'checkout' };
+}
+
+async function viaFilesBulk({ client, projectId, branch, writer, log }) {
+  let cursor = null;
+  let etag;
+  let page = 0;
+  const truncated = [];
+  for (;;) {
+    const args = {
+      project_id: projectId,
+      include_assets: true,
+      max_file_bytes: BULK_MAX_FILE_BYTES,
+      max_total_bytes: BULK_MAX_TOTAL_BYTES,
+    };
+    if (branch) args.branch = branch;
+    if (cursor) args.cursor = cursor;
+    const data = (await client.callToolJson('project_files_bulk_get', args))?.data;
+    if (!data || !Array.isArray(data.files)) throw new IncompleteListError();
+    page += 1;
+    if (page === 1) etag = data.working_tree_etag ?? null;
+    else if (branch && (data.working_tree_etag ?? null) !== etag) throw new BranchMovedError('the branch changed between pages');
+    for (const file of data.files) {
+      if (file.truncated) truncated.push(file.path);
+      else await writer.write(file.path, fileBytes(file));
+    }
+    if (!data.partial || !data.next_cursor) break;
+    if (data.next_cursor === cursor) throw new IncompleteListError();
+    cursor = data.next_cursor;
+    log(`  files ${writer.written.size}`);
+  }
+  // Files over the per-file cap, one by one. One that cannot be read (for
+  // example a stored copy that is gone) is reported as skipped; the rest of
+  // the pull goes on.
+  for (const path of truncated) {
+    if (writer.written.has(path)) continue;
+    const args = { project_id: projectId, file_path: path };
+    if (branch) args.branch = branch;
+    let body;
+    try {
+      body = await client.callToolJson('project_file_get', args);
+    } catch {
+      writer.skipped.push(path);
+      continue;
+    }
+    await writer.write(path, fileBytes(body?.data || body?.file || null));
+  }
+  return { head_commit_id: null, working_tree_etag: etag ?? null, lane: 'files-bulk' };
+}
+
+/**
+ * Pull every file of Your site (branch null) or of `branch` into `dest`.
+ * Returns { written, paths: [written paths], skipped: [paths], head_commit_id,
+ * working_tree_etag, lane }. Throws, having removed nothing, when the tree
+ * could not be read completely.
+ */
+export async function downloadProjectTree({ client, projectId, branch = null, dest, log = console.log }) {
+  let lane = viaCheckout;
+  for (let attempt = 0; ; attempt++) {
+    const writer = new Writer(dest);
+    try {
+      const result = await lane({ client, projectId, branch, writer, log });
+      return { ...result, written: writer.written.size, paths: [...writer.written], skipped: writer.skipped };
+    } catch (err) {
+      if (err instanceof BranchMovedError) {
+        if (attempt >= MAX_RESTARTS) {
+          throw new Error(`${branchLabel(branch)} kept changing while it was downloading. Try again in a moment.`);
+        }
+        log(`  ${branchLabel(branch)} changed while downloading; starting over.`);
+        continue;
+      }
+      if (err instanceof IncompleteListError && lane !== viaCheckout) throw err;
+      const error = readToolError(err);
+      if (FINAL_CODES.has(error.code) || error.status === 401 || error.status === 403) throw friendlyError(error, branch);
+      if (lane === viaCheckout) {
+        log(error.code === 'content_too_large'
+          ? '  This site is too large to send in one answer here; downloading it in pages instead.'
+          : '  Downloading through the file list instead.');
+        lane = viaFilesBulk;
+        continue;
+      }
+      throw friendlyError(error, branch);
+    }
+  }
+}