@hiveku-apps/sync 0.3.5 → 0.4.0

package/src/pullTree.mjs

@@ -0,0 +1,211 @@
+/**
+ * Download a project's files (Your site, or a branch's working tree) into a
+ * local folder, one page at a time, so a site of any size can be pulled.
+ *
+ * First choice: project_vcs_checkout with `limit` (and `cursor` after the
+ * first page). Paths come in byte order; a page also stops at 16 MB and a
+ * single bigger file comes alone, so no file is ever cut short. The answer
+ * carries `head_commit_id` and, on a branch, `working_tree_etag` (the tree the
+ * page came from: when it changes between pages someone saved on the branch
+ * mid-download, and the download starts over). An unpaged answer (no
+ * `next_cursor` key: a builder or MCP server from before paging) is the whole
+ * tree in one answer, which is complete too.
+ *
+ * Fallback: project_files_bulk_get paged by `cursor` (it never refuses a large
+ * site). Used when checkout cannot page here: a Your-site checkout over 150 MB
+ * answers 413 content_too_large to a caller whose `limit` was not forwarded,
+ * and an older server may not have paging at all. Files over its per-file cap
+ * come back empty with `truncated: true` and are fetched one by one with
+ * project_file_get.
+ *
+ * Builder contract: hiveku_builder feature/versions-core
+ * src/app/api/olympus/builder/projects/[projectId]/vcs/checkout/route.ts and
+ * .../files-bulk/route.ts.
+ */
+
+import { mkdir } from 'node:fs/promises';
+import { dirname, resolve, sep } from 'node:path';
+
+import { writeFileAtomic } from './util.mjs';
+import { branchLabel, readToolError } from './versions.mjs';
+
+export const CHECKOUT_PAGE_FILES = 500;
+const BULK_MAX_FILE_BYTES = 10 * 1024 * 1024;
+const BULK_MAX_TOTAL_BYTES = 32 * 1024 * 1024;
+const MAX_RESTARTS = 3;
+const BINARY_SENTINEL = '__BINARY_BASE64__';
+
+/** Answers that mean the project or branch is not there: another route will not help. */
+const FINAL_CODES = new Set(['branch_missing', 'invalid_branch', 'project_missing']);
+
+/** The bytes to write for one file of an answer, or null when it carries no content. */
+export function fileBytes(file) {
+  const content = file && (typeof file.content === 'string' ? file.content : typeof file.file_content === 'string' ? file.file_content : null);
+  if (content === null) return null;
+  if (content.startsWith(BINARY_SENTINEL)) return Buffer.from(content.slice(BINARY_SENTINEL.length), 'base64');
+  if (file.encoding === 'base64') return Buffer.from(content, 'base64');
+  return Buffer.from(content, 'utf8');
+}
+
+/** Where a project path lands under `dest`, or null when it would land outside it. */
+export function localPathFor(dest, projectPath) {
+  const rel = String(projectPath || '').replace(/^\/+/, '');
+  if (!rel) return null;
+  const root = resolve(dest);
+  const abs = resolve(root, rel);
+  return abs.startsWith(root + sep) ? abs : null;
+}
+
+class Writer {
+  constructor(dest) {
+    this.dest = dest;
+    this.written = new Set();
+    this.skipped = [];
+  }
+
+  async write(projectPath, bytes) {
+    const abs = localPathFor(this.dest, projectPath);
+    if (!abs || bytes === null) {
+      this.skipped.push(projectPath);
+      return;
+    }
+    await mkdir(dirname(abs), { recursive: true });
+    await writeFileAtomic(abs, bytes);
+    this.written.add(projectPath);
+  }
+}
+
+class BranchMovedError extends Error {}
+
+/** Said when Hiveku's answer cannot be used; a pull that stops leaves every local file in place. */
+const INCOMPLETE_LIST = 'Hiveku sent an incomplete file list. Nothing in this folder was removed; try the pull again.';
+
+class IncompleteListError extends Error {
+  constructor() {
+    super(INCOMPLETE_LIST);
+  }
+}
+
+function friendlyError(error, branch) {
+  if (error.code === 'branch_missing') {
+    return new Error(`There is no ${branchLabel(branch)} on this project. Leave out --branch to pull Your site.`);
+  }
+  if (error.code === 'invalid_branch') return new Error(`"${branch}" is not a valid branch name.`);
+  // A route's own error text is written for people; the client's raw
+  // "Tool <name> errored: {...}" wrapper is not.
+  if (error.text && !/^Tool \S+ errored:/.test(error.text)) return new Error(error.text);
+  return new Error(`Hiveku could not send the files${error.status ? ` (it answered ${error.status})` : ''}. Nothing in this folder was removed; try the pull again.`);
+}
+
+async function viaCheckout({ client, projectId, branch, writer, log }) {
+  let cursor = null;
+  let etag;
+  let head = null;
+  let total = null;
+  let page = 0;
+  for (;;) {
+    const args = { project_id: projectId, limit: CHECKOUT_PAGE_FILES };
+    if (branch) args.branch = branch;
+    if (cursor) args.cursor = cursor;
+    const data = (await client.callToolJson('project_vcs_checkout', args))?.data;
+    if (!data || !Array.isArray(data.files)) throw new IncompleteListError();
+    page += 1;
+    if (page === 1) {
+      etag = data.working_tree_etag ?? null;
+      head = data.head_commit_id ?? null;
+    } else if (branch && (data.working_tree_etag ?? null) !== etag) {
+      throw new BranchMovedError('the branch changed between pages');
+    }
+    for (const file of data.files) await writer.write(file.path, fileBytes(file));
+    if (typeof data.total_files === 'number') total = data.total_files;
+    const paged = Object.prototype.hasOwnProperty.call(data, 'next_cursor');
+    if (!paged || !data.next_cursor) break;
+    if (data.next_cursor === cursor) throw new IncompleteListError();
+    cursor = data.next_cursor;
+    log(`  files ${writer.written.size}${total !== null ? `/${total}` : ''}`);
+  }
+  return { head_commit_id: head, working_tree_etag: etag ?? null, lane: 'checkout' };
+}
+
+async function viaFilesBulk({ client, projectId, branch, writer, log }) {
+  let cursor = null;
+  let etag;
+  let page = 0;
+  const truncated = [];
+  for (;;) {
+    const args = {
+      project_id: projectId,
+      include_assets: true,
+      max_file_bytes: BULK_MAX_FILE_BYTES,
+      max_total_bytes: BULK_MAX_TOTAL_BYTES,
+    };
+    if (branch) args.branch = branch;
+    if (cursor) args.cursor = cursor;
+    const data = (await client.callToolJson('project_files_bulk_get', args))?.data;
+    if (!data || !Array.isArray(data.files)) throw new IncompleteListError();
+    page += 1;
+    if (page === 1) etag = data.working_tree_etag ?? null;
+    else if (branch && (data.working_tree_etag ?? null) !== etag) throw new BranchMovedError('the branch changed between pages');
+    for (const file of data.files) {
+      if (file.truncated) truncated.push(file.path);
+      else await writer.write(file.path, fileBytes(file));
+    }
+    if (!data.partial || !data.next_cursor) break;
+    if (data.next_cursor === cursor) throw new IncompleteListError();
+    cursor = data.next_cursor;
+    log(`  files ${writer.written.size}`);
+  }
+  // Files over the per-file cap, one by one. One that cannot be read (for
+  // example a stored copy that is gone) is reported as skipped; the rest of
+  // the pull goes on.
+  for (const path of truncated) {
+    if (writer.written.has(path)) continue;
+    const args = { project_id: projectId, file_path: path };
+    if (branch) args.branch = branch;
+    let body;
+    try {
+      body = await client.callToolJson('project_file_get', args);
+    } catch {
+      writer.skipped.push(path);
+      continue;
+    }
+    await writer.write(path, fileBytes(body?.data || body?.file || null));
+  }
+  return { head_commit_id: null, working_tree_etag: etag ?? null, lane: 'files-bulk' };
+}
+
+/**
+ * Pull every file of Your site (branch null) or of `branch` into `dest`.
+ * Returns { written, paths: [written paths], skipped: [paths], head_commit_id,
+ * working_tree_etag, lane }. Throws, having removed nothing, when the tree
+ * could not be read completely.
+ */
+export async function downloadProjectTree({ client, projectId, branch = null, dest, log = console.log }) {
+  let lane = viaCheckout;
+  for (let attempt = 0; ; attempt++) {
+    const writer = new Writer(dest);
+    try {
+      const result = await lane({ client, projectId, branch, writer, log });
+      return { ...result, written: writer.written.size, paths: [...writer.written], skipped: writer.skipped };
+    } catch (err) {
+      if (err instanceof BranchMovedError) {
+        if (attempt >= MAX_RESTARTS) {
+          throw new Error(`${branchLabel(branch)} kept changing while it was downloading. Try again in a moment.`);
+        }
+        log(`  ${branchLabel(branch)} changed while downloading; starting over.`);
+        continue;
+      }
+      if (err instanceof IncompleteListError && lane !== viaCheckout) throw err;
+      const error = readToolError(err);
+      if (FINAL_CODES.has(error.code) || error.status === 401 || error.status === 403) throw friendlyError(error, branch);
+      if (lane === viaCheckout) {
+        log(error.code === 'content_too_large'
+          ? '  This site is too large to send in one answer here; downloading it in pages instead.'
+          : '  Downloading through the file list instead.');
+        lane = viaFilesBulk;
+        continue;
+      }
+      throw friendlyError(error, branch);
+    }
+  }
+}