@hiveku-apps/sync 0.3.6 → 0.5.0

package/src/projects.mjs

@@ -1,15 +1,16 @@
 /**
  * `hiveku-sync` project commands — pull a Hiveku coder project into
- * ./projects/<slug>/, show diff vs remote, and commit edits back via MCP.
+ * ./projects/<slug>/, show diff vs remote, and (with `commit --github`) push
+ * edits to a connected GitHub branch via MCP.
  *
- * No native git clone. The MCP server handles GitHub commits via the
- * account's installed GitHub App — same code path as the v3 editor sidebar.
+ * No native git clone. Versions of Your site and of branches live in
+ * versions.mjs; the paged download pull-project uses lives in pullTree.mjs.
  */
 
 import { createHash } from 'node:crypto';
 import { existsSync } from 'node:fs';
-import { mkdir, readdir, readFile, stat } from 'node:fs/promises';
-import { dirname, join, relative, sep } from 'node:path';
+import { readdir, readFile, stat } from 'node:fs/promises';
+import { join, relative, sep } from 'node:path';
 
 import { HivekuMcpClient } from './mcp.mjs';
 import {
@@ -20,6 +21,9 @@ import {
   readJson,
 } from './util.mjs';
 import { isAccountMemoryPath } from './account-memory.mjs';
+import { downloadProjectTree } from './pullTree.mjs';
+import { readPulledFiles, RUN_HINTS_FILE, setAsideStaleFiles, shownPath, writePulledFiles } from './pullRecord.mjs';
+import { branchFlag, branchLabel } from './versions.mjs';
 // NOTE: push.mjs imports helpers from this module; the cycle is safe because
 // both sides reference the imports only inside functions (never at module eval).
 import { computePushPlan } from './push.mjs';
@@ -100,25 +104,6 @@ export async function clientFromCwd({ profileOverride } = {}) {
   return { client, baseDir, baseUrl, profile };
 }
 
-async function listAllRemoteFiles(client, projectId) {
-  const all = [];
-  let page = 1;
-  const limit = 500;
-  while (true) {
-    const result = await client.callToolJson('project_files_list', {
-      project_id: projectId,
-      page,
-      limit,
-    });
-    const batch = result?.data || [];
-    all.push(...batch);
-    const totalPages = result?.pagination?.total_pages ?? 1;
-    if (page >= totalPages || batch.length === 0) break;
-    page += 1;
-  }
-  return all;
-}
-
 /**
  * Every local file push, commit and status consider. The pulled account memory
  * (hiveku-data/account/) is left out wherever it sits: it is a read-only copy
@@ -135,8 +120,7 @@ export async function walkLocalFiles(rootDir) {
       return;
     }
     for (const entry of entries) {
-      if (entry.name.startsWith('.')) continue;
-      if (IGNORE_DIRS.has(entry.name)) continue;
+      if (walkSkipsName(entry.name)) continue;
       const full = join(dir, entry.name);
       if (isAccountMemoryPath(relative(rootDir, full).split(sep).join('/'))) continue;
       if (entry.isDirectory()) {
@@ -150,6 +134,24 @@ export async function walkLocalFiles(rootDir) {
   return out;
 }
 
+/** A file or folder name walkLocalFiles never enters: dot entries and build/tool folders. */
+function walkSkipsName(name) {
+  return name.startsWith('.') || IGNORE_DIRS.has(name);
+}
+
+/**
+ * Could walkLocalFiles ever list this project path? A path it cannot list
+ * (a dot file such as .gitignore, anything under node_modules/ or .next/, the
+ * pulled account memory) always reads as missing locally, so it must never be
+ * treated as deleted here.
+ */
+export function localWalkCanList(projectPath) {
+  const rel = String(projectPath || '').replace(/^\/+/, '');
+  if (!rel) return false;
+  if (rel.split('/').some((segment) => !segment || walkSkipsName(segment))) return false;
+  return !isAccountMemoryPath(rel);
+}
+
 async function buildLocalManifest(rootDir) {
   const files = await walkLocalFiles(rootDir);
   const manifest = [];
@@ -376,6 +378,8 @@ export async function runProjectsList({ profile, client: injectedClient } = {})
 
 export async function runPullProject({ slugOrId, profile, branch, client: injectedClient, baseDir: baseDirOverride }) {
   if (!slugOrId) throw new Error('Provide a project slug or UUID');
+  // null = Your site (main); a bare --branch with no name is refused.
+  const pullBranch = branchFlag(branch) ?? null;
   const { client, baseDir } = await resolveClient({ client: injectedClient, profile, baseDir: baseDirOverride });
   const project = await resolveProject(client, slugOrId);
   // An external site has no file tree: the pull would write nothing and stamp
@@ -387,47 +391,45 @@ export async function runPullProject({ slugOrId, profile, branch, client: inject
 
   console.log(`Pulling ${project.name || project.slug || projectId} (type=${project.project_type || '?'})`);
   console.log(`  Project ID: ${projectId}`);
+  console.log(`  ${pullBranch ? `Branch: ${pullBranch}` : 'Your site'}`);
   console.log(`  Destination: ${dest}`);
 
-  const files = await listAllRemoteFiles(client, projectId);
-  if (files.length === 0) {
+  // What the previous pull wrote here, so files the new tree lacks can be
+  // taken out of the folder (pullRecord.mjs).
+  const previous = await readPulledFiles(baseDir, localSlug);
+
+  // Paged, so a site of any size comes down (a site over 150 MB cannot be
+  // sent in one answer). See pullTree.mjs.
+  const tree = await downloadProjectTree({ client, projectId, branch: pullBranch, dest });
+  const written = tree.written;
+  const skipped = tree.skipped.length;
+  if (written === 0 && skipped === 0) {
     console.log('  No files to pull (project is empty).');
   }
 
-  let written = 0;
-  let skipped = 0;
-  for (const fileMeta of files) {
-    const result = await client.callToolJson('project_file_get', {
-      project_id: projectId,
-      file_path: fileMeta.file_path,
-    });
-    const fileBody = result?.file || result?.data?.file || result?.data;
-    if (!fileBody || typeof fileBody.file_content !== 'string') {
-      skipped += 1;
-      continue;
-    }
-    let contentBuf;
-    if (fileBody.file_content.startsWith('__BINARY_BASE64__')) {
-      contentBuf = Buffer.from(
-        fileBody.file_content.substring('__BINARY_BASE64__'.length),
-        'base64',
-      );
-    } else {
-      contentBuf = Buffer.from(fileBody.file_content, 'utf8');
-    }
-    const outPath = join(dest, fileMeta.file_path);
-    await mkdir(dirname(outPath), { recursive: true });
-    await writeFileAtomic(outPath, contentBuf);
-    written += 1;
-  }
+  // The download is complete: move out what the previous pull wrote and this
+  // tree lacks (another branch's files, files deleted on Hiveku), so a push
+  // cannot send them back.
+  const localFiles = (await walkLocalFiles(dest)).map((abs) => relative(dest, abs).split(sep).join('/'));
+  const tidy = await setAsideStaleFiles({
+    baseDir,
+    slug: localSlug,
+    dest,
+    projectId,
+    previous,
+    treePaths: [...(tree.paths || []), ...tree.skipped],
+    localFiles,
+  });
+  await writePulledFiles(baseDir, localSlug, { branch: pullBranch, files: tree.paths || [] });
 
-  // Drop a RUN.md keyed off project_type so the user knows how to start it.
+  // Run hints keyed off project_type, inside the folder's .hiveku/ so a push
+  // never sends them into the site.
   const runMd = [
     `# ${project.name || project.slug || projectId}`,
     '',
     `Project type: \`${project.project_type || 'unknown'}\``,
     `Hiveku project id: \`${projectId}\``,
-    branch ? `Tracking branch: \`${branch}\`` : '',
+    pullBranch ? `Tracking branch: \`${pullBranch}\`` : '',
     '',
     '## Run locally',
     '',
@@ -439,15 +441,18 @@ export async function runPullProject({ slugOrId, profile, branch, client: inject
     '',
     "This folder is NOT a git clone — it's a snapshot. `push` writes your edits to the",
     'DEPLOY SOURCE (what `deploy` builds from), lane-routing images/fonts/video to the',
-    'S3/CDN asset store and everything else to versioned code files.',
+    'S3/CDN asset store and everything else to the project files, then saves them as',
+    `one version of ${pullBranch ? `branch \`${pullBranch}\`` : 'Your site'}, named in plain language for the site owner.`,
     '',
     'From this folder:',
     '',
     '```bash',
     'hiveku-sync pull-env                          # project secrets → .env.local',
     'hiveku-sync status                            # diff local vs the deploy source',
-    'hiveku-sync push                              # save changes back',
+    'hiveku-sync push -m "Updated the pricing section on the Home page"',
     'hiveku-sync push --deploy development --wait  # ...then deploy and wait',
+    'hiveku-sync versions                          # the version history',
+    'hiveku-sync rollback <version id>             # what going back would change (dry run)',
     '```',
     '',
     'Or ask Claude in this workspace to push/deploy for you.',
@@ -455,7 +460,7 @@ export async function runPullProject({ slugOrId, profile, branch, client: inject
   ]
     .filter((line) => line !== undefined)
     .join('\n');
-  await writeFileAtomic(join(dest, 'RUN.md'), runMd);
+  await writeFileAtomic(join(dest, RUN_HINTS_FILE), runMd);
 
   // Persist project metadata so status/push know which project this folder maps to.
   const manifest = await readProjectsManifest(baseDir);
@@ -465,14 +470,46 @@ export async function runPullProject({ slugOrId, profile, branch, client: inject
     name: project.name || project.slug || localSlug,
     project_type: project.project_type || null,
     external_platform: project.external_platform || null,
-    branch: branch || null,
+    branch: pullBranch,
+    // What the pull came from: the version it was at, and on a branch the
+    // working-tree fingerprint. push compares it with the one
+    // project_files_status reports, to see whether someone else saved on the
+    // branch since (pushGuards.mjs), and records the tree its own push left.
+    head_commit_id: tree.head_commit_id ?? null,
+    working_tree_etag: tree.working_tree_etag ?? null,
     last_pull_at: new Date().toISOString(),
   };
   await writeProjectsManifest(baseDir, manifest);
 
-  console.log(`  Wrote ${written} file${written !== 1 ? 's' : ''}${skipped ? ` (${skipped} skipped — no content)` : ''}`);
-  console.log(`  Run hints in ${join(dest, 'RUN.md')}`);
-  return { dest, written, skipped, projectId, slug: localSlug };
+  console.log(`  Wrote ${written} file${written !== 1 ? 's' : ''}${skipped ? ` (${skipped} skipped: no content or an unsafe path)` : ''}`);
+  if (tidy.setAside.length) {
+    const n = tidy.setAside.length;
+    console.log(
+      `  Moved ${n} file${n !== 1 ? 's' : ''} that ${n !== 1 ? 'are' : 'is'} not on ${branchLabel(pullBranch)} out of this folder ` +
+        `(an earlier pull wrote ${n !== 1 ? 'them' : 'it'}): they are in ${shownPath(baseDir, tidy.setAsideDir)}/`,
+    );
+  }
+  if (tidy.notOnTarget.length) {
+    const n = tidy.notOnTarget.length;
+    console.log(
+      `  ${n} file${n !== 1 ? 's' : ''} here ${n !== 1 ? 'are' : 'is'} not on ${branchLabel(pullBranch)} and a push would add ` +
+        `${n !== 1 ? 'them' : 'it'}: ${tidy.notOnTarget.slice(0, 10).join(', ')}${n > 10 ? ` and ${n - 10} more` : ''}. ` +
+        'Keep the ones that are your own new work; move the others out of the folder before you push.',
+    );
+  }
+  console.log(`  Run hints in ${join(dest, RUN_HINTS_FILE)}`);
+  return {
+    dest,
+    written,
+    skipped,
+    projectId,
+    slug: localSlug,
+    branch: pullBranch,
+    head_commit_id: tree.head_commit_id ?? null,
+    working_tree_etag: tree.working_tree_etag ?? null,
+    set_aside: tidy.setAside,
+    not_on_target: tidy.notOnTarget,
+  };
 }
 
 export async function findProjectFolder({ slug, baseDir }) {
@@ -507,7 +544,7 @@ function summarizeDiff(diff) {
   };
 }
 
-export async function runProjectStatus({ slug, profile, json = false, client: injectedClient, baseDir: baseDirOverride }) {
+export async function runProjectStatus({ slug, profile, json = false, branch, client: injectedClient, baseDir: baseDirOverride }) {
   // Resolve baseDir as the account folder. If we're inside ./projects/<x>/,
   // walk up two levels. Otherwise treat cwd as the account folder.
   const cwd = process.cwd();
@@ -525,12 +562,16 @@ export async function runProjectStatus({ slug, profile, json = false, client: in
 
   // Deploy-source-accurate + asset-lane-aware — the SAME diff `push` runs, so
   // status never flags unchanged images as only_local (code goes through
-  // project_files_status; assets through assets_list).
-  const { diff, assetAll, assetToUpload, assetSkipped, plan } = await computePushPlan({
+  // project_files_status; assets through assets_list). A folder pulled from a
+  // branch diffs against that branch's working tree.
+  const flagged = branchFlag(branch);
+  const statusBranch = flagged !== undefined ? flagged : (info.branch && info.branch !== 'main' ? info.branch : null);
+  const { diff, remoteOnly, notMirrored, assetAll, assetToUpload, assetSkipped, plan } = await computePushPlan({
     client,
     projectId: info.project_id,
     folder,
     includeDeletes: true,
+    branch: statusBranch,
   });
 
   if (json) {
@@ -540,6 +581,11 @@ export async function runProjectStatus({ slug, profile, json = false, client: in
           slug: resolvedSlug,
           project_id: info.project_id,
           code: diff,
+          // only_remote split: what push --include-deletes would delete, and
+          // what this folder never holds (dot files, build folders, files
+          // Hiveku manages), which is never a deletion.
+          remote_only_deletable: remoteOnly,
+          remote_only_not_mirrored: notMirrored,
           assets: {
             to_upload: assetToUpload.map((f) => f.path),
             unchanged: assetSkipped,
@@ -556,10 +602,14 @@ export async function runProjectStatus({ slug, profile, json = false, client: in
 
   const changed = diff.changed?.length || 0;
   const onlyLocal = diff.only_local?.length || 0;
-  const onlyRemote = diff.only_remote?.length || 0;
+  const onlyRemote = remoteOnly.length;
   const unchanged = diff.same_count || 0;
   console.log(`Project: ${resolvedSlug}  (${info.project_id})`);
-  console.log('Compared against the DEPLOY SOURCE (builder_code_versions + asset store).\n');
+  console.log(
+    statusBranch
+      ? `Compared against branch "${statusBranch}" (its working tree) and the shared image store.\n`
+      : 'Compared against Your site (what a deploy ships) and the shared image store.\n',
+  );
   console.log('Code:');
   console.log(`  changed:     ${changed}`);
   console.log(`  added:       ${onlyLocal}`);
@@ -579,10 +629,17 @@ export async function runProjectStatus({ slug, profile, json = false, client: in
   printList(diff.changed, 'M', 'Changed code:');
   printList(diff.only_local, '+', 'Added code (will be created on push):');
   printList(
-    diff.only_remote,
+    remoteOnly,
     '-',
     'Remote-only code (deleted locally or never pulled — push --include-deletes to delete):',
+    (path) => path,
   );
+  if (notMirrored.length) {
+    console.log(
+      `\n${notMirrored.length} more file${notMirrored.length !== 1 ? 's are' : ' is'} only on Hiveku because this folder never holds ` +
+        'them (hidden files such as .gitignore, build folders, files Hiveku manages). A push never deletes them.',
+    );
+  }
   printList(assetToUpload, 'A', 'Assets to upload (new or changed):');
   return { diff, plan };
 }