@hiveku-apps/sync 0.3.5 → 0.4.0

package/src/init.mjs

@@ -96,7 +96,7 @@ Hiveku dashboard under Account memory.
 
 Then run \`hiveku-sync pull\` to refresh the local copies.
 
-## Coder projects (pull / status / push)
+## Coder projects (pull / push / versions)
 
 Each account has zero or more coder projects (Next.js, Vite, Python Lambda,
 Expo). To work on one locally:
@@ -105,25 +105,54 @@ Expo). To work on one locally:
 hiveku-sync projects                        # list the account's projects
 hiveku-sync pull-project <slug-or-id>       # snapshot into ./projects/<slug>/
 cd projects/<slug>
-# follow the run instructions in RUN.md (npm install && npm run dev for nextjs, etc.)
+# follow the run instructions in .hiveku/RUN.md (npm install && npm run dev for nextjs, etc.)
 \`\`\`
 
-This is **not a git clone** — the local folder is a snapshot. To check whether
-edits are in sync with the Hiveku DB, and to commit changes back to the
-project's connected GitHub branch (same path the v3 sidebar uses):
+This is **not a git clone** — the local folder is a snapshot of **Your site**
+(the project's main source; \`--branch <name>\` pulls a branch instead).
+\`push\` saves your edits to Hiveku and then saves them as ONE **version**, which
+can be rolled back later:
 
 \`\`\`bash
-hiveku-sync status                          # diff local vs Hiveku DB
-hiveku-sync push -m "fix: header overlap"   # save + commit changed files via MCP
+hiveku-sync status                          # diff local vs Hiveku
+hiveku-sync push -m "Updated the pricing section on the Home page"
+hiveku-sync versions                        # the version history, newest first
+hiveku-sync rollback <version id>           # dry run: what going back would change
+hiveku-sync deploy production               # publishing is always its own step
 \`\`\`
 
-You can also ask Claude in this workspace to:
-- "Pull project <slug>"
-- "Show me which files are out of sync with Hiveku"
-- "Commit the changes in projects/<slug> to the dev branch with message '...'"
-
-Claude has the \`project_files_list\`, \`project_file_get\`, \`project_files_status\`,
-and \`project_commit\` MCP tools available under the **${profile}** profile.
+- **Version names are read by the site owner.** Describe what changed for their
+  visitors ("Added a contact form to the About page"): never file paths, file
+  extensions, \`fix:\`-style prefixes or tool names. A version holds everything
+  waiting on Your site, including the Hiveku AI's and the editor's changes, so
+  without \`-m\` push names it from all of that; \`--no-version\` skips it.
+- **One version per change the owner would recognise**: push as often as you
+  like with \`--no-version\`, then save the version once (\`hiveku-sync commit -m "..."\`)
+  before you deploy.
+- **Exit codes:** 0 done; 1 some files did not save, or the push was refused
+  (listed; no version, no deploy); 2 every file IS saved but the version was not
+  (it prints the \`hiveku-sync commit\` that retries it); 3 the files and the
+  version ARE saved and only the deploy failed.
+- **On a branch, someone else may have saved since you pulled.** push then
+  refuses and lists the files that may hold their work: pull again (it replaces
+  your local copies, so keep your edits somewhere first), or \`--force\` to
+  overwrite. A pull moves the files an earlier pull wrote that are not on what
+  it pulls now (another branch's, or deleted on Hiveku) out of the project folder
+  into \`.hiveku/set-aside/\`, so they are never pushed back by mistake.
+- **Rolling back** writes a NEW version equal to the old one, so it can be undone,
+  and it never publishes. Run the dry run, show the owner what it changes, and only
+  after they agree run the \`--apply\` command it prints. Then offer \`deploy\`
+  separately.
+
+Through the MCP tools the same loop is: save (\`project_files_bulk_save\` /
+\`project_file_save\`), verify, then \`project_vcs_commit({ project_id, message })\`
+with **no files** (one version, plain-language name; 409 \`nothing_to_commit\`
+means it is already a version), then \`deploy_site\`.
+\`project_vcs_status\` says whether changes are waiting to be saved as a version;
+\`project_vcs_rollback\` rolls back (dry run by default; applying needs the dry
+run's \`head_commit_id\` as \`expected_head_commit_id\`, and on Your site its
+\`live_fingerprint\` as \`expected_live_fingerprint\`). The \`project_*\` tools come
+with the \`dev\` and \`full\` profiles (this folder uses **${profile}**).
 
 ## When Hiveku itself gets in your way
 

package/src/projects.mjs

@@ -1,15 +1,16 @@
 /**
  * `hiveku-sync` project commands — pull a Hiveku coder project into
- * ./projects/<slug>/, show diff vs remote, and commit edits back via MCP.
+ * ./projects/<slug>/, show diff vs remote, and (with `commit --github`) push
+ * edits to a connected GitHub branch via MCP.
  *
- * No native git clone. The MCP server handles GitHub commits via the
- * account's installed GitHub App — same code path as the v3 editor sidebar.
+ * No native git clone. Versions of Your site and of branches live in
+ * versions.mjs; the paged download pull-project uses lives in pullTree.mjs.
  */
 
 import { createHash } from 'node:crypto';
 import { existsSync } from 'node:fs';
-import { mkdir, readdir, readFile, stat } from 'node:fs/promises';
-import { dirname, join, relative, sep } from 'node:path';
+import { readdir, readFile, stat } from 'node:fs/promises';
+import { join, relative, sep } from 'node:path';
 
 import { HivekuMcpClient } from './mcp.mjs';
 import {
@@ -20,6 +21,9 @@ import {
   readJson,
 } from './util.mjs';
 import { isAccountMemoryPath } from './account-memory.mjs';
+import { downloadProjectTree } from './pullTree.mjs';
+import { readPulledFiles, RUN_HINTS_FILE, setAsideStaleFiles, shownPath, writePulledFiles } from './pullRecord.mjs';
+import { branchFlag, branchLabel } from './versions.mjs';
 // NOTE: push.mjs imports helpers from this module; the cycle is safe because
 // both sides reference the imports only inside functions (never at module eval).
 import { computePushPlan } from './push.mjs';
@@ -100,25 +104,6 @@ export async function clientFromCwd({ profileOverride } = {}) {
   return { client, baseDir, baseUrl, profile };
 }
 
-async function listAllRemoteFiles(client, projectId) {
-  const all = [];
-  let page = 1;
-  const limit = 500;
-  while (true) {
-    const result = await client.callToolJson('project_files_list', {
-      project_id: projectId,
-      page,
-      limit,
-    });
-    const batch = result?.data || [];
-    all.push(...batch);
-    const totalPages = result?.pagination?.total_pages ?? 1;
-    if (page >= totalPages || batch.length === 0) break;
-    page += 1;
-  }
-  return all;
-}
-
 /**
  * Every local file push, commit and status consider. The pulled account memory
  * (hiveku-data/account/) is left out wherever it sits: it is a read-only copy
@@ -135,8 +120,7 @@ export async function walkLocalFiles(rootDir) {
       return;
     }
     for (const entry of entries) {
-      if (entry.name.startsWith('.')) continue;
-      if (IGNORE_DIRS.has(entry.name)) continue;
+      if (walkSkipsName(entry.name)) continue;
       const full = join(dir, entry.name);
       if (isAccountMemoryPath(relative(rootDir, full).split(sep).join('/'))) continue;
       if (entry.isDirectory()) {
@@ -150,6 +134,24 @@ export async function walkLocalFiles(rootDir) {
   return out;
 }
 
+/** A file or folder name walkLocalFiles never enters: dot entries and build/tool folders. */
+function walkSkipsName(name) {
+  return name.startsWith('.') || IGNORE_DIRS.has(name);
+}
+
+/**
+ * Could walkLocalFiles ever list this project path? A path it cannot list
+ * (a dot file such as .gitignore, anything under node_modules/ or .next/, the
+ * pulled account memory) always reads as missing locally, so it must never be
+ * treated as deleted here.
+ */
+export function localWalkCanList(projectPath) {
+  const rel = String(projectPath || '').replace(/^\/+/, '');
+  if (!rel) return false;
+  if (rel.split('/').some((segment) => !segment || walkSkipsName(segment))) return false;
+  return !isAccountMemoryPath(rel);
+}
+
 async function buildLocalManifest(rootDir) {
   const files = await walkLocalFiles(rootDir);
   const manifest = [];
@@ -376,6 +378,8 @@ export async function runProjectsList({ profile, client: injectedClient } = {})
 
 export async function runPullProject({ slugOrId, profile, branch, client: injectedClient, baseDir: baseDirOverride }) {
   if (!slugOrId) throw new Error('Provide a project slug or UUID');
+  // null = Your site (main); a bare --branch with no name is refused.
+  const pullBranch = branchFlag(branch) ?? null;
   const { client, baseDir } = await resolveClient({ client: injectedClient, profile, baseDir: baseDirOverride });
   const project = await resolveProject(client, slugOrId);
   // An external site has no file tree: the pull would write nothing and stamp
@@ -387,47 +391,45 @@ export async function runPullProject({ slugOrId, profile, branch, client: inject
 
   console.log(`Pulling ${project.name || project.slug || projectId} (type=${project.project_type || '?'})`);
   console.log(`  Project ID: ${projectId}`);
+  console.log(`  ${pullBranch ? `Branch: ${pullBranch}` : 'Your site'}`);
   console.log(`  Destination: ${dest}`);
 
-  const files = await listAllRemoteFiles(client, projectId);
-  if (files.length === 0) {
+  // What the previous pull wrote here, so files the new tree lacks can be
+  // taken out of the folder (pullRecord.mjs).
+  const previous = await readPulledFiles(baseDir, localSlug);
+
+  // Paged, so a site of any size comes down (a site over 150 MB cannot be
+  // sent in one answer). See pullTree.mjs.
+  const tree = await downloadProjectTree({ client, projectId, branch: pullBranch, dest });
+  const written = tree.written;
+  const skipped = tree.skipped.length;
+  if (written === 0 && skipped === 0) {
     console.log('  No files to pull (project is empty).');
   }
 
-  let written = 0;
-  let skipped = 0;
-  for (const fileMeta of files) {
-    const result = await client.callToolJson('project_file_get', {
-      project_id: projectId,
-      file_path: fileMeta.file_path,
-    });
-    const fileBody = result?.file || result?.data?.file || result?.data;
-    if (!fileBody || typeof fileBody.file_content !== 'string') {
-      skipped += 1;
-      continue;
-    }
-    let contentBuf;
-    if (fileBody.file_content.startsWith('__BINARY_BASE64__')) {
-      contentBuf = Buffer.from(
-        fileBody.file_content.substring('__BINARY_BASE64__'.length),
-        'base64',
-      );
-    } else {
-      contentBuf = Buffer.from(fileBody.file_content, 'utf8');
-    }
-    const outPath = join(dest, fileMeta.file_path);
-    await mkdir(dirname(outPath), { recursive: true });
-    await writeFileAtomic(outPath, contentBuf);
-    written += 1;
-  }
+  // The download is complete: move out what the previous pull wrote and this
+  // tree lacks (another branch's files, files deleted on Hiveku), so a push
+  // cannot send them back.
+  const localFiles = (await walkLocalFiles(dest)).map((abs) => relative(dest, abs).split(sep).join('/'));
+  const tidy = await setAsideStaleFiles({
+    baseDir,
+    slug: localSlug,
+    dest,
+    projectId,
+    previous,
+    treePaths: [...(tree.paths || []), ...tree.skipped],
+    localFiles,
+  });
+  await writePulledFiles(baseDir, localSlug, { branch: pullBranch, files: tree.paths || [] });
 
-  // Drop a RUN.md keyed off project_type so the user knows how to start it.
+  // Run hints keyed off project_type, inside the folder's .hiveku/ so a push
+  // never sends them into the site.
   const runMd = [
     `# ${project.name || project.slug || projectId}`,
     '',
     `Project type: \`${project.project_type || 'unknown'}\``,
     `Hiveku project id: \`${projectId}\``,
-    branch ? `Tracking branch: \`${branch}\`` : '',
+    pullBranch ? `Tracking branch: \`${pullBranch}\`` : '',
     '',
     '## Run locally',
     '',
@@ -439,15 +441,18 @@ export async function runPullProject({ slugOrId, profile, branch, client: inject
     '',
     "This folder is NOT a git clone — it's a snapshot. `push` writes your edits to the",
     'DEPLOY SOURCE (what `deploy` builds from), lane-routing images/fonts/video to the',
-    'S3/CDN asset store and everything else to versioned code files.',
+    'S3/CDN asset store and everything else to the project files, then saves them as',
+    `one version of ${pullBranch ? `branch \`${pullBranch}\`` : 'Your site'}, named in plain language for the site owner.`,
     '',
     'From this folder:',
     '',
     '```bash',
     'hiveku-sync pull-env                          # project secrets → .env.local',
     'hiveku-sync status                            # diff local vs the deploy source',
-    'hiveku-sync push                              # save changes back',
+    'hiveku-sync push -m "Updated the pricing section on the Home page"',
     'hiveku-sync push --deploy development --wait  # ...then deploy and wait',
+    'hiveku-sync versions                          # the version history',
+    'hiveku-sync rollback <version id>             # what going back would change (dry run)',
     '```',
     '',
     'Or ask Claude in this workspace to push/deploy for you.',
@@ -455,7 +460,7 @@ export async function runPullProject({ slugOrId, profile, branch, client: inject
   ]
     .filter((line) => line !== undefined)
     .join('\n');
-  await writeFileAtomic(join(dest, 'RUN.md'), runMd);
+  await writeFileAtomic(join(dest, RUN_HINTS_FILE), runMd);
 
   // Persist project metadata so status/push know which project this folder maps to.
   const manifest = await readProjectsManifest(baseDir);
@@ -465,14 +470,46 @@ export async function runPullProject({ slugOrId, profile, branch, client: inject
     name: project.name || project.slug || localSlug,
     project_type: project.project_type || null,
     external_platform: project.external_platform || null,
-    branch: branch || null,
+    branch: pullBranch,
+    // What the pull came from: the version it was at, and on a branch the
+    // working-tree fingerprint. push compares it with the one
+    // project_files_status reports, to see whether someone else saved on the
+    // branch since (pushGuards.mjs), and records the tree its own push left.
+    head_commit_id: tree.head_commit_id ?? null,
+    working_tree_etag: tree.working_tree_etag ?? null,
     last_pull_at: new Date().toISOString(),
   };
   await writeProjectsManifest(baseDir, manifest);
 
-  console.log(`  Wrote ${written} file${written !== 1 ? 's' : ''}${skipped ? ` (${skipped} skipped — no content)` : ''}`);
-  console.log(`  Run hints in ${join(dest, 'RUN.md')}`);
-  return { dest, written, skipped, projectId, slug: localSlug };
+  console.log(`  Wrote ${written} file${written !== 1 ? 's' : ''}${skipped ? ` (${skipped} skipped: no content or an unsafe path)` : ''}`);
+  if (tidy.setAside.length) {
+    const n = tidy.setAside.length;
+    console.log(
+      `  Moved ${n} file${n !== 1 ? 's' : ''} that ${n !== 1 ? 'are' : 'is'} not on ${branchLabel(pullBranch)} out of this folder ` +
+        `(an earlier pull wrote ${n !== 1 ? 'them' : 'it'}): they are in ${shownPath(baseDir, tidy.setAsideDir)}/`,
+    );
+  }
+  if (tidy.notOnTarget.length) {
+    const n = tidy.notOnTarget.length;
+    console.log(
+      `  ${n} file${n !== 1 ? 's' : ''} here ${n !== 1 ? 'are' : 'is'} not on ${branchLabel(pullBranch)} and a push would add ` +
+        `${n !== 1 ? 'them' : 'it'}: ${tidy.notOnTarget.slice(0, 10).join(', ')}${n > 10 ? ` and ${n - 10} more` : ''}. ` +
+        'Keep the ones that are your own new work; move the others out of the folder before you push.',
+    );
+  }
+  console.log(`  Run hints in ${join(dest, RUN_HINTS_FILE)}`);
+  return {
+    dest,
+    written,
+    skipped,
+    projectId,
+    slug: localSlug,
+    branch: pullBranch,
+    head_commit_id: tree.head_commit_id ?? null,
+    working_tree_etag: tree.working_tree_etag ?? null,
+    set_aside: tidy.setAside,
+    not_on_target: tidy.notOnTarget,
+  };
 }
 
 export async function findProjectFolder({ slug, baseDir }) {
@@ -507,7 +544,7 @@ function summarizeDiff(diff) {
   };
 }
 
-export async function runProjectStatus({ slug, profile, json = false, client: injectedClient, baseDir: baseDirOverride }) {
+export async function runProjectStatus({ slug, profile, json = false, branch, client: injectedClient, baseDir: baseDirOverride }) {
   // Resolve baseDir as the account folder. If we're inside ./projects/<x>/,
   // walk up two levels. Otherwise treat cwd as the account folder.
   const cwd = process.cwd();
@@ -525,12 +562,16 @@ export async function runProjectStatus({ slug, profile, json = false, client: in
 
   // Deploy-source-accurate + asset-lane-aware — the SAME diff `push` runs, so
   // status never flags unchanged images as only_local (code goes through
-  // project_files_status; assets through assets_list).
-  const { diff, assetAll, assetToUpload, assetSkipped, plan } = await computePushPlan({
+  // project_files_status; assets through assets_list). A folder pulled from a
+  // branch diffs against that branch's working tree.
+  const flagged = branchFlag(branch);
+  const statusBranch = flagged !== undefined ? flagged : (info.branch && info.branch !== 'main' ? info.branch : null);
+  const { diff, remoteOnly, notMirrored, assetAll, assetToUpload, assetSkipped, plan } = await computePushPlan({
     client,
     projectId: info.project_id,
     folder,
     includeDeletes: true,
+    branch: statusBranch,
   });
 
   if (json) {
@@ -540,6 +581,11 @@ export async function runProjectStatus({ slug, profile, json = false, client: in
           slug: resolvedSlug,
           project_id: info.project_id,
           code: diff,
+          // only_remote split: what push --include-deletes would delete, and
+          // what this folder never holds (dot files, build folders, files
+          // Hiveku manages), which is never a deletion.
+          remote_only_deletable: remoteOnly,
+          remote_only_not_mirrored: notMirrored,
           assets: {
             to_upload: assetToUpload.map((f) => f.path),
             unchanged: assetSkipped,
@@ -556,10 +602,14 @@ export async function runProjectStatus({ slug, profile, json = false, client: in
 
   const changed = diff.changed?.length || 0;
   const onlyLocal = diff.only_local?.length || 0;
-  const onlyRemote = diff.only_remote?.length || 0;
+  const onlyRemote = remoteOnly.length;
   const unchanged = diff.same_count || 0;
   console.log(`Project: ${resolvedSlug}  (${info.project_id})`);
-  console.log('Compared against the DEPLOY SOURCE (builder_code_versions + asset store).\n');
+  console.log(
+    statusBranch
+      ? `Compared against branch "${statusBranch}" (its working tree) and the shared image store.\n`
+      : 'Compared against Your site (what a deploy ships) and the shared image store.\n',
+  );
   console.log('Code:');
   console.log(`  changed:     ${changed}`);
   console.log(`  added:       ${onlyLocal}`);
@@ -579,10 +629,17 @@ export async function runProjectStatus({ slug, profile, json = false, client: in
   printList(diff.changed, 'M', 'Changed code:');
   printList(diff.only_local, '+', 'Added code (will be created on push):');
   printList(
-    diff.only_remote,
+    remoteOnly,
     '-',
     'Remote-only code (deleted locally or never pulled — push --include-deletes to delete):',
+    (path) => path,
   );
+  if (notMirrored.length) {
+    console.log(
+      `\n${notMirrored.length} more file${notMirrored.length !== 1 ? 's are' : ' is'} only on Hiveku because this folder never holds ` +
+        'them (hidden files such as .gitignore, build folders, files Hiveku manages). A push never deletes them.',
+    );
+  }
   printList(assetToUpload, 'A', 'Assets to upload (new or changed):');
   return { diff, plan };
 }

package/src/pullRecord.mjs

@@ -0,0 +1,98 @@
+/**
+ * What pull-project wrote into a project folder, so the next pull can take
+ * away what the new tree no longer has.
+ *
+ * A pull writes every file of Your site (or of a branch) into
+ * projects/<slug>/ and never removed anything. Once `--branch` downloads the
+ * branch's real tree, switching a folder between Your site and a branch left
+ * the other tree's files behind, and the next push (which targets the pulled
+ * branch) sent them as new files and saved them as a version: a branch-only
+ * page ended up on Your site. A file deleted on Hiveku since the last pull
+ * came back the same way.
+ *
+ * So each pull records the paths it wrote (.hiveku/pulled/<slug>.json in the
+ * account folder), and the next pull moves the recorded files the new tree
+ * lacks out of the project folder, into .hiveku/set-aside/<slug>/<time>/ in
+ * the account folder: never deleted, and outside everything push reads. A
+ * file the pull did not write (your own new file) is never moved; the pull
+ * lists it instead, because a push would add it. The run hints hiveku-sync
+ * used to write as RUN.md in the project folder (they were pushed into the
+ * site) now live in the project folder's .hiveku/RUN.md; an old generated
+ * RUN.md is set aside like a stale file.
+ */
+
+import { existsSync } from 'node:fs';
+import { mkdir, readFile, rename } from 'node:fs/promises';
+import { dirname, join, relative, sep } from 'node:path';
+
+import { readJson, writeFileAtomic } from './util.mjs';
+import { localPathFor } from './pullTree.mjs';
+
+const PULLED_DIR = '.hiveku/pulled';
+const SET_ASIDE_DIR = '.hiveku/set-aside';
+/** Where pull-project writes its run hints inside the project folder (never pushed: push skips dot folders). */
+export const RUN_HINTS_FILE = '.hiveku/RUN.md';
+
+/** A project path as the local walk spells it (no leading slash). */
+export function recordPath(projectPath) {
+  return String(projectPath || '').replace(/^\/+/, '');
+}
+
+/** The paths the last pull wrote into projects/<slug>/, or null when no pull recorded them (an older hiveku-sync). */
+export async function readPulledFiles(baseDir, slug) {
+  const data = await readJson(join(baseDir, PULLED_DIR, `${slug}.json`));
+  return data && Array.isArray(data.files) ? data : null;
+}
+
+export async function writePulledFiles(baseDir, slug, { branch, files }) {
+  const record = { branch: branch || null, pulled_at: new Date().toISOString(), files: [...new Set(files.map(recordPath))].sort() };
+  await writeFileAtomic(join(baseDir, PULLED_DIR, `${slug}.json`), JSON.stringify(record, null, 2));
+  return record;
+}
+
+/** Is this the RUN.md an older hiveku-sync generated for this project (not the site's own)? */
+async function isGeneratedRunMd(dest, projectId) {
+  try {
+    const text = await readFile(join(dest, 'RUN.md'), 'utf8');
+    return text.includes(`Hiveku project id: \`${projectId}\``) && /## Ship changes back/.test(text);
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * After a complete download: move the files the previous pull wrote that the
+ * new tree lacks out of the folder, and name the other local files the tree
+ * lacks. `localFiles` are the folder's files as push sees them (relative
+ * paths). Returns { setAside: [paths], setAsideDir, notOnTarget: [paths] }.
+ */
+export async function setAsideStaleFiles({ baseDir, slug, dest, projectId, previous, treePaths, localFiles }) {
+  const inTree = new Set([...treePaths].map(recordPath));
+  const wrotePreviously = new Set((previous?.files || []).map(recordPath));
+  const extra = localFiles.filter((p) => !inTree.has(p));
+  const stale = [];
+  const notOnTarget = [];
+  for (const path of extra) {
+    if (wrotePreviously.has(path) || (path === 'RUN.md' && (await isGeneratedRunMd(dest, projectId)))) stale.push(path);
+    else notOnTarget.push(path);
+  }
+  if (stale.length === 0) return { setAside: [], setAsideDir: null, notOnTarget };
+  const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const setAsideDir = join(baseDir, SET_ASIDE_DIR, slug, stamp);
+  const moved = [];
+  for (const path of stale) {
+    const from = localPathFor(dest, path);
+    const to = localPathFor(setAsideDir, path);
+    if (!from || !to || !existsSync(from)) continue;
+    await mkdir(dirname(to), { recursive: true });
+    await rename(from, to);
+    moved.push(path);
+  }
+  return { setAside: moved, setAsideDir: moved.length ? setAsideDir : null, notOnTarget };
+}
+
+/** A folder path for a person, relative to the account folder when it is inside it. */
+export function shownPath(baseDir, abs) {
+  const rel = relative(baseDir, abs);
+  return rel && !rel.startsWith('..') ? rel.split(sep).join('/') : abs;
+}