canopycms 0.11.1 → 0.12.1

package/dist/worker/branch-merge.js

@@ -1,11 +1,13 @@
 import { randomUUID } from 'node:crypto';
 import fs from 'node:fs/promises';
 import path from 'node:path';
+import { readSchemaSnapshot } from '../branch-schema-cache.js';
 import { extractIdFromFilename } from '../content-id-index.js';
 import { normalizeFilesystemPath } from '../paths/normalize.js';
 import { ROOT_COLLECTION_ID } from '../paths/types.js';
 import { isNonFastForwardRejection } from '../utils/git.js';
 import { getErrorMessage, redactCredentials } from '../utils/error.js';
+import { resolveFieldMerge } from './field-merge.js';
 import { runGit } from './git-process.js';
 import { pinnedReceivePack, sharedRepoGit, sharedRepoProcessOptions } from './shared-repo-git.js';
 /**
@@ -16,10 +18,12 @@ import { pinnedReceivePack, sharedRepoGit, sharedRepoProcessOptions } from './sh
  */
 const OBJECT_ID = /^[0-9a-f]{40}([0-9a-f]{24})?$/;
 /**
- * The conflicts {@link buildMergeCommit} settles by keeping `head`'s side of one path. Every other
- * kind (a rename colliding with another, a file meeting a directory, a directory rename) moves or
- * joins paths, so no one side of one path is the branch's version, and settling it by rule loses a
- * file. Messages that are not `CONFLICT` are git's notes, such as `Auto-merging`.
+ * The conflicts {@link planMerge} settles by keeping `head`'s side of one path. Every other kind (a
+ * rename colliding with another, a file meeting a directory, a directory rename) moves or joins
+ * paths, so no one side of one path is the branch's version, and settling it by rule loses a file;
+ * only the field merge, which pairs entries by content ID, settles those, and only where it decides
+ * every path the conflict names. Messages that are not `CONFLICT` are git's notes, such as
+ * `Auto-merging`.
  */
 const SETTLED_BY_KEEPING_HEAD = new Set([
     'CONFLICT (contents)',
@@ -46,27 +50,44 @@ function parseMergeTree(output, head, other) {
         const [, mode, oid, stage, path] = match;
         stages.push({ mode, oid, stage, path });
     }
-    const unsettled = new Set();
+    const unsettled = [];
     for (i++; i < fields.length && fields[i] !== '';) {
-        const kind = fields[i + parseInt(fields[i], 10) + 1];
-        if (kind?.startsWith('CONFLICT') && !SETTLED_BY_KEEPING_HEAD.has(kind))
-            unsettled.add(kind);
-        i += parseInt(fields[i], 10) + 3;
+        const count = parseInt(fields[i], 10);
+        const kind = fields[i + count + 1];
+        if (kind?.startsWith('CONFLICT') && !SETTLED_BY_KEEPING_HEAD.has(kind)) {
+            unsettled.push({ kind, paths: fields.slice(i + 1, i + 1 + count) });
+        }
+        i += count + 3;
     }
     // A path git could not decode as UTF-8 reads back as U+FFFD, which names no file.
-    if (stages.some(({ path }) => path.includes('\ufffd')))
-        unsettled.add('an unreadable file name');
-    return { tree, stages, unsettled: [...unsettled] };
+    const undecodable = stages.filter(({ path }) => path.includes('\ufffd')).map(({ path }) => path);
+    if (undecodable.length > 0)
+        unsettled.push({ kind: 'an unreadable file name', paths: [] });
+    return { tree, stages, unsettled };
 }
+const NO_FIELD_MERGE = {
+    updates: new Map(),
+    covered: new Set(),
+    conflicts: [],
+    keyRemaps: [],
+    paths: new Map(),
+};
 /**
- * The merge of `other` into `head`, as a commit no ref names, with `head` the first parent. A path
- * git cannot merge keeps `head`'s side, stage 2, at the path git reports, or stays deleted when
- * `head` has no such stage: the rule the rebase loop applies to a conflict, since `head` has the
- * editor's work. The commit takes its identity from the clone's `user.*`, written when the clone
- * is provisioned.
+ * The merge of `other` into `head`, as a tree. Every entry and `.collection.json` both sides
+ * changed is merged field by field (field-merge.ts), which settles git's conflicts there, whatever
+ * their kind. A path git cannot merge that is no entry keeps `head`'s side, stage 2, at the path
+ * git reports, or stays deleted when `head` has no such stage: `head` has the editor's work. With
+ * no single merge base, git's merge stands as that rule leaves it, since a field merge needs one
+ * version both sides started from.
  */
-export async function buildMergeCommit(git, clonePath, args) {
-    const { head, other, message, timeoutMs } = args;
+export async function planMerge(git, clonePath, args) {
+    const { head, other, contentRoot, timeoutMs } = args;
+    const bases = (await git.raw(['merge-base', '--all', '--end-of-options', head, other]))
+        .split('\n')
+        .filter((line) => OBJECT_ID.test(line));
+    if (bases.length === 0) {
+        return { kind: 'unsettled', conflicts: ['no history in common with it'], paths: [] };
+    }
     const output = await git.raw([
         'merge-tree',
         '--write-tree',
@@ -75,16 +96,99 @@ export async function buildMergeCommit(git, clonePath, args) {
         head,
         other,
     ]);
-    const { tree: mergedTree, stages, unsettled } = parseMergeTree(output, head, other);
-    const conflicted = [...new Set(stages.map(({ path }) => path))];
-    if (unsettled.length > 0)
-        return { kind: 'unsettled', conflicts: unsettled, paths: conflicted };
-    const tree = stages.length === 0
-        ? mergedTree
-        : await keepHeadStages(clonePath, { mergedTree, stages, conflicted, timeoutMs });
+    const { tree: gitTree, stages, unsettled } = parseMergeTree(output, head, other);
+    const gitConflicted = [...new Set(stages.map(({ path }) => path))];
+    const field = bases.length === 1
+        ? await resolveFieldMerge({
+            git,
+            clonePath,
+            ancestor: bases[0],
+            head,
+            other,
+            mergedTree: gitTree,
+            conflicted: new Set(gitConflicted),
+            contentRoot,
+            flatSchema: await readSchemaSnapshot(clonePath),
+            collectionIdOf: (file) => conflictFilesToContentIds([file], contentRoot)[0],
+            timeoutMs,
+        })
+        : NO_FIELD_MERGE;
+    const unsettledLeft = unsettled.filter(({ paths }) => paths.length === 0 || paths.some((p) => !field.covered.has(p)));
+    if (field.unsettled)
+        unsettledLeft.push({ kind: field.unsettled, paths: [] });
+    if (unsettledLeft.length > 0) {
+        return {
+            kind: 'unsettled',
+            conflicts: [...new Set(unsettledLeft.map(({ kind }) => kind))],
+            paths: gitConflicted,
+        };
+    }
+    const updates = new Map(field.updates);
+    const kept = gitConflicted.filter((file) => !field.covered.has(file));
+    const headStages = new Map(stages.filter((s) => s.stage === '2').map((s) => [s.path, s]));
+    for (const file of kept) {
+        const stage = headStages.get(file);
+        updates.set(file, stage ? { path: file, mode: stage.mode, oid: stage.oid } : null);
+    }
+    const tree = updates.size === 0 ? gitTree : await withUpdates(clonePath, { gitTree, updates, timeoutMs });
+    const conflicts = [
+        ...field.conflicts,
+        ...keptVersions(kept, stages, contentRoot, field.conflicts),
+    ];
+    return {
+        kind: 'planned',
+        tree,
+        gitTree,
+        gitConflicted,
+        conflicts,
+        keptPaths: [...new Set([...field.conflicts.map(({ path }) => path), ...kept])],
+        keyRemaps: field.keyRemaps,
+        paths: field.paths,
+        asGitDoes: gitConflicted.length === 0 &&
+            conflicts.length === 0 &&
+            field.keyRemaps.length === 0 &&
+            tree === gitTree,
+    };
+}
+/**
+ * The files the merge kept `head`'s side of as entry-level conflicts, by the content ID their path
+ * carries, one per ID and none for an ID the field merge already names. A path with none is
+ * dropped rather than guessed at.
+ */
+function keptVersions(kept, stages, contentRoot, named) {
+    const seen = new Set(named.map(({ contentId }) => contentId));
+    const found = [];
+    for (const file of kept) {
+        const [contentId] = conflictFilesToContentIds([file], contentRoot);
+        if (!contentId || seen.has(contentId))
+            continue;
+        seen.add(contentId);
+        const oid = (stage) => stages.find((s) => s.path === file && s.stage === stage)?.oid;
+        const [ancestor, branch, base] = [oid('1'), oid('2'), oid('3')];
+        found.push({
+            contentId,
+            path: file,
+            kind: 'unreadable',
+            sides: {
+                ...(ancestor && { ancestor }),
+                ...(base && { base }),
+                ...(branch && { branch }),
+            },
+        });
+    }
+    return found;
+}
+/**
+ * {@link planMerge}'s merge as a commit no ref names, with `head` the first parent. The commit takes
+ * its identity from the clone's `user.*`, written when the clone is provisioned.
+ */
+export async function commitMerge(git, plan, { head, other, message }) {
+    const { tree, gitTree: _gitTree, kind: _kind, ...rest } = plan;
     const sha = (await git.raw(['commit-tree', '-p', head, '-p', other, '-m', message, '--end-of-options', tree])).trim();
-    return { kind: 'built', sha, conflicted };
+    return { kind: 'built', sha, ...rest };
 }
+/** Whether `plan` is git's own merge with nothing for a person. */
+export const mergesAsGitDoes = (plan) => plan.kind === 'planned' && plan.asGitDoes;
 /**
  * Whether `merged` holds exactly what `from` and `other` merge to cleanly, and so nothing else. A
  * conflict counts as no: whoever settled it chose what `merged` holds.
@@ -103,9 +207,9 @@ export async function isCleanMergeOf(git, merged, from, other) {
         return false;
     return (await git.revparse(['--verify', '--end-of-options', `${merged}^{tree}`])).trim() === tree;
 }
-/** `mergedTree` with each of `conflicted` as `head` has it, through a throwaway index. */
-async function keepHeadStages(clonePath, args) {
-    const { mergedTree, stages, conflicted, timeoutMs } = args;
+/** `gitTree` with each of `updates` in place, through a throwaway index. */
+async function withUpdates(clonePath, args) {
+    const { gitTree, updates, timeoutMs } = args;
     const shared = sharedRepoProcessOptions(clonePath, 'worktree');
     const indexFile = path.join(clonePath, '.git', `canopy-merge-index-${randomUUID()}`);
     const options = {
@@ -113,15 +217,11 @@ async function keepHeadStages(clonePath, args) {
         env: { ...shared.env, GIT_INDEX_FILE: indexFile },
         inactivityTimeoutMs: timeoutMs,
     };
-    const kept = new Map(stages.filter((s) => s.stage === '2').map((s) => [s.path, s]));
-    // Mode 0 removes a path: what is left when `head` has no side of it.
-    const zeros = '0'.repeat(mergedTree.length);
-    const entries = conflicted.map((file) => {
-        const entry = kept.get(file);
-        return `${entry?.mode ?? '0'} ${entry?.oid ?? zeros}\t${file}\0`;
-    });
+    // Mode 0 removes a path.
+    const zeros = '0'.repeat(gitTree.length);
+    const entries = [...updates].map(([file, blob]) => `${blob?.mode ?? '0'} ${blob?.oid ?? zeros}\t${file}\0`);
     try {
-        await runGit(['read-tree', '--end-of-options', mergedTree], options);
+        await runGit(['read-tree', '--end-of-options', gitTree], options);
         await runGit(['update-index', '-z', '--index-info'], { ...options, stdin: entries.join('') });
         return (await runGit(['write-tree'], options)).trim();
     }
@@ -170,7 +270,7 @@ export async function commitsApart(git, a, b) {
  *
  * Pure.
  */
-export function conflictFilesToContentIds(conflictedFiles, contentRoot) {
+function conflictFilesToContentIds(conflictedFiles, contentRoot) {
     // Entry files carry their ID in the filename. A `.collection.json` has none
     // of its own (extractIdFromFilename returns null for dot-prefixed files), so
     // the ID comes from the parent directory; the root content directory has no
@@ -211,18 +311,3 @@ export function conflictFilesToContentIds(conflictedFiles, contentRoot) {
         .filter((id) => id !== null);
     return [...new Set(conflictIds)];
 }
-/**
- * The conflict record after a rebase or merge: the IDs already recorded plus
- * its own. A conflict means this branch's version replaced base's change to
- * that entry, which a later clean rebase does not undo, so the record only
- * grows until the branch merges (buildMergedBranchUpdate clears it).
- *
- * Pure.
- */
-export function recordConflicts(recorded, found) {
-    const conflictFiles = [...new Set([...(recorded ?? []), ...found])];
-    return {
-        conflictStatus: conflictFiles.length > 0 ? 'conflicts-detected' : 'clean',
-        conflictFiles,
-    };
-}

package/dist/worker/cms-worker.d.ts

@@ -205,6 +205,9 @@ export declare class CmsWorker {
     private log;
     private statusReport?;
     private warnedDivergences;
+    private branchSyncRequests;
+    private gitSyncRequested;
+    private wakeGitSync;
     private lockLostMessage?;
     private statusWriterAtLock?;
     private settleSelfStopped;
@@ -333,8 +336,18 @@ export declare class CmsWorker {
      * Run `fn` repeatedly, the next invocation starting `interval` ms after the
      * previous one COMPLETES. setTimeout chaining rather than setInterval, so
      * executions cannot overlap when one runs longer than the interval.
+     *
+     * `early` lets a request cut the rest short to {@link EARLY_RUN_MIN_REST_MS}, or `interval` if
+     * shorter: `setWake`
+     * receives a function that does so while the loop rests, and undefined while a run is in
+     * flight; a request made during a run shows in `requested()` when it ends. Any number of
+     * requests coalesce into one early run.
      */
     private scheduleLoop;
+    /** See {@link WorkerContext.requestBranchSync}. */
+    private requestBranchSync;
+    /** See {@link WorkerContext.takeBranchSyncRequests}; called once per git sync cycle. */
+    private takeBranchSyncRequests;
     private get baseBranch();
     private get sanitizedBaseBranch();
     private requireBaseBranch;
@@ -477,9 +490,9 @@ export declare class CmsWorker {
      */
     private branchWorkspacePath;
     /**
-     * Test-only seam: awaited inside `rebaseActiveBranches()`'s conflict round,
-     * after `git rebase` reported conflicted files and BEFORE the
-     * `checkout --theirs` resolution loop overwrites them. No-op in production.
+     * Test-only seam: awaited inside `rebaseActiveBranches()`'s rebase round,
+     * after `git rebase` reported conflicted files and BEFORE the rebase is
+     * aborted for a merge. No-op in production.
      *
      * A test subclass overrides this to land a real `ContentStore` write at
      * exactly the instant the rebase is mid-flight, with no sleeps or shell

package/dist/worker/cms-worker.js

@@ -227,6 +227,11 @@ export async function recordWorkerStartupFailure(options) {
         await release?.().catch(() => { });
     }
 }
+/**
+ * The shortest rest before a run `scheduleLoop`'s `early` starts, so a request repeated while a
+ * branch stays locked cannot keep the git sync, and its GitHub fetch, running back to back.
+ */
+const EARLY_RUN_MIN_REST_MS = 10_000;
 export class CmsWorker {
     constructor(config) {
         this.config = config;
@@ -241,6 +246,10 @@ export class CmsWorker {
         this.releaseLockFn = null;
         this.log = cmsTaskQueueLogger;
         this.warnedDivergences = new Map();
+        // Branches asked for since the git sync last took them (requestBranchSync), by directory name.
+        this.branchSyncRequests = new Set();
+        // Set by any request; cleared by the next cycle, which takes it.
+        this.gitSyncRequested = false;
         /**
          * Settles once the worker has stopped for a reason it chose itself (its lock was
          * compromised), after the drain and its record in worker-status.json. Never settles for a
@@ -356,6 +365,8 @@ export class CmsWorker {
             ensureStatusReport: () => this.ensureStatusReport(),
             ensureSettingsBranch: () => this.ensureSettingsBranch(),
             warnedDivergences: () => this.warnedDivergences,
+            requestBranchSync: (branchRef) => this.requestBranchSync(branchRef),
+            takeBranchSyncRequests: () => this.takeBranchSyncRequests(),
             afterConflictDetectedForTesting: () => this.afterConflictDetectedForTesting(),
             afterRebaseCompletedForTesting: () => this.afterRebaseCompletedForTesting(),
         };
@@ -487,7 +498,12 @@ export class CmsWorker {
         const taskInterval = this.config.taskPollInterval ?? 5_000;
         const gitInterval = this.gitSyncIntervalMs;
         this.scheduleLoop('task queue', () => this.processTaskQueue(), taskInterval);
-        this.scheduleLoop('git sync', () => this.syncGit(), gitInterval);
+        this.scheduleLoop('git sync', () => this.syncGit(), gitInterval, {
+            requested: () => this.gitSyncRequested,
+            setWake: (wake) => {
+                this.wakeGitSync = wake;
+            },
+        });
         if (this.config.refreshAuthCache) {
             const cacheInterval = this.config.authCacheRefreshInterval ?? 15 * 60_000;
             this.scheduleLoop('auth cache refresh', () => this.refreshAuthCache(), cacheInterval);
@@ -695,22 +711,53 @@ export class CmsWorker {
      * Run `fn` repeatedly, the next invocation starting `interval` ms after the
      * previous one COMPLETES. setTimeout chaining rather than setInterval, so
      * executions cannot overlap when one runs longer than the interval.
+     *
+     * `early` lets a request cut the rest short to {@link EARLY_RUN_MIN_REST_MS}, or `interval` if
+     * shorter: `setWake`
+     * receives a function that does so while the loop rests, and undefined while a run is in
+     * flight; a request made during a run shows in `requested()` when it ends. Any number of
+     * requests coalesce into one early run.
      */
-    scheduleLoop(label, fn, interval) {
-        const run = () => {
+    scheduleLoop(label, fn, interval, early) {
+        // `delay` counts from `restStartedAt`, so a later wake never pushes the next run out.
+        const run = (delay, restStartedAt = Date.now()) => {
             if (!this.running)
                 return;
-            const timeout = setTimeout(async () => {
+            const fire = async () => {
                 this.activeTimeouts.delete(timeout);
+                early?.setWake(undefined);
                 const operation = fn().catch((err) => {
                     workerLogError('Worker loop error:', err instanceof Error ? err.message : err);
                 });
                 await this.trackOperation(label, operation);
-                run();
-            }, interval);
+                run(early?.requested() ? Math.min(interval, EARLY_RUN_MIN_REST_MS) : interval);
+            };
+            const timeout = setTimeout(fire, Math.max(0, delay - (Date.now() - restStartedAt)));
             this.activeTimeouts.add(timeout);
+            early?.setWake(() => {
+                const rested = Date.now() - restStartedAt;
+                const sooner = Math.max(0, Math.min(interval, EARLY_RUN_MIN_REST_MS) - rested);
+                if (sooner >= delay - rested || !this.activeTimeouts.delete(timeout))
+                    return;
+                clearTimeout(timeout);
+                run(rested + sooner, restStartedAt);
+            });
         };
-        run();
+        run(early?.requested() ? Math.min(interval, EARLY_RUN_MIN_REST_MS) : interval);
+    }
+    /** See {@link WorkerContext.requestBranchSync}. */
+    requestBranchSync(branchRef) {
+        if (branchRef !== undefined)
+            this.branchSyncRequests.add(sanitizeBranchName(branchRef));
+        this.gitSyncRequested = true;
+        this.wakeGitSync?.();
+    }
+    /** See {@link WorkerContext.takeBranchSyncRequests}; called once per git sync cycle. */
+    takeBranchSyncRequests() {
+        const taken = { branches: this.branchSyncRequests, requested: this.gitSyncRequested };
+        this.branchSyncRequests = new Set();
+        this.gitSyncRequested = false;
+        return taken;
     }
     get baseBranch() {
         return this.requireBaseBranch().name;
@@ -1232,9 +1279,9 @@ export class CmsWorker {
         return path.join(this.contentBranchesPath, sanitizeBranchName(branchRefName));
     }
     /**
-     * Test-only seam: awaited inside `rebaseActiveBranches()`'s conflict round,
-     * after `git rebase` reported conflicted files and BEFORE the
-     * `checkout --theirs` resolution loop overwrites them. No-op in production.
+     * Test-only seam: awaited inside `rebaseActiveBranches()`'s rebase round,
+     * after `git rebase` reported conflicted files and BEFORE the rebase is
+     * aborted for a merge. No-op in production.
      *
      * A test subclass overrides this to land a real `ContentStore` write at
      * exactly the instant the rebase is mid-flight, with no sleeps or shell
@@ -1261,8 +1308,8 @@ export class CmsWorker {
     // `rebaseActiveBranches` also has a consumer outside the test files:
     // apps/test-app/app/api/e2e-test/rebase/route.ts drives it from an e2e
     // fixture route. Do not delete it as unused.
-    async rebaseActiveBranches(gatewayPolicy, gitHub) {
-        return runRebaseCycle(this.ctx(), gatewayPolicy, gitHub);
+    async rebaseActiveBranches(gatewayPolicy, gitHub, requested) {
+        return runRebaseCycle(this.ctx(), gatewayPolicy, gitHub, requested);
     }
     // --- Git-sync cluster (worker/git-sync.ts) -----------------------------
     //

package/dist/worker/field-merge.d.ts

@@ -0,0 +1,65 @@
+import type { SimpleGit } from 'simple-git';
+import type { FlatSchemaItem } from '../config/types.js';
+import { type KeyRemap } from '../merge/entry-merge.js';
+import { type FoundEntryConflict } from '../merge-conflicts.js';
+import type { ContentId } from '../paths/types.js';
+/**
+ * The worker's side of the field merge: which files of a merge the engine (`merge/entry-merge.ts`)
+ * decides, and the blobs it decides them with. A merge's three versions are the merge base
+ * (ancestor), the commit coming in (base) and the branch's own (branch). Every entry file and
+ * `.collection.json` changed on both sides goes through the engine, whether or not git found a
+ * conflict in it: a merge git calls clean can still be wrong, such as a duplicate frontmatter key.
+ *
+ * Entries are paired by the content ID in their file names, and `.collection.json` files by their
+ * collection's, not by path, so a rename on one side meets an edit on the other whatever git's
+ * rename detection makes of it. A collection directory one side moved moves the other side's files
+ * in it too. An entry changed on one side only takes that side's version: where git's rename
+ * detection put something else there, the pairing by ID puts it right.
+ */
+/** A file in one version of the tree. */
+interface TreeFile {
+    path: string;
+    oid: string;
+    mode: string;
+}
+/** What the resolver makes of a merge, for `planMerge` to stage. */
+export interface FieldMergeResult {
+    /** Paths whose blob in the merge replaces git's: a file, or null for none. */
+    updates: Map<string, TreeFile | null>;
+    /** Every path of the units the resolver decided: git's conflicts on them are settled. */
+    covered: Set<string>;
+    conflicts: FoundEntryConflict[];
+    /** Item keys the merge changed, by the logical path the branch's comment threads name. */
+    keyRemaps: Array<{
+        entryPath: string;
+        contentId?: string;
+        remaps: KeyRemap[];
+    }>;
+    /** Where each entry or collection with a conflict on record may now be: its path in the merge. */
+    paths: Map<string, string>;
+    /** Why git's merge can't be settled by rule, where it can't. */
+    unsettled?: string;
+}
+export interface FieldMergeArgs {
+    git: SimpleGit;
+    clonePath: string;
+    ancestor: string;
+    head: string;
+    other: string;
+    /** The tree git's merge wrote. */
+    mergedTree: string;
+    /** Paths git left conflicted. */
+    conflicted: ReadonlySet<string>;
+    contentRoot: string;
+    /** The branch's entry types, which name conflicts as the "Show changes" drawer does. */
+    flatSchema: readonly FlatSchemaItem[] | undefined;
+    /** The content ID a `.collection.json` path belongs to. */
+    collectionIdOf: (file: string) => ContentId | undefined;
+    timeoutMs: number;
+}
+/**
+ * Decide every entry file and `.collection.json` the merge touches. Nothing is staged here: the
+ * result says which blobs replace git's, and which conflicts a person must resolve.
+ */
+export declare function resolveFieldMerge(args: FieldMergeArgs): Promise<FieldMergeResult>;
+export {};