canopycms 0.12.1 → 0.13.0

package/dist/worker/conflicts-status.js

@@ -0,0 +1,204 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { BranchMetadataFileManager } from '../branch-metadata.js';
+import { BranchMetadataCorruptError } from '../branch-metadata-file.js';
+import { editorDeepLink, resolveEditorUrl } from '../editor-url.js';
+import { MergeConflictStore, MergeConflictsFileFormatError, MergeConflictsFileMalformedError, openConflictCount, } from '../merge-conflicts.js';
+import { getErrorMessage, isNodeError, sanitizeErrorMessage } from '../utils/error.js';
+import { escapeForLog } from '../utils/escape-text.js';
+import { GatewayRefusalError, MAX_COMMIT_STATUS_URL_LENGTH } from './branch-policy.js';
+import { getHttpStatus, isPermissionDenial } from './github-errors.js';
+import { workerLog, workerLogWarn } from './log.js';
+/** The most branch names a failure report lists. */
+const MAX_REPORTED_BRANCHES = 50;
+/**
+ * What `branchRoot`'s conflict record calls for: failure with the number of choices open, or while
+ * the record is held after a damaged one was set aside, or does not parse; success otherwise.
+ * @internal Exported for tests.
+ */
+export async function conflictsStatusOf(branchRoot) {
+    try {
+        const record = await new MergeConflictStore(branchRoot).read();
+        if (record?.held) {
+            return {
+                state: 'failure',
+                description: 'Conflict record set aside: an admin must release the branch in the editor',
+            };
+        }
+        const open = record ? openConflictCount(record.entries) : 0;
+        if (open === 0)
+            return { state: 'success', description: 'No conflicting changes to resolve' };
+        return {
+            state: 'failure',
+            description: open === 1
+                ? '1 conflicting change to resolve in the editor'
+                : `${open} conflicting changes to resolve in the editor`,
+        };
+    }
+    catch (err) {
+        if (err instanceof MergeConflictsFileMalformedError ||
+            err instanceof MergeConflictsFileFormatError) {
+            return { state: 'failure', description: "Couldn't read this branch's conflicts" };
+        }
+        throw err;
+    }
+}
+/**
+ * Set the conflicts status on every pull request whose head, as `githubTips` (this cycle's GitHub
+ * fetch) holds it, lacks the one its branch calls for. Returns the failure to report, undefined
+ * when every status was set, or null when the stage could not say: the branches could not be
+ * listed, or a drain cut it off. Never throws.
+ */
+export async function setConflictsStatuses(ctx, githubTips) {
+    let due;
+    let unreadable;
+    try {
+        ;
+        ({ due, unreadable } = await dueStatuses(ctx, githubTips));
+    }
+    catch (err) {
+        workerLogWarn(`Could not read the branches' conflicts for their status: ${getErrorMessage(err)}`);
+        return null;
+    }
+    const set = ctx.conflictsStatusesSet();
+    const failed = [];
+    let failure;
+    for (const [index, { branch, status, key }] of due.entries()) {
+        if (ctx.shutdownSignal().aborted)
+            return null;
+        try {
+            // Bounded, so a GitHub that stalls holds the sync for one timeout, not one per branch.
+            await ctx
+                .github()
+                .setConflictsStatus(status, AbortSignal.any([ctx.shutdownSignal(), AbortSignal.timeout(ctx.taskTimeoutMs)]));
+            set.set(branch, key);
+            workerLog(`Set the conflicts status of PR #${status.pullRequest} (${escapeForLog(branch)}) at ` +
+                `${status.sha.slice(0, 7)}: ${status.state}`);
+        }
+        catch (err) {
+            if (ctx.shutdownSignal().aborted)
+                return null;
+            if (err instanceof GatewayRefusalError && err.reason === 'not-head') {
+                // GitHub moves a pull request's head a moment after the push, so this cycle's tip can be
+                // ahead of it: one more sync for that tip, and no loop if the two never agree.
+                const asked = `not-head:${branch}`;
+                if (set.get(asked) !== status.sha) {
+                    set.set(asked, status.sha);
+                    ctx.requestBranchSync(branch);
+                }
+                continue;
+            }
+            const message = sanitizeErrorMessage(getErrorMessage(err));
+            failed.push(branch);
+            workerLogWarn(`Could not set the conflicts status of ${escapeForLog(branch)}: ${message}`);
+            // The gateway's refusals for the pull request itself report as `other`, retried each cycle.
+            const reason = err instanceof GatewayRefusalError
+                ? 'other'
+                : isPermissionDenial(err)
+                    ? 'permission'
+                    : isUnreachable(err)
+                        ? 'unreachable'
+                        : 'other';
+            // A refused credential names its remedy over any other failure.
+            if (reason === 'permission' || !failure)
+                failure = { reason, message };
+            // Every later write would meet the same GitHub; the next cycle tries again.
+            if (reason !== 'other') {
+                failed.push(...due.slice(index + 1).map((d) => d.branch));
+                break;
+            }
+        }
+    }
+    // After GitHub's answers, whose reason and remedy come first.
+    for (const u of unreadable)
+        failed.push(u.branch);
+    failure ??= unreadable[0] && { reason: 'other', message: unreadable[0].message };
+    if (!failure)
+        return undefined;
+    return {
+        ...failure,
+        branches: failed.slice(0, MAX_REPORTED_BRANCHES),
+        count: failed.length,
+        at: new Date().toISOString(),
+    };
+}
+/**
+ * A failure to reach GitHub or the gateway, or GitHub busy: no status at all, a timeout, a rate
+ * limit (a 403 that is not a permission denial), or a server error.
+ */
+function isUnreachable(err) {
+    const status = getHttpStatus(err);
+    return status === null || status === 403 || status === 408 || status === 429 || status >= 500;
+}
+/**
+ * Each branch with a pull request whose head lacks the status it calls for, and each whose conflict
+ * record could not be read, which costs only its own status. A branch.json that cannot be read says
+ * nothing of a pull request, so it is logged and skipped. Throws only when the branches directory
+ * cannot be listed.
+ */
+async function dueStatuses(ctx, githubTips) {
+    let entries;
+    try {
+        entries = await fs.readdir(ctx.contentBranchesPath, { withFileTypes: true });
+    }
+    catch (err) {
+        if (isNodeError(err) && err.code === 'ENOENT')
+            return { due: [], unreadable: [] };
+        throw err;
+    }
+    const set = ctx.conflictsStatusesSet();
+    const due = [];
+    const unreadable = [];
+    const cannotRead = (branch, err) => {
+        const message = sanitizeErrorMessage(getErrorMessage(err));
+        workerLogWarn(`Could not read ${escapeForLog(branch)} for its conflicts status: ${message}`);
+        unreadable.push({ branch, message });
+    };
+    for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
+        const dir = entry.name;
+        // The base branch has no pull request; `.trash-*` directories hold purged branches.
+        if (!entry.isDirectory() || dir.startsWith('.') || dir === ctx.sanitizedBaseBranch)
+            continue;
+        const branchRoot = path.join(ctx.contentBranchesPath, dir);
+        let meta;
+        try {
+            meta = await BranchMetadataFileManager.loadOnly(branchRoot);
+        }
+        catch (err) {
+            if (!(err instanceof BranchMetadataCorruptError) &&
+                !(isNodeError(err) && err.code === 'ENOENT'))
+                workerLogWarn(`Could not read ${escapeForLog(dir)}'s branch.json for its conflicts status: ` +
+                    sanitizeErrorMessage(getErrorMessage(err)));
+            continue;
+        }
+        const branch = meta?.branch;
+        if (!branch || branch.status === 'archived' || branch.pullRequestNumber === undefined)
+            continue;
+        if (branch.pullRequestState === 'merged' || branch.pullRequestState === 'closed')
+            continue;
+        const sha = githubTips.get(branch.name);
+        if (sha === undefined)
+            continue;
+        let text;
+        try {
+            text = await conflictsStatusOf(branchRoot);
+        }
+        catch (err) {
+            cannotRead(branch.name, err);
+            continue;
+        }
+        const editorUrl = branch.editorUrl && resolveEditorUrl({ configured: branch.editorUrl });
+        const link = editorUrl && editorDeepLink(editorUrl, branch.name);
+        const status = {
+            pullRequest: branch.pullRequestNumber,
+            sha,
+            ...text,
+            // A link the gateway would refuse is left off, so it never costs the status.
+            ...(link && link.length <= MAX_COMMIT_STATUS_URL_LENGTH && { targetUrl: link }),
+        };
+        const key = JSON.stringify(status);
+        if (set.get(branch.name) !== key)
+            due.push({ branch: branch.name, status, key });
+    }
+    return { due, unreadable };
+}

package/dist/worker/field-merge.d.ts

@@ -17,7 +17,7 @@ import type { ContentId } from '../paths/types.js';
  * detection put something else there, the pairing by ID puts it right.
  */
 /** A file in one version of the tree. */
-interface TreeFile {
+export interface TreeFile {
     path: string;
     oid: string;
     mode: string;
@@ -56,10 +56,25 @@ export interface FieldMergeArgs {
     /** The content ID a `.collection.json` path belongs to. */
     collectionIdOf: (file: string) => ContentId | undefined;
     timeoutMs: number;
+    /**
+     * Git's merge stands, and only the item keys of an entry both sides changed change: to the field
+     * merge's, where it holds the same content with no conflict ({@link rekeyAsMerged}).
+     */
+    rekeyOnly?: boolean;
 }
+/** `diff-tree --raw` from `from` to `to`, without rename detection: each path's two files. */
+export declare function changedFiles(git: SimpleGit, from: string, to: string): Promise<Array<{
+    path: string;
+    before?: TreeFile;
+    after?: TreeFile;
+}>>;
+/**
+ * Whether two versions of one entry file under `contentRoot` differ in item keys alone: what a
+ * re-key changes. False for anything else, a file's mode or path included.
+ */
+export declare function differsInItemKeysOnly(git: SimpleGit, x: TreeFile, y: TreeFile, contentRoot: string): Promise<boolean>;
 /**
  * Decide every entry file and `.collection.json` the merge touches. Nothing is staged here: the
  * result says which blobs replace git's, and which conflicts a person must resolve.
  */
 export declare function resolveFieldMerge(args: FieldMergeArgs): Promise<FieldMergeResult>;
-export {};

package/dist/worker/field-merge.js

@@ -1,8 +1,10 @@
 import path from 'node:path';
 import { parse as yamlParse } from 'yaml';
 import { entryLogicalPathOfFile, extractIdFromFilename } from '../content-id-index.js';
+import { ITEM_KEY } from '../item-keys.js';
 import { parseEntryParts } from '../content-listing.js';
 import { mergeCollectionMeta, mergeEntry, } from '../merge/entry-merge.js';
+import { rekeyAsMerged } from '../merge/rekey.js';
 import { exactToken } from '../merge/value-equality.js';
 import { LOCATION_CONFLICT_KEY, } from '../merge-conflicts.js';
 import { normalizeFilesystemPath } from '../paths/normalize.js';
@@ -38,7 +40,7 @@ function unitKeyOf(file, root, collectionIdOf) {
     return { key: collection ? `meta:${collection}` : `meta-at:${file}` };
 }
 /** `diff-tree --raw` from `from` to `to`, without rename detection: each path's two files. */
-async function changedFiles(git, from, to) {
+export async function changedFiles(git, from, to) {
     const out = await git.raw([
         'diff-tree',
         '-r',
@@ -322,6 +324,44 @@ const LOCATION = {
     sidePaths: {},
     kind: 'changed',
 };
+/** Whether `text` reads back as `merged`, item keys and all. */
+function readsAs(text, format, merged) {
+    try {
+        if (!holdsRecord(text, format))
+            return false;
+        const back = parseEntryParts(text, format);
+        return sameContent({ data: back.data, ...(back.body !== undefined && { body: back.body }) }, merged);
+    }
+    catch {
+        return false;
+    }
+}
+/** `value` with every item key dropped, and nothing else changed. */
+const withoutItemKeys = (value) => {
+    if (Array.isArray(value))
+        return value.map(withoutItemKeys);
+    if (!isPlainRecord(value))
+        return value;
+    return Object.fromEntries(Object.entries(value)
+        .filter(([name]) => name !== ITEM_KEY)
+        .map(([name, v]) => [name, withoutItemKeys(v)]));
+};
+/**
+ * Whether two versions of one entry file under `contentRoot` differ in item keys alone: what a
+ * re-key changes. False for anything else, a file's mode or path included.
+ */
+export async function differsInItemKeysOnly(git, x, y, contentRoot) {
+    const format = formatOf(y.path);
+    if (!format || x.path !== y.path || x.mode !== y.mode)
+        return false;
+    if (entryLogicalPathOfFile(y.path, normalizeFilesystemPath(contentRoot)) === null)
+        return false;
+    const [a, b] = await Promise.all([readEntry(git, x, format), readEntry(git, y, format)]);
+    return (!!a &&
+        !!b &&
+        a.side.body === b.side.body &&
+        exactToken(withoutItemKeys(a.side.data)) === exactToken(withoutItemKeys(b.side.data)));
+}
 class Resolver {
     constructor(args, moves) {
         this.args = args;
@@ -473,6 +513,52 @@ class Resolver {
             ...(merged.keyRemaps.length > 0 && { remaps: merged.keyRemaps }),
         };
     }
+    /** In {@link FieldMergeArgs.rekeyOnly}: git's merged entry under the field merge's item keys. */
+    async rekeyed(unit, gitFile) {
+        const { git } = this.args;
+        const [o, a, b] = [unit.ancestor, sideFile(unit, 'base'), sideFile(unit, 'branch')];
+        if (!a || !b || !gitFile || a.oid === b.oid)
+            return null;
+        const format = formatOf(gitFile.path);
+        if (!format || [o, a, b].some((f) => f && formatOf(f.path) !== format))
+            return null;
+        const [oSide, aSide, bSide, gitSide] = await Promise.all([o, a, b, gitFile].map((f) => (f ? readEntry(git, f, format) : undefined)));
+        if (!aSide || !bSide || !gitSide || (o && !oSide))
+            return null;
+        const fields = fieldsFor(this.args.flatSchema, b.path, this.root);
+        let result;
+        try {
+            result = mergeEntry({
+                format,
+                ancestor: oSide?.side ?? { data: {} },
+                base: aSide.side,
+                branch: bSide.side,
+                ...(fields && { fields }),
+            });
+        }
+        catch {
+            return null;
+        }
+        if (result.kind !== 'merged' || result.conflicts.length > 0)
+            return null;
+        if (result.body !== gitSide.side.body)
+            return null;
+        const data = rekeyAsMerged(gitSide.side.data, result.data);
+        if (!isPlainRecord(data) || exactToken(data) === exactToken(gitSide.side.data))
+            return null;
+        const merged = { data, ...(result.body !== undefined && { body: result.body }) };
+        const file = await this.bytesFor({
+            base: { file: gitFile, text: gitSide.text },
+            branch: gitFile,
+            isBase: false,
+            isBranch: false,
+            reads: (text) => readsAs(text, format, merged),
+            serialize: (onto) => serializeEntry(format, merged, onto),
+        }, gitFile.path);
+        if (!file)
+            return null;
+        return { file, ...(result.keyRemaps.length > 0 && { remaps: result.keyRemaps }) };
+    }
     /** A `.collection.json` both sides changed and both still have; else git's rule. */
     async collectionMeta(unit, gitClean, gitFile) {
         const { git } = this.args;
@@ -581,7 +667,14 @@ export async function resolveFieldMerge(args) {
         const gitClean = ![...paths].some((p) => args.conflicted.has(p));
         const only = gitHeld.size === 1 ? [...gitHeld.values()][0] : undefined;
         let resolution;
-        if (unit.touched.base && unit.touched.branch) {
+        if (args.rekeyOnly) {
+            if (unit.kind !== 'entry' || !unit.touched.base || !unit.touched.branch)
+                continue;
+            resolution = await resolver.rekeyed(unit, only);
+            if (resolution === null)
+                continue;
+        }
+        else if (unit.touched.base && unit.touched.branch) {
             resolution =
                 unit.kind === 'entry'
                     ? await resolver.entry(unit, gitClean, only)

package/dist/worker/git-sync.d.ts

@@ -16,7 +16,8 @@ import type { WorkerContext } from './worker-context.js';
  * says the serving editor lacks a schema the incoming content names), push this
  * deployment's own settings
  * branch, fast-forward the base branch's workspace, ask the gateway for its branch policy,
- * rebase every branch that is behind it (rebase.ts), then sweep old tasks, expired trashed
+ * rebase every branch that is behind it (rebase.ts), set each pull request's conflicts status
+ * (conflicts-status.ts), then sweep old tasks, expired trashed
  * branch directories and, where retention is on, expired archived branches.
  *
  * One ordering is load-bearing and nothing enforces it: `runRebaseCycle` MUST
@@ -32,7 +33,7 @@ import type { WorkerContext } from './worker-context.js';
  * The whole cycle is wrapped so both outcomes record a worker-status.json
  * snapshot, since an operator's only view of this loop is the admin panel.
  */
-export type GitSyncContext = Pick<WorkerContext, 'baseBranch' | 'sanitizedBaseBranch' | 'contentBranchesPath' | 'remoteGitPath' | 'schemaHoldMaxMs' | 'archivedBranchRetentionDays' | 'taskDir' | 'taskTimeoutMs' | 'log' | 'github' | 'ensureSettingsBranch' | 'ensureStatusReport' | 'warnedDivergences' | 'isRunning' | 'takeBranchSyncRequests' | 'isDraining' | 'shutdownSignal'> & RebaseContext;
+export type GitSyncContext = Pick<WorkerContext, 'baseBranch' | 'sanitizedBaseBranch' | 'contentBranchesPath' | 'remoteGitPath' | 'schemaHoldMaxMs' | 'archivedBranchRetentionDays' | 'taskDir' | 'taskTimeoutMs' | 'log' | 'github' | 'ensureSettingsBranch' | 'ensureStatusReport' | 'warnedDivergences' | 'conflictsStatusesSet' | 'isRunning' | 'takeBranchSyncRequests' | 'isDraining' | 'shutdownSignal'> & RebaseContext;
 /**
  * Push THIS deployment's own settings branch (`ensureSettingsBranch()`) from
  * remote.git to GitHub. Non-fatal. Skipped when `trackedTips`, GitHub's tips

package/dist/worker/git-sync.js

@@ -14,6 +14,7 @@ import { escapeForLog } from '../utils/escape-text.js';
 import { CANOPY_META_DIR, isCanopyInternalPath, isNonFastForwardRejection } from '../utils/git.js';
 import { MAX_REPORTED_PATHS, TRACKED_CANOPY_STATE_FIX, isUntracked, listTrackedCanopyState, restoreRetiredSchemaCache, splitByUpstreamTracking, untrackInIndex, } from './canopy-state.js';
 import { sweepArchivedBranches } from './archived-sweep.js';
+import { setConflictsStatuses } from './conflicts-status.js';
 import { hasPendingHistoryRewrite, recordSeenOnGitHub } from './history-rewrite.js';
 import { runRebaseCycle } from './rebase.js';
 import { describeMove } from './github-gateway.js';
@@ -519,6 +520,9 @@ export async function syncGit(ctx) {
         const rebaseSummary = await runRebaseCycle(ctx, branchPolicy, { tips: githubTips, fetchedAt }, requested.branches);
         if (stoppedForDrain(ctx, 'the cleanup sweeps'))
             return;
+        // After the rebase cycle, whose merges can change a branch's conflicts. Heads the cycle's own
+        // pushes move are set by the cycle each push wakes.
+        const conflictsStatusFailure = await setConflictsStatuses(ctx, githubTips);
         await cleanupOldTasks(ctx.taskDir, undefined, ctx.log);
         // For System health: the branches the gateway would refuse to publish, read after the rebase
         // cycle, which archives merged branches and restores renamed ones. Without a policy, a fatal
@@ -551,6 +555,11 @@ export async function syncGit(ctx) {
         const report = ctx.ensureStatusReport();
         report.lastGitSyncAt = new Date().toISOString();
         delete report.lastGitSyncError;
+        // null: the stage could not say, so the last report stands.
+        if (conflictsStatusFailure)
+            report.conflictsStatusFailure = conflictsStatusFailure;
+        else if (conflictsStatusFailure === undefined)
+            delete report.conflictsStatusFailure;
         // Recorded once the cycle has done everything it can, so System health shows it.
         if (needsOperator !== null) {
             report.lastGitSyncError = { message: needsOperator, at: report.lastGitSyncAt };

package/dist/worker/github-commits.d.ts

@@ -92,9 +92,9 @@ export declare function catchUpWithPublished(ctx: GitHubCommitsContext, args: Gi
 /** Bring GitHub's commits into the clone. Throws what git throws; the caller records it. */
 export declare function bringInGitHubCommits(ctx: GitHubCommitsContext, args: GitHubCommitsArgs): Promise<GitHubCommitsOutcome>;
 /**
- * `other` merged into the branch: built, recorded, published to `remote.git`, queued for GitHub,
- * and the clone moved to it, in that order, so each step leaves what the next cycle picks up and
- * only the last touches the working tree. `fromGitHub` says `other` is GitHub's tip rather than the
+ * `other` merged into the branch: built, recorded, published to `remote.git` and queued for GitHub
+ * unless it holds unsubmitted work (publish-decision.ts), and the clone moved to it, in that order,
+ * so each step leaves what the next cycle picks up and only the last touches the working tree. `fromGitHub` says `other` is GitHub's tip rather than the
  * base. `inReview` takes only a merge with nothing for a person to choose; any other leaves the
  * branch as it is and records nothing, and GitHub's own conflict notice stands on the pull request. A lock lost before the clone moves leaves it where it was: `wait`.
  */