canopycms 0.0.67 → 0.0.68-int.100

package/dist/branch-workspace.js

@@ -1,121 +1,267 @@
+import fs from 'node:fs/promises';
 import path from 'node:path';
-import { ensureBranchRoot } from './paths/index.js';
-import { getBranchMetadataFileManager, loadBranchContext } from './branch-metadata.js';
+import { BranchPathError, isSettingsBranchName, resolveBranchPath } from './paths/index.js';
+import { buildInitialBranchMetadata, loadBranchContext } from './branch-metadata.js';
+import { BRANCH_META_DIR, BRANCH_META_FILE, readBranchMetadataFile } from './branch-metadata-file.js';
+import { BranchRegistry } from './branch-registry.js';
+import { BranchDirOccupiedError, BranchProvisioningBusyError, CREATE_RESIDUE_MIN_QUIET_MS, classifyFinalDir, removeLeftoverDir, lockBranchName, publishStaging, quietAgeMs, stageBranchWorkspace, } from './branch-provisioning.js';
+import { recordSparseCone, sparseConeFor } from './branch-sparse.js';
 import { readsFromCheckout } from './build-mode.js';
+import { invalidateBranchContentCaches } from './content-index-generation.js';
 import { operatingStrategy } from './operating-mode/index.js';
-import { GitManager } from './git-manager.js';
-import { createDebugLogger } from './utils/debug.js';
+import { GitManager, managedWorkspaceConfig } from './git-manager.js';
+import { getErrorMessage } from './utils/error.js';
 import { resolveBaseBranch } from './utils/git.js';
-import { acquireProvisioningLock } from './utils/provisioning-lock.js';
-const log = createDebugLogger({ prefix: 'BranchWorkspace' });
-// In-memory lock to prevent concurrent workspace initialization
-const workspaceInitLocks = new Map();
+import { canopyLogWarn } from './utils/logger.js';
+import { OccWriteConflictError, writeOccJsonFile } from './utils/occ-json-write.js';
+import { ProvisionLog } from './utils/provision-log.js';
+import { branchProvisioningLockName, PROVISIONING_LOCK_STALE_MS } from './utils/provisioning-lock.js';
+let testHooks = {};
+/** @internal Exported for tests; no argument clears them. */
+export function setProvisioningTestHooks(hooks = {}) {
+    testHooks = hooks;
+}
+/** One provisioning per branch path per process; a joiner gets the leader's branch as `exists`. */
+const provisionsInFlight = new Map();
 /**
- * Manages per-branch filesystem workspace: resolves root, ensures metadata,
- * and updates the branch registry.
+ * Provisions per-branch workspaces under [PROV-1]: a branch's directory appears only by rename,
+ * complete with branch.json (branch-provisioning.ts).
  */
 export class BranchWorkspaceManager {
     constructor(config) {
         this.config = config;
     }
-    async ensureGitWorkspace(options) {
-        return log.timed('workspace', 'ensureGitWorkspace', async () => {
-            // One initialization per branch workspace per process.
-            const existingLock = workspaceInitLocks.get(options.branchRoot);
-            if (existingLock) {
-                await existingLock;
-                return;
-            }
-            const lockPromise = (async () => {
-                // The in-memory lock above only serializes one process, and separate
-                // processes can provision the same workspace at once (several Lambda
-                // containers on one EFS root; the dev server beside a content-reading
-                // script), which without a cross-process lock means both cloning into
-                // it: "destination path already exists". initializeWorkspace is
-                // idempotent, so the waiter finds the workspace already cloned. A
-                // build's content reads never get here — loadOrCreateBranchContext
-                // returns the checkout before provisioning.
-                let releaseLock;
-                try {
-                    log.debug('workspace', 'Ensuring git workspace', {
-                        branchName: options.branchName,
-                        mode: options.mode,
-                    });
-                    releaseLock = await acquireProvisioningLock(path.dirname(options.branchRoot), `.${path.basename(options.branchRoot)}.init.lock`);
-                    await GitManager.initializeWorkspace({
-                        workspacePath: options.branchRoot,
-                        branchName: options.branchName,
-                        mode: options.mode,
-                        baseBranch: options.baseBranch,
-                        sourceRoot: this.config.sourceRoot,
-                        defaultRemoteUrl: this.config.defaultRemoteUrl,
-                        remoteUrl: options.remoteUrl,
-                        remoteName: this.config.defaultRemoteName,
-                        allowNetworkRemoteInProd: this.config.allowNetworkRemoteInProd,
-                        branchType: 'content',
-                        gitBotAuthorName: this.config.gitBotAuthorName,
-                        gitBotAuthorEmail: this.config.gitBotAuthorEmail,
-                        gitExcludePattern: operatingStrategy(options.mode).getGitExcludePattern(),
-                    });
-                }
-                finally {
-                    // Release the cross-process lock first, then clear the in-memory lock.
-                    if (releaseLock) {
-                        try {
-                            await releaseLock();
-                        }
-                        catch (err) {
-                            log.debug('workspace', 'Failed to release workspace-init lock', { err });
-                        }
-                    }
-                    workspaceInitLocks.delete(options.branchRoot);
-                }
-            })();
-            workspaceInitLocks.set(options.branchRoot, lockPromise);
-            await lockPromise;
-        });
-    }
+    /** The branch's context, provisioning it first when it does not exist. */
     async openOrCreateBranch(options) {
-        const { branchName, mode, basePathOverride, title, description, access, createdBy, remoteUrl } = options;
-        const { branchRoot, baseRoot, branchName: safeName, } = await ensureBranchRoot({
-            mode,
-            branchName,
-            basePathOverride,
-        });
-        // Resolve the fork point once so the workspace clone and the recorded
-        // metadata are guaranteed to agree (config value, or dev-mode git HEAD).
+        return (await this.provisionBranch(options)).context;
+    }
+    /**
+     * Provision a branch workspace, or report the one already there.
+     *
+     * @throws BranchProvisioningBusyError when another process is mid-way through this branch, or
+     *   what is at its path is too fresh to judge (retriable)
+     * @throws BranchDirOccupiedError in prod, when its path holds a repository this deployment did
+     *   not create
+     * @throws BranchMetadataCorruptError when its branch.json does not parse
+     */
+    async provisionBranch(options) {
+        // resolveBranchPath refuses the reserved prefix; this also refuses an adopter's
+        // configured settings branch name, which only the config knows.
+        if (isSettingsBranchName(options.branchName, operatingStrategy(options.mode).getSettingsBranchName(this.config))) {
+            throw new BranchPathError('Settings branches are not content branches');
+        }
+        const { branchRoot, baseRoot, branchName: safeName } = resolveBranchPath(options);
+        const paths = {
+            branchRoot,
+            baseRoot,
+            dirName: path.basename(branchRoot),
+            safeName,
+        };
+        const existing = await readBranchMetadataFile(branchRoot);
+        if (existing) {
+            return { kind: 'exists', context: { branch: existing.branch, branchRoot, baseRoot } };
+        }
+        const inFlight = provisionsInFlight.get(branchRoot);
+        if (inFlight)
+            return { kind: 'exists', context: (await inFlight).context };
+        const run = this.provisionUnshared(options, paths);
+        provisionsInFlight.set(branchRoot, run);
+        try {
+            return await run;
+        }
+        finally {
+            provisionsInFlight.delete(branchRoot);
+        }
+    }
+    async provisionUnshared(options, paths) {
+        const { branchRoot, baseRoot, dirName } = paths;
+        await fs.mkdir(baseRoot, { recursive: true });
+        // Resolve the fork point once so the clone and the recorded metadata agree (config value, or
+        // dev-mode git HEAD), and the remote before any clone is paid for.
         const baseBranch = await resolveBaseBranch({
             defaultBaseBranch: this.config.defaultBaseBranch,
-            mode,
+            mode: options.mode,
             detectFrom: this.config.sourceRoot
                 ? path.resolve(process.cwd(), this.config.sourceRoot)
                 : undefined,
         });
-        await this.ensureGitWorkspace({
-            branchRoot,
-            branchName: safeName,
-            mode,
+        const remoteUrl = await GitManager.resolveCloneRemoteUrl({
+            mode: options.mode,
+            remoteUrl: options.remoteUrl,
+            defaultRemoteUrl: this.config.defaultRemoteUrl,
             baseBranch,
-            remoteUrl,
+            sourceRoot: this.config.sourceRoot,
+            allowNetworkRemoteInProd: this.config.allowNetworkRemoteInProd,
         });
-        // save() covers creation and update, keeping existing values and
-        // invalidating the registry.
-        const metadata = getBranchMetadataFileManager(branchRoot, baseRoot);
-        const meta = await metadata.save({
+        const metadata = buildInitialBranchMetadata({
             branch: {
-                name: safeName,
-                title,
-                description,
-                access,
-                createdBy,
+                name: paths.safeName,
+                title: options.title,
+                description: options.description,
+                access: options.access,
+                createdBy: options.createdBy,
                 baseBranch,
             },
         });
-        return {
-            branch: meta.branch,
-            branchRoot,
+        // Advisory and lock-free: refuse early, before paying for a clone, what publish would
+        // refuse anyway. Publish re-checks everything under the lock.
+        const state = await classifyFinalDir(branchRoot, remoteUrl);
+        if (state.kind === 'live' || state.kind === 'corrupt')
+            return this.existing(paths);
+        if (state.kind === 'protected')
+            throw new BranchProvisioningBusyError(dirName);
+        if (state.kind === 'foreign') {
+            // dev's CLI sync builds a branch workspace by `git init` and real commits, with no
+            // remote and no branch.json; it is adopted where it is.
+            if (options.mode !== 'dev')
+                throw new BranchDirOccupiedError(dirName);
+            return this.adoptInPlace(options, paths, baseBranch, metadata);
+        }
+        if (state.kind === 'residue' && (await this.recentlyActive(paths))) {
+            throw new BranchProvisioningBusyError(dirName);
+        }
+        const sparseCone = sparseConeFor(this.config.contentRoot);
+        await recordSparseCone(baseRoot, sparseCone).catch((err) => {
+            canopyLogWarn(`[canopy] Could not record the sparse-checkout cone: ${getErrorMessage(err)}`);
+        });
+        const provisionLog = new ProvisionLog(dirName);
+        let outcome;
+        try {
+            outcome = await this.stageAndPublish(paths, {
+                provisionLog,
+                baseBranch,
+                remoteUrl,
+                metadata,
+                mode: options.mode,
+                sparseCone,
+            });
+        }
+        catch (err) {
+            provisionLog.finish('error');
+            throw err;
+        }
+        provisionLog.finish(outcome.kind === 'created' ? 'ok' : 'exists');
+        return outcome;
+    }
+    async stageAndPublish(paths, run) {
+        const { branchRoot, baseRoot, dirName } = paths;
+        const staged = await stageBranchWorkspace({
             baseRoot,
-        };
+            dirName,
+            branchName: paths.safeName,
+            baseBranch: run.baseBranch,
+            remoteUrl: run.remoteUrl,
+            clone: {
+                remoteName: this.config.defaultRemoteName,
+                config: managedWorkspaceConfig(this.config.gitBotAuthorName, this.config.gitBotAuthorEmail),
+            },
+            gitExcludePattern: operatingStrategy(run.mode).getGitExcludePattern(),
+            metadata: run.metadata,
+            provisionLog: run.provisionLog,
+            sparseCone: run.sparseCone,
+        });
+        if (staged.kind === 'exists')
+            return this.existing(paths);
+        let published;
+        try {
+            await testHooks.beforePublish?.({ stagingPath: staged.stagingPath, finalPath: branchRoot });
+            published = await run.provisionLog.step('publish', () => publishStaging({
+                baseRoot,
+                dirName,
+                stagingPath: staged.stagingPath,
+                writeId: staged.meta.writeId,
+                expectedRemoteUrl: run.remoteUrl,
+                inspectBlocked: testHooks.inspectBlocked,
+            }));
+        }
+        finally {
+            if (published !== 'published')
+                await removeLeftoverDir(staged.stagingPath);
+        }
+        if (published === 'exists')
+            return this.existing(paths);
+        await run.provisionLog.step('register', () => register(paths));
+        return { kind: 'created', context: { branch: staged.meta.branch, branchRoot, baseRoot } };
+    }
+    /**
+     * The one exception to [PROV-1]: a dev workspace that already holds real work is kept where it
+     * is, given branch.json under the branch's lock.
+     */
+    async adoptInPlace(options, paths, baseBranch, metadata) {
+        const { branchRoot, baseRoot, dirName } = paths;
+        const provisionLog = new ProvisionLog(dirName);
+        let created = false;
+        try {
+            const release = await provisionLog.step('lock', () => lockBranchName(baseRoot, dirName));
+            try {
+                if (!(await readBranchMetadataFile(branchRoot))) {
+                    await GitManager.initializeWorkspace({
+                        workspacePath: branchRoot,
+                        branchName: paths.safeName,
+                        mode: options.mode,
+                        baseBranch,
+                        sourceRoot: this.config.sourceRoot,
+                        defaultRemoteUrl: this.config.defaultRemoteUrl,
+                        remoteUrl: options.remoteUrl,
+                        remoteName: this.config.defaultRemoteName,
+                        allowNetworkRemoteInProd: this.config.allowNetworkRemoteInProd,
+                        branchType: 'content',
+                        gitBotAuthorName: this.config.gitBotAuthorName,
+                        gitBotAuthorEmail: this.config.gitBotAuthorEmail,
+                        gitExcludePattern: operatingStrategy(options.mode).getGitExcludePattern(),
+                        provisionLog,
+                    });
+                    created = await provisionLog.step('metadata', () => writeOccJsonFile(path.join(branchRoot, BRANCH_META_DIR, BRANCH_META_FILE), { ...metadata }, { expectedVersion: null, trailingNewline: true }).then(() => true, (err) => {
+                        if (err instanceof OccWriteConflictError)
+                            return false;
+                        throw err;
+                    }));
+                }
+            }
+            finally {
+                await release().catch(() => { });
+            }
+            if (created)
+                await provisionLog.step('register', () => register(paths));
+        }
+        catch (err) {
+            provisionLog.finish('error');
+            throw err;
+        }
+        provisionLog.finish(created ? 'ok' : 'exists');
+        if (!created)
+            return this.existing(paths);
+        const meta = await readBranchMetadataFile(branchRoot);
+        if (!meta)
+            throw new BranchProvisioningBusyError(dirName);
+        return { kind: 'created', context: { branch: meta.branch, branchRoot, baseRoot } };
+    }
+    /** Whether residue at the branch path may still belong to a live process. */
+    async recentlyActive({ branchRoot, baseRoot, dirName }) {
+        const lockAgeMs = await fs.stat(path.join(baseRoot, branchProvisioningLockName(dirName))).then((stat) => Date.now() - stat.mtimeMs, () => Infinity);
+        if (lockAgeMs < PROVISIONING_LOCK_STALE_MS)
+            return true;
+        return (await quietAgeMs(branchRoot)) < CREATE_RESIDUE_MIN_QUIET_MS;
+    }
+    async existing({ branchRoot, baseRoot, dirName, }) {
+        const meta = await readBranchMetadataFile(branchRoot);
+        if (!meta)
+            throw new BranchProvisioningBusyError(dirName);
+        return { kind: 'exists', context: { branch: meta.branch, branchRoot, baseRoot } };
+    }
+}
+/**
+ * A new branch directory replaces whatever an earlier one of the same name left in this
+ * process's caches, and joins the registry. The branch is already published, so a failure here
+ * is logged rather than failing the create; the registry's miss backstop covers it.
+ */
+async function register({ branchRoot, baseRoot }) {
+    try {
+        await invalidateBranchContentCaches(branchRoot);
+        await new BranchRegistry(baseRoot).invalidate();
+    }
+    catch (err) {
+        canopyLogWarn(`[canopy] Registering new branch ${branchRoot} failed: ${getErrorMessage(err)}`);
     }
 }
 export { loadBranchContext } from './branch-metadata.js';

package/dist/build/asset-refs.d.ts

@@ -0,0 +1,93 @@
+/**
+ * Collect the asset URLs a static build references, so a release can make them exist first.
+ *
+ * The public `/assets/t/` path serves only what is stored, and widths are chosen by site code at
+ * render time, so build output is the one place every final URL appears. The adopter contract
+ * that follows: every `/assets/t/` URL the site can request must appear as text in its build
+ * output (ARCHITECTURE.md, "Why transform by URL directive and materialize at release?").
+ */
+import { z } from 'zod';
+/** @internal Written into the scanned directory, so a manifest the adopter builds afterwards covers it. */
+export declare const ASSET_REFS_FILENAME = "canopy-asset-refs.json";
+declare const assetRefsFileSchema: z.ZodObject<{
+    version: z.ZodLiteral<1>;
+    transforms: z.ZodArray<z.ZodObject<{
+        key: z.ZodString;
+        routes: z.ZodArray<z.ZodString, "many">;
+        files: z.ZodArray<z.ZodString, "many">;
+    }, "strict", z.ZodTypeAny, {
+        key: string;
+        files: string[];
+        routes: string[];
+    }, {
+        key: string;
+        files: string[];
+        routes: string[];
+    }>, "many">;
+    statics: z.ZodArray<z.ZodObject<{
+        key: z.ZodString;
+        routes: z.ZodArray<z.ZodString, "many">;
+        files: z.ZodArray<z.ZodString, "many">;
+    }, "strict", z.ZodTypeAny, {
+        key: string;
+        files: string[];
+        routes: string[];
+    }, {
+        key: string;
+        files: string[];
+        routes: string[];
+    }>, "many">;
+}, "strict", z.ZodTypeAny, {
+    version: 1;
+    transforms: {
+        key: string;
+        files: string[];
+        routes: string[];
+    }[];
+    statics: {
+        key: string;
+        files: string[];
+        routes: string[];
+    }[];
+}, {
+    version: 1;
+    transforms: {
+        key: string;
+        files: string[];
+        routes: string[];
+    }[];
+    statics: {
+        key: string;
+        files: string[];
+        routes: string[];
+    }[];
+}>;
+export type AssetRefsFile = z.infer<typeof assetRefsFileSchema>;
+/** A URL in the output that no stored object can answer. */
+export interface AssetRefProblem {
+    file: string;
+    url: string;
+    error: string;
+}
+/** The output references URLs that cannot be served from the store; nothing was written. */
+export declare class AssetRefsError extends Error {
+    readonly problems: readonly AssetRefProblem[];
+    constructor(problems: readonly AssetRefProblem[]);
+}
+export interface CollectAssetRefsResult {
+    refs: AssetRefsFile;
+    /** Absolute path of the refs file written. */
+    filePath: string;
+    scannedFiles: number;
+}
+/**
+ * Scan `outDir` for `/assets/t/...` transform URLs and `/assets/{hash32}/...` static URLs and
+ * write their stored keys to `outDir/canopy-asset-refs.json`, sorted so the same output always
+ * yields the same file. Throws `AssetRefsError`, writing nothing, if any transform URL is not in
+ * canonical form or any static URL is malformed: only canonical keys are stored, and the public
+ * path is to compute nothing on a miss.
+ */
+export declare function collectAssetRefs(outDir: string): Promise<CollectAssetRefsResult>;
+/** Read and validate a refs file `collectAssetRefs` wrote. */
+export declare function readAssetRefsFile(filePath: string): Promise<AssetRefsFile>;
+export {};