apify-test-tools 0.10.2-beta.2 → 0.10.2-beta.4

package/bin/build.ts

@@ -1,22 +1,23 @@
 import type * as ApifyClientTypes from 'apify-client';
 import { ActorSourceType, ApifyClient } from 'apify-client';
 
-import { normalizeRepoUrl } from './git.js';
+import { getCurrentBranch, normalizeRepoUrl, resolveRepoUrl } from './git.js';
 import { logger } from './logger.js';
 import type { ActorConfig, BuildData, DryRunPlan } from './types.js';
 
 type BuildPrActorOptions = {
-    buildTag?: string;
     versionNumber: string;
     gitRepoUrl: string;
     actorConfig: ActorConfig;
     actorInfo: ApifyClientTypes.Actor;
-    useDockerCache: boolean;
 };
 
 // Fixed version number used to build from local source files, since there is no real version to track.
 export const LOCAL_SOURCE_VERSION_NUMBER = '0.98';
-const DEFAULT_TEST_VERSION_NUMBER = '0.99';
+// Version that build-for-test builds, so tests never touch the version users run
+export const TEST_VERSION_NUMBER = '0.99';
+// Stands for each Actor's default version: the one its default build tag points to
+export const DEFAULT_VERSION_KEYWORD = 'default';
 
 // Shown whenever an Actor isn't set up the way CI builds need it
 const ACTOR_SETUP_REQUIREMENT =
@@ -319,20 +320,18 @@ export const runAndSummarizeBuilds = async (
 type GitBuildPlan = {
     actorFullName: string;
     versionNumber: string;
-    buildTag?: string;
     gitRepoUrl: string;
-    useDockerCache: boolean;
 };
 
 type RunBuildsOptions = {
     actorConfigs: ActorConfig[];
-    isLatest?: boolean;
+    // A version number like "0.99", or DEFAULT_VERSION_KEYWORD for each Actor's default version
+    versionNumber: string;
     repoUrl: string;
     // False when the repo URL was passed explicitly, see assertRepoUrlMatchesDefaultVersion
     shouldVerifyRepoUrl: boolean;
     branch: string;
     dryRun: boolean;
-    useDockerCache: boolean;
 };
 
 export const runBuilds = async ({
@@ -340,9 +339,8 @@ export const runBuilds = async ({
     shouldVerifyRepoUrl,
     branch,
     actorConfigs,
-    isLatest = false,
+    versionNumber,
     dryRun,
-    useDockerCache,
 }: RunBuildsOptions): Promise<BuildData[] | DryRunPlan<GitBuildPlan>> => {
     // Fetched once per Actor and reused for the checks, the version numbers and the build itself
     const buildConfigs: BuildPrActorOptions[] = await Promise.all(
@@ -352,6 +350,15 @@ export const runBuilds = async ({
                 actorConfig.actorFullName,
                 actorInfo,
             );
+            // Builds keep the tags their version gives them, so a default version without the default build tag
+            // would build fine but leave users on the previous build
+            if (versionNumber === DEFAULT_VERSION_KEYWORD && defaultVersion?.buildTag !== defaultBuildTag) {
+                logger.warn(
+                    `[${actorConfig.actorFullName}] Default version ${defaultVersionNumber} has build tag ` +
+                        `"${defaultVersion?.buildTag ?? 'none'}", not "${defaultBuildTag}", so users keep running the previous build. ` +
+                        'Set the build tag of the version on the platform to change that.',
+                );
+            }
             if (shouldVerifyRepoUrl) {
                 assertRepoUrlMatchesDefaultVersion(actorConfig.actorFullName, defaultVersion, repoUrl);
             }
@@ -365,9 +372,7 @@ export const runBuilds = async ({
                 actorConfig,
                 actorInfo,
                 gitRepoUrl,
-                versionNumber: isLatest ? defaultVersionNumber : DEFAULT_TEST_VERSION_NUMBER,
-                buildTag: isLatest ? defaultBuildTag : undefined,
-                useDockerCache,
+                versionNumber: versionNumber === DEFAULT_VERSION_KEYWORD ? defaultVersionNumber : versionNumber,
             };
         }),
     );
@@ -375,21 +380,15 @@ export const runBuilds = async ({
     // Everything above only reads, so a dry run has checked what the real run would, short of writing
     if (dryRun) {
         logger.info('[DRY RUN] Would build:');
-        for (const { actorConfig, versionNumber, gitRepoUrl } of buildConfigs) {
-            logger.info(`  ${actorConfig.actorFullName} (${versionNumber}) from ${gitRepoUrl}`);
+        const builds = buildConfigs.map((buildConfig) => ({
+            actorFullName: buildConfig.actorConfig.actorFullName,
+            versionNumber: buildConfig.versionNumber,
+            gitRepoUrl: buildConfig.gitRepoUrl,
+        }));
+        for (const build of builds) {
+            logger.info(`  ${build.actorFullName} (${build.versionNumber}) from ${build.gitRepoUrl}`);
         }
-        return {
-            dryRun: true,
-            builds: buildConfigs.map(
-                ({ actorConfig, versionNumber, buildTag, gitRepoUrl, useDockerCache: useCache }) => ({
-                    actorFullName: actorConfig.actorFullName,
-                    versionNumber,
-                    buildTag,
-                    gitRepoUrl,
-                    useDockerCache: useCache,
-                }),
-            ),
-        };
+        return { dryRun: true, builds };
     }
 
     const buildConfigsByActorFullName = new Map(
@@ -397,20 +396,38 @@ export const runBuilds = async ({
     );
 
     return runAndSummarizeBuilds(actorConfigs, 'BUILDS', async (actorConfig, builder) => {
-        const {
-            actorInfo,
-            buildTag,
-            versionNumber,
-            gitRepoUrl,
-            useDockerCache: useCache,
-        } = buildConfigsByActorFullName.get(actorConfig.actorFullName)!;
+        const buildConfig = buildConfigsByActorFullName.get(actorConfig.actorFullName)!;
+        // No buildTag: the version keeps the one it has, so its build gets the tag the version already gives it
         const actorVersion: ApifyClientTypes.ActorVersion = {
-            buildTag,
-            versionNumber,
-            gitRepoUrl,
+            versionNumber: buildConfig.versionNumber,
+            gitRepoUrl: buildConfig.gitRepoUrl,
             sourceType: ActorSourceType.GitRepo,
         };
-        return builder.createVersionAndBuild(versionNumber, actorVersion, useCache, actorInfo);
+        // Always with the Docker cache: CI always asked for it, and a clean build can still be started on the platform
+        return builder.createVersionAndBuild(buildConfig.versionNumber, actorVersion, true, buildConfig.actorInfo);
+    });
+};
+
+type BuildActorsOptions = Omit<RunBuildsOptions, 'repoUrl' | 'shouldVerifyRepoUrl' | 'branch'> & {
+    // Defaults to the `origin` remote, see resolveRepoUrl
+    repoUrl?: string;
+};
+
+/**
+ * Builds exactly the given Actors from the checked-out Git branch, as it is on the remote. Fails on a detached HEAD,
+ * where there is no branch to build. Used by `build` and the commands built on top of it.
+ */
+export const buildActors = async ({ actorConfigs, repoUrl, ...options }: BuildActorsOptions) => {
+    if (actorConfigs.length === 0) {
+        logger.info('No Actors to build');
+        // Still the dry-run shape in a dry run, so it can't pass for a real (empty) result
+        return options.dryRun ? { dryRun: true as const, builds: [] } : [];
+    }
+    return runBuilds({
+        ...options,
+        ...(await resolveRepoUrl(repoUrl)),
+        actorConfigs,
+        branch: await getCurrentBranch(),
     });
 };
 

package/bin/diff-changes.ts

@@ -9,7 +9,6 @@ import { type RelativeDir, RelativeFile } from './utils/path/repo-relative.js';
 interface ShouldBuildAndTestOptions {
     filepathsChanged: string[];
     actorConfigs: ActorConfig[];
-    isLatest?: boolean;
     commits: Commit[];
     // Glob relative to the repo root. Matching files count as changes even when .dockerignore lists them.
     testFilesGlob?: string;
@@ -112,6 +111,7 @@ const isExcludedBySibling = (filePath: RelativeFile, actor: ActorConfig, allActo
 type ActorChangeEntry = {
     actorConfig: ActorConfig;
     files: string[];
+    hasFunctionalChange: boolean;
 };
 
 type ChangeGroup = { actors: string[]; files: string[] };
@@ -164,13 +164,20 @@ const logChangeGroups = (groups: ChangeGroup[]): void => {
     }
 };
 
+export interface ChangedActors {
+    /** Actors with at least one functional change. They need a new build and tests. */
+    functional: ActorConfig[];
+    /** Actors whose changes are all cosmetic. They only need a new build when released. */
+    cosmetic: ActorConfig[];
+}
+
+/** Finds the Actors the changed files belong to, split by the most impactful change each Actor has. */
 export const getChangedActors = async ({
     filepathsChanged,
     actorConfigs,
-    isLatest = false,
     commits,
     testFilesGlob,
-}: ShouldBuildAndTestOptions): Promise<ActorConfig[]> => {
+}: ShouldBuildAndTestOptions): Promise<ChangedActors> => {
     const actorsChangedMap = new Map<string, ActorChangeEntry>();
     const testFilesInContext = new Set<string>();
     const isTestFile = testFilesGlob ? picomatch(testFilesGlob, { nocase: true, dot: true }) : () => false;
@@ -191,21 +198,21 @@ export const getChangedActors = async ({
             }
 
             if (change.impact === 'ignored' || change.impact === 'outside-context') continue;
-            if (change.impact === 'cosmetic' && !isLatest) continue;
 
-            const entry = actorsChangedMap.get(actorConfig.folder.path) ?? { actorConfig, files: [] };
+            const entry = actorsChangedMap.get(actorConfig.folder.path) ?? {
+                actorConfig,
+                files: [],
+                hasFunctionalChange: false,
+            };
             entry.files.push(originalFilePath);
+            entry.hasFunctionalChange ||= change.impact === 'functional';
             actorsChangedMap.set(actorConfig.folder.path, entry);
         }
     }
 
-    const actorsChanged = Array.from(actorsChangedMap.values()).map((entry) => entry.actorConfig);
-
     // Log changes grouped by actor set, so changes shared across actors are logged once
     // instead of being repeated per actor.
-    const fileToActors = buildFileToActorsMap(actorsChangedMap);
-    const groups = groupFilesByActorSet(fileToActors);
-    logChangeGroups(groups);
+    logChangeGroups(groupFilesByActorSet(buildFileToActorsMap(actorsChangedMap)));
 
     for (const testFile of filepathsChanged.filter(
         (filePath) => isTestFile(filePath) && !testFilesInContext.has(filePath),
@@ -216,12 +223,13 @@ export const getChangedActors = async ({
         );
     }
 
-    if (actorsChanged.length > 0) {
-        const actors = actorsChanged.map((config) => config.actorFullName);
-        logger.info(`[DIFF]: Actors to be built and tested: ${actors.join(', ')}`);
-    } else {
-        logger.info(`[DIFF]: No relevant files changed, skipping builds and tests`);
-    }
+    const entries = Array.from(actorsChangedMap.values());
+    const functional = entries.filter((entry) => entry.hasFunctionalChange).map((entry) => entry.actorConfig);
+    const cosmetic = entries.filter((entry) => !entry.hasFunctionalChange).map((entry) => entry.actorConfig);
+
+    const names = (configs: ActorConfig[]) => configs.map((config) => config.actorFullName).join(', ') || 'none';
+    logger.info(`[DIFF]: Actors with functional changes: ${names(functional)}`);
+    logger.info(`[DIFF]: Actors with only cosmetic changes: ${names(cosmetic)}`);
 
-    return actorsChanged;
+    return { functional, cosmetic };
 };

package/bin/git.ts

@@ -6,7 +6,7 @@ export const GIT_FORMAT_SEPARATOR = '»¦«';
 const GIT_LOG_FORMAT = ['%H', '%aN<%aE>', '%aD', '%s'].join(GIT_FORMAT_SEPARATOR);
 
 /**
- * Gets the list of changed files between the given commits (inclusive).
+ * Gets the list of changed files between the given commits (inclusive), so it diffs from the parent of the oldest.
  */
 export const getChangedFiles = async (commits: Commit[]) => {
     // getCommits never returns an empty list (the rerun check returns all commits when the base commit
@@ -15,10 +15,19 @@ export const getChangedFiles = async (commits: Commit[]) => {
         throw new Error('Cannot get changed files: the commit list is empty. This should never happen.');
     }
 
-    const changedFilesString = await runGitCommand(
-        ['diff', '--name-only', `${commits[0].sha}~..${commits[commits.length - 1].sha}`],
-        { operation: 'get the list of changed files between commits' },
-    );
+    return getChangedFilesBetween(`${commits[0].sha}~`, commits[commits.length - 1].sha);
+};
+
+/**
+ * Files that differ between two commits, i.e. changed by the commits in the range (fromRef, toRef]. fromRef's own
+ * changes are not included: to include them, pass its parent, `${sha}~` (as getChangedFiles does).
+ * Diffing the endpoints, rather than walking a commit list, is what keeps merges right: the oldest commit of a
+ * merged branch can predate fromRef, and its changes are already in fromRef.
+ */
+export const getChangedFilesBetween = async (fromRef: string, toRef: string) => {
+    const changedFilesString = await runGitCommand(['diff', '--name-only', `${fromRef}..${toRef}`], {
+        operation: 'get the list of changed files between commits',
+    });
 
     const changedFiles = changedFilesString.split('\n').filter(Boolean);
     logger.info(`Changed files (up to 50): ${changedFiles.slice(0, 50).join(', ')}`);
@@ -103,11 +112,14 @@ export const parseBaseCommit = (shaOrCommit: string | undefined): string | undef
     return sha;
 };
 
-const fetchAllBranchCommits = async (sourceBranch: string, targetBranch: string): Promise<Commit[]> => {
-    const output = await runGitCommand(
-        ['log', `--pretty=format:${GIT_LOG_FORMAT}`, `${targetBranch}..${sourceBranch}`],
-        { operation: 'list commits reachable from the source ref but absent from the target ref' },
-    );
+/**
+ * Commits reachable from toRef but not from fromRef, oldest first: the range (fromRef, toRef], so fromRef itself is
+ * not included. To include it, pass its parent, `${sha}~`.
+ */
+export const getCommitsBetween = async (fromRef: string, toRef: string): Promise<Commit[]> => {
+    const output = await runGitCommand(['log', `--pretty=format:${GIT_LOG_FORMAT}`, `${fromRef}..${toRef}`], {
+        operation: 'list commits reachable from the source ref but absent from the target ref',
+    });
     const commitsStrings = output.split('\n');
     const commits = commitsStrings.map((commitString) => parseCommit(commitString));
     commits.reverse();
@@ -124,7 +136,7 @@ export const getCommits = async ({
     baseCommit,
 }: Pick<Config, 'sourceBranch' | 'targetBranch' | 'baseCommit'>): Promise<Commit[]> => {
     const baseCommitSha = parseBaseCommit(baseCommit);
-    const commits = await fetchAllBranchCommits(sourceBranch, targetBranch);
+    const commits = await getCommitsBetween(targetBranch, sourceBranch);
 
     // The last validated (base) commit being the branch HEAD means nothing new was pushed since the last
     // validation — the dev reran the workflow (or force-pushed to the same state) to trigger a clean test
@@ -169,21 +181,25 @@ export const parseCommit = (commitString: string): Commit => {
 };
 
 /**
- * Returns the currently checked-out branch. Release builds point the Actor version at this branch,
+ * Returns the currently checked-out branch. Builds point the Actor version at this branch,
  * so a detached HEAD (no branch to point at) is an error rather than a guess.
  */
 export const getCurrentBranch = async (): Promise<string> => {
     const branch = await runGitCommand(['rev-parse', '--abbrev-ref', 'HEAD'], {
-        operation: 'determine the current branch for release',
+        operation: 'determine the checked-out branch',
     });
     if (branch === 'HEAD') {
         throw new Error(
-            'Cannot determine the branch to release: HEAD is detached. Check out the branch you want to release.',
+            'Cannot determine the branch to build from: HEAD is detached. Check out the branch you want to build.',
         );
     }
     return branch;
 };
 
+/** Resolves a ref (branch, remote branch, SHA, HEAD, ...) to the full SHA of the commit it points to. */
+export const resolveCommitSha = async (ref: string): Promise<string> =>
+    runGitCommand(['rev-parse', '--verify', `${ref}^{commit}`], { operation: `resolve ${ref} to a commit` });
+
 /**
  * Reads the repository URL from the `origin` remote, rewritten to the SSH form the Apify platform
  * uses for Git repo sources, e.g. git@github.com:apify-store/google-maps
@@ -236,15 +252,15 @@ const ZERO_SHA_REGEX = /^0{40}$/;
  * Unlike the PR path (getCommits), there is no lenient fallback. Falling back to "everything"
  * would rebuild the latest build of every Actor and post it to Slack, so every problem is an error.
  */
-export const resolveReleaseBaseCommit = async (baseCommit: string): Promise<string> => {
+export const resolveReleaseBaseCommit = async (baseCommit: string, head = 'HEAD'): Promise<string> => {
     const sha = parseBaseCommit(baseCommit);
     if (!sha) {
-        throw new Error('--base-commit is required for release. See the README section "Releasing Actors".');
+        throw new Error('A base commit is required for release. See the README section "Releasing Actors".');
     }
     if (ZERO_SHA_REGEX.test(sha)) {
         throw new Error(
             `Base commit is ${sha}, which means the branch was just created and there is no previous release to diff against. ` +
-                `Rerun with --base-commit set to the last commit before the changes you want to release.`,
+                `Rerun with the base commit set to the last commit before the changes you want to release.`,
         );
     }
     // --quiet suppresses Git's error text for a missing commit, but Git still exits with code 1.
@@ -261,16 +277,16 @@ export const resolveReleaseBaseCommit = async (baseCommit: string): Promise<stri
         );
     }
     // git merge-base A B outputs the common ancestor. If that equals A, then A is an ancestor of B.
-    const mergeBase = await runGitCommand(['merge-base', verifiedSha, 'HEAD'], {
-        operation: 'check whether the release base commit is an ancestor of HEAD',
+    const mergeBase = await runGitCommand(['merge-base', verifiedSha, head], {
+        operation: `check whether the release base commit is an ancestor of ${head}`,
     }).catch((error: unknown) => {
         if (error instanceof GitCommandError && error.exitCode === 1) return '';
         throw error;
     });
     if (mergeBase !== sha) {
         throw new Error(
-            `Base commit ${sha} is not an ancestor of HEAD, most likely because the branch was force-pushed. ` +
-                `The changed files cannot be determined reliably. Rerun with --base-commit set to an ancestor of HEAD ` +
+            `Base commit ${sha} is not an ancestor of ${head}, most likely because the branch was force-pushed. ` +
+                `The changed files cannot be determined reliably. Rerun with the base commit set to an ancestor of ${head} ` +
                 `from before the changes you want to release.`,
         );
     }
@@ -279,12 +295,16 @@ export const resolveReleaseBaseCommit = async (baseCommit: string): Promise<stri
 
 const CHANGELOG_PATH = 'CHANGELOG.md';
 
-/** Returns the lines added to the root CHANGELOG.md between baseSha and HEAD, or null if it didn't change. */
-const getChangelogAdditions = async (baseSha: string, changedFiles: string[]): Promise<string | null> => {
+/** Returns the lines added to the root CHANGELOG.md between fromRef and toRef, or null if it didn't change. */
+export const getChangelogAdditions = async (
+    fromRef: string,
+    toRef: string,
+    changedFiles: string[],
+): Promise<string | null> => {
     if (!changedFiles.includes(CHANGELOG_PATH)) {
         return null;
     }
-    const diff = await runGitCommand(['diff', baseSha, 'HEAD', '--', CHANGELOG_PATH], {
+    const diff = await runGitCommand(['diff', fromRef, toRef, '--', CHANGELOG_PATH], {
         operation: 'read changelog changes since the last release',
     });
 
@@ -307,23 +327,3 @@ const getChangelogAdditions = async (baseSha: string, changedFiles: string[]): P
     }
     return added.join('\n').trim();
 };
-
-/**
- * Everything that changed since the last release (baseSha, exclusive) up to HEAD, or null when there
- * are no new commits. Changed files come from diffing the range directly instead of from the commit
- * list: with a merge commit, the oldest commit of the merged branch can be older than baseSha, and
- * diffing from its parent would pull in already-released changes.
- */
-export const getReleaseChanges = async (baseSha: string) => {
-    if (
-        (await runGitCommand(['rev-parse', 'HEAD'], { operation: 'identify the current release commit' })) === baseSha
-    ) {
-        return null;
-    }
-    const commits = await fetchAllBranchCommits('HEAD', baseSha);
-    const changedFiles = await runGitCommand(['diff', '--name-only', baseSha, 'HEAD'], {
-        operation: 'list files changed since the last release',
-    }).then((output) => output.split('\n').filter(Boolean));
-    const changelog = await getChangelogAdditions(baseSha, changedFiles);
-    return { commits, changedFiles, changelog };
-};