apify-test-tools 0.9.1-beta.9 → 0.10.0

package/bin/build.ts

@@ -1,6 +1,7 @@
-import type { ActorVersion, Build } from 'apify-client';
+import type * as ApifyClientTypes from 'apify-client';
 import { ActorSourceType, ApifyClient } from 'apify-client';
 
+import { normalizeRepoUrl } from './git.js';
 import type { ActorConfig, BuildData } from './types.js';
 
 type BuildPrActorOptions = {
@@ -8,6 +9,7 @@ type BuildPrActorOptions = {
     versionNumber: string;
     gitRepoUrl: string;
     actorConfig: ActorConfig;
+    actorInfo: ApifyClientTypes.Actor;
     useDockerCache: boolean;
 };
 
@@ -15,69 +17,118 @@ type BuildPrActorOptions = {
 export const LOCAL_SOURCE_VERSION_NUMBER = '0.98';
 const DEFAULT_TEST_VERSION_NUMBER = '0.99';
 
+// Shown whenever an Actor isn't set up the way CI builds need it
+const ACTOR_SETUP_REQUIREMENT =
+    'Before using apify-test-tools, every Actor needs at least one build under its default build tag (usually "latest"), ' +
+    'built from a Git repository version whose URL points to this repository. Build it once manually on the platform.';
+
+/**
+ * Finds the Actor's default version: the one whose build the default build tag points to.
+ * Usually tagged 'latest' but not necessarily (can be e.g. 'version-0').
+ */
+export const resolveDefaultVersion = (actorFullName: string, actorInfo: ApifyClientTypes.Actor) => {
+    const defaultBuildTag = actorInfo.defaultRunOptions.build;
+    console.error(`Default build tag for ${actorFullName} is ${defaultBuildTag}`);
+
+    // We could technically allow this but in most cases this is accidentally set wrongly and there is a workaround
+    if (defaultBuildTag.match(/\d+\.\d+\.\d+/)) {
+        throw new Error(
+            `[${actorFullName}] Default build is a build number, not a tag. While this could work, ` +
+                `we want to have a default as tag so this is often an accidental misconfiguration from the dev`,
+        );
+    }
+    // I reported that buildNumber should probably not be optional
+    const defaultBuildNumber = actorInfo.taggedBuilds?.[defaultBuildTag]?.buildNumber;
+    if (!defaultBuildNumber) {
+        throw new Error(`[${actorFullName}] No build found for tag "${defaultBuildTag}". ${ACTOR_SETUP_REQUIREMENT}`);
+    }
+    const defaultVersionNumber = defaultBuildNumber.match(/(\d+\.\d+)\.\d+/)![1];
+    console.error(`Default version for ${actorFullName} is ${defaultVersionNumber}`);
+
+    const defaultVersion = actorInfo.versions.find((version) => version.versionNumber === defaultVersionNumber);
+
+    return { defaultBuildNumber, defaultVersionNumber, defaultBuildTag, defaultVersion };
+};
+
+/**
+ * Guards against building an Actor from a different repository than it is published from, e.g. when the
+ * local `origin` remote is a fork or a mirror. Every build overwrites the version's Git URL, so building
+ * from the wrong remote would silently repoint the Actor. The default version is the source of truth
+ * since it is what users run. Skipped when the repo URL is passed explicitly, which is how you move an
+ * Actor to a new repository on purpose.
+ */
+export const assertRepoUrlMatchesDefaultVersion = (
+    actorFullName: string,
+    defaultVersion: ApifyClientTypes.ActorVersion | undefined,
+    repoUrl: string,
+) => {
+    if (!defaultVersion) {
+        throw new Error(
+            `[${actorFullName}] The version of the default build no longer exists. ${ACTOR_SETUP_REQUIREMENT}`,
+        );
+    }
+    if (defaultVersion.sourceType !== ActorSourceType.GitRepo) {
+        throw new Error(
+            `[${actorFullName}] Default version ${defaultVersion.versionNumber} has source type "${defaultVersion.sourceType}", ` +
+                `not "${ActorSourceType.GitRepo}". ${ACTOR_SETUP_REQUIREMENT}`,
+        );
+    }
+    if (normalizeRepoUrl(defaultVersion.gitRepoUrl) !== normalizeRepoUrl(repoUrl)) {
+        throw new Error(
+            `[${actorFullName}] Repository mismatch: the git remote "origin" is ${repoUrl} but default version ` +
+                `${defaultVersion.versionNumber} is built from ${defaultVersion.gitRepoUrl}. Fix the remote or the Actor's source, ` +
+                `or pass --repo-url to build from a different repository on purpose.`,
+        );
+    }
+};
+
 export class ApifyBuilder {
     private constructor(
         private readonly apifyClient: ApifyClient,
         private readonly actorFullName: string,
     ) {}
 
-    // Usually 'latest' but not necessarily (can be e.g. 'version-0')
-    getDefaultVersionAndTag = async (): Promise<{
-        defaultBuildNumber: string;
-        defaultVersionNumber: string;
-        defaultBuildTag: string;
-    }> => {
-        const actorClient = this.apifyClient.actor(this.actorFullName);
-        const actorInfo = await actorClient.get();
-
+    getActorInfo = async (): Promise<ApifyClientTypes.Actor> => {
+        const actorInfo = await this.apifyClient.actor(this.actorFullName).get();
         if (!actorInfo) {
             throw new Error(
                 `[${this.actorFullName}] not found. It is not published or we are missing token to access it privately or its name is misspelled`,
             );
         }
+        return actorInfo;
+    };
 
-        const defaultBuildTag = actorInfo.defaultRunOptions.build;
-        console.error(`Default build tag for ${this.actorFullName} is ${defaultBuildTag}`);
+    getDefaultVersionAndTag = async () => resolveDefaultVersion(this.actorFullName, await this.getActorInfo());
 
-        // We could technically allow this but in most cases this is accidentally set wrongly and there is a workaround
-        if (defaultBuildTag.match(/\d+\.\d+\.\d+/)) {
-            throw new Error(
-                `[${this.actorFullName}] Default build is a build number, not a tag. While this could work, ` +
-                    `we want to have a default as tag so this is often an accidental misconfiguration from the dev`,
-            );
-        }
-        // I reported that buildNumber should probably not be optional
-        if (!actorInfo.taggedBuilds?.[defaultBuildTag]?.buildNumber) {
-            throw new Error(
-                `[${this.actorFullName}] No build found for tag "${defaultBuildTag}". ` +
-                    `The first build must be triggered manually on the platform before CI can take over.`,
-            );
-        }
-        const defaultBuildNumber = actorInfo.taggedBuilds![defaultBuildTag].buildNumber!;
-        const defaultVersionNumber = defaultBuildNumber.match(/(\d+\.\d+)\.\d+/)![1];
-        console.error(`Default version for ${this.actorFullName} is ${defaultVersionNumber}`);
+    // Mirrors `apify push`: stores the zip in the Actor's `actor-<id>-source` KV store and returns a signed
+    // record URL the builder can download without a token.
+    uploadSourceZip = async (versionNumber: string, zip: Buffer): Promise<string> => {
+        const actorInfo = await this.apifyClient.actor(this.actorFullName).get();
+        if (!actorInfo) throw new Error(`No actor named '${this.actorFullName}' was found on the platform.`);
+
+        const store = await this.apifyClient.keyValueStores().getOrCreate(`actor-${actorInfo.id}-source`);
+        const storeClient = this.apifyClient.keyValueStore(store.id);
+        const key = `version-${versionNumber}.zip`;
+        await storeClient.setRecord({ key, value: zip as never, contentType: 'application/zip' });
 
-        return { defaultBuildNumber, defaultVersionNumber, defaultBuildTag };
+        const url = new URL(await storeClient.getRecordPublicUrl(key));
+        url.searchParams.set('disableRedirect', 'true');
+        return url.toString();
     };
 
+    // Pass actorInfo when the caller already fetched it, to save an API call
     createVersionAndBuild = async (
         versionNumber: string,
-        actorVersion: ActorVersion,
+        actorVersion: ApifyClientTypes.ActorVersion,
         useCache: boolean,
+        actorInfo?: ApifyClientTypes.Actor,
     ): Promise<BuildData> => {
         const actorClient = this.apifyClient.actor(this.actorFullName);
-        const actorInfo = await actorClient.get();
-        if (!actorInfo) {
-            throw new Error(
-                `No actor named '${this.actorFullName}' was found on the platform. If this` +
-                    ' is unexpected, make sure the actor you are targeting is spelled the' +
-                    ' same as the folder in the repository.',
-            );
-        }
+        const { versions } = actorInfo ?? (await this.getActorInfo());
 
         // Prepare version
-        const versionExists = !actorInfo.versions.find((version) => version.versionNumber === versionNumber);
-        if (versionExists) {
+        const versionMissing = !versions.find((version) => version.versionNumber === versionNumber);
+        if (versionMissing) {
             // create new version
             await actorClient.versions().create(actorVersion);
         } else {
@@ -92,7 +143,7 @@ export class ApifyBuilder {
         return { buildId: id, actorRawId: actId, buildNumber, actorFullName: this.actorFullName };
     };
 
-    waitForBuildToFinish = async (buildId: string): Promise<Build> => {
+    waitForBuildToFinish = async (buildId: string): Promise<ApifyClientTypes.Build> => {
         const build = await this.apifyClient.build(buildId).waitForFinish();
         const versionNumber = build.buildNumber;
         if (build.status === 'FAILED' || build.status === 'TIMED-OUT') {
@@ -152,7 +203,7 @@ export class ApifyBuilder {
         const DEFAULT_DAYS_BACK_PROD_VERSIONS = 30;
         const DEFAULT_DAYS_BACK_DEVEL = 7;
 
-        const actorInfo = (await this.apifyClient.actor(this.actorFullName).get())!;
+        const actorInfo = await this.getActorInfo();
 
         // 'devel' used to be hardcoded for testing version 0.99, once we get rid of this tag everywhere, we can remove this code
         const taggedDevelBuildNumber: string | undefined = actorInfo.taggedBuilds!.devel?.buildNumber;
@@ -167,7 +218,7 @@ export class ApifyBuilder {
         const { items } = await this.apifyClient.actor(this.actorFullName).builds().list();
 
         // Deleting default build throws an error, so we skip it
-        const { defaultBuildNumber, defaultBuildTag } = await this.getDefaultVersionAndTag();
+        const { defaultBuildNumber, defaultBuildTag } = resolveDefaultVersion(this.actorFullName, actorInfo);
 
         const daysAgoUnixProd = Date.now() - DEFAULT_DAYS_BACK_PROD_VERSIONS * 24 * 60 * 60 * 1000;
         const daysAgoUnixDevel = Date.now() - DEFAULT_DAYS_BACK_DEVEL * 24 * 60 * 60 * 1000;
@@ -270,6 +321,8 @@ type RunBuildsOptions = {
     actorConfigs: ActorConfig[];
     isLatest?: boolean;
     repoUrl: string;
+    // False when the repo URL was passed explicitly, see assertRepoUrlMatchesDefaultVersion
+    shouldVerifyRepoUrl: boolean;
     branch: string;
     dryRun: boolean;
     useDockerCache: boolean;
@@ -277,34 +330,40 @@ type RunBuildsOptions = {
 
 export const runBuilds = async ({
     repoUrl,
+    shouldVerifyRepoUrl,
     branch,
     actorConfigs,
     isLatest = false,
     dryRun,
     useDockerCache,
 }: RunBuildsOptions): Promise<BuildData[]> => {
-    const buildConfigs: BuildPrActorOptions[] = [];
-
-    for (const actorConfig of actorConfigs) {
-        let versionNumber: string;
-        let buildTag: string | undefined;
-
-        if (isLatest) {
-            const { defaultVersionNumber, defaultBuildTag } =
-                await ApifyBuilder.fromActorConfig(actorConfig).getDefaultVersionAndTag();
-            versionNumber = defaultVersionNumber;
-            buildTag = defaultBuildTag;
-        } else {
-            versionNumber = DEFAULT_TEST_VERSION_NUMBER;
-        }
+    // Fetched once per Actor and reused for the checks, the version numbers and the build itself
+    const buildConfigs: BuildPrActorOptions[] = await Promise.all(
+        actorConfigs.map(async (actorConfig) => {
+            const actorInfo = await ApifyBuilder.fromActorConfig(actorConfig).getActorInfo();
+            const { defaultVersionNumber, defaultBuildTag, defaultVersion } = resolveDefaultVersion(
+                actorConfig.actorFullName,
+                actorInfo,
+            );
+            if (shouldVerifyRepoUrl) {
+                assertRepoUrlMatchesDefaultVersion(actorConfig.actorFullName, defaultVersion, repoUrl);
+            }
 
-        // Depending on if these are miniactors or standaloneActors
-        let gitRepoUrl = `${repoUrl}#${branch}`;
-        if (actorConfig.folder) {
-            gitRepoUrl = `${gitRepoUrl}:${actorConfig.folder}`;
-        }
-        buildConfigs.push({ actorConfig, gitRepoUrl, versionNumber, buildTag, useDockerCache });
-    }
+            // Depending on if these are miniactors or standaloneActors
+            let gitRepoUrl = `${repoUrl}#${branch}`;
+            if (actorConfig.folder) {
+                gitRepoUrl = `${gitRepoUrl}:${actorConfig.folder}`;
+            }
+            return {
+                actorConfig,
+                actorInfo,
+                gitRepoUrl,
+                versionNumber: isLatest ? defaultVersionNumber : DEFAULT_TEST_VERSION_NUMBER,
+                buildTag: isLatest ? defaultBuildTag : undefined,
+                useDockerCache,
+            };
+        }),
+    );
 
     if (dryRun) {
         console.error('[DRY RUN] Would build:');
@@ -322,18 +381,19 @@ export const runBuilds = async ({
 
     return runAndSummarizeBuilds(actorConfigs, 'BUILDS', async (actorConfig, builder) => {
         const {
+            actorInfo,
             buildTag,
             versionNumber,
             gitRepoUrl,
             useDockerCache: useCache,
         } = buildConfigsByActorFullName.get(actorConfig.actorFullName)!;
-        const actorVersion: ActorVersion = {
+        const actorVersion: ApifyClientTypes.ActorVersion = {
             buildTag,
             versionNumber,
             gitRepoUrl,
             sourceType: ActorSourceType.GitRepo,
         };
-        return builder.createVersionAndBuild(versionNumber, actorVersion, useCache);
+        return builder.createVersionAndBuild(versionNumber, actorVersion, useCache, actorInfo);
     });
 };
 

package/bin/git.ts

@@ -145,3 +145,143 @@ export const parseCommit = (commitString: string): Commit => {
         message,
     };
 };
+
+/**
+ * Returns the currently checked-out branch. Release builds point the Actor version at this branch,
+ * so a detached HEAD (no branch to point at) is an error rather than a guess.
+ */
+export const getCurrentBranch = (): string => {
+    const branch = spawnCommandInGhWorkspace('git rev-parse --abbrev-ref HEAD');
+    if (branch === 'HEAD') {
+        throw new Error(
+            'Cannot determine the branch to release: HEAD is detached. Check out the branch you want to release.',
+        );
+    }
+    return branch;
+};
+
+/**
+ * Reads the repository URL from the `origin` remote, rewritten to the SSH form the Apify platform
+ * uses for Git repo sources, e.g. git@github.com:apify-store/google-maps
+ */
+const getOriginRepoUrl = (): string => {
+    return spawnCommandInGhWorkspace('git remote get-url origin').replace(
+        /^https:\/\/github\.com\//,
+        'git@github.com:',
+    );
+};
+
+/**
+ * Picks the repo URL to build from. By default it's the `origin` remote, and runBuilds then checks it against
+ * each Actor's default version, so a fork or mirror remote can't repoint a published Actor. An explicit
+ * --repo-url skips that check: passing it is how you move an Actor to another repository on purpose.
+ */
+export const resolveRepoUrl = (explicitRepoUrl: string | undefined) => ({
+    repoUrl: explicitRepoUrl ?? getOriginRepoUrl(),
+    shouldVerifyRepoUrl: explicitRepoUrl === undefined,
+});
+
+/**
+ * Makes repo URLs comparable regardless of their form. All of these normalize to `github.com/org/repo`:
+ * - git@github.com:org/repo.git
+ * - https://github.com/org/repo
+ * - ssh://git@github.com/org/repo#master:actors/foo (Actor versions carry a `#branch:folder` fragment)
+ */
+export const normalizeRepoUrl = (repoUrl: string): string => {
+    return repoUrl
+        .trim()
+        .split('#')[0]
+        .replace(/^[a-z+]+:\/\//i, '')
+        .replace(/^[^@/]+@/, '')
+        .replace(':', '/')
+        .replace(/\/+$/, '')
+        .replace(/\.git$/, '')
+        .toLowerCase();
+};
+
+/** Repository name from its URL, e.g. `google-maps` for git@github.com:apify-store/google-maps.git */
+export const getRepoName = (repoUrl: string): string => {
+    return normalizeRepoUrl(repoUrl).split('/').pop()!;
+};
+
+// Sent by e.g. GitHub as the "before" commit of a push that created the branch
+const ZERO_SHA_REGEX = /^0{40}$/;
+
+/**
+ * Validates the base commit of a release: the last commit whose changes are already released.
+ * Unlike the PR path (getCommits), there is no lenient fallback. Falling back to "everything"
+ * would rebuild the latest build of every Actor and post it to Slack, so every problem is an error.
+ */
+export const resolveReleaseBaseCommit = (baseCommit: string): string => {
+    const sha = parseBaseCommit(baseCommit);
+    if (!sha) {
+        throw new Error('--base-commit is required for release. See the README section "Releasing Actors".');
+    }
+    if (ZERO_SHA_REGEX.test(sha)) {
+        throw new Error(
+            `Base commit is ${sha}, which means the branch was just created and there is no previous release to diff against. ` +
+                `Rerun with --base-commit set to the last commit before the changes you want to release.`,
+        );
+    }
+    // --quiet makes rev-parse print nothing (instead of an error) when the commit is missing
+    if (!spawnCommandInGhWorkspace(`git rev-parse --verify --quiet "${sha}^{commit}"`)) {
+        throw new Error(
+            `Base commit ${sha} is not in the local git history. Either the checkout is shallow ` +
+                `(fetch the full history, e.g. fetch-depth: 0 in actions/checkout) or the branch was force-pushed.`,
+        );
+    }
+    // git merge-base A B outputs the common ancestor. If that equals A, then A is an ancestor of B.
+    if (spawnCommandInGhWorkspace(`git merge-base ${sha} HEAD`) !== sha) {
+        throw new Error(
+            `Base commit ${sha} is not an ancestor of HEAD, most likely because the branch was force-pushed. ` +
+                `The changed files cannot be determined reliably. Rerun with --base-commit set to an ancestor of HEAD ` +
+                `from before the changes you want to release.`,
+        );
+    }
+    return sha;
+};
+
+const CHANGELOG_PATH = 'CHANGELOG.md';
+
+/** Returns the lines added to the root CHANGELOG.md between baseSha and HEAD, or null if it didn't change. */
+const getChangelogAdditions = (baseSha: string, changedFiles: string[]): string | null => {
+    if (!changedFiles.includes(CHANGELOG_PATH)) {
+        return null;
+    }
+    const diff = spawnCommandInGhWorkspace('git', ['diff', baseSha, 'HEAD', '--', CHANGELOG_PATH]);
+
+    const added: string[] = [];
+    let startedChangelog = false;
+    for (const line of diff.split('\n')) {
+        // The diff is already limited to the changelog but better to double check
+        if (line.startsWith('+++') && line.toLowerCase().includes(CHANGELOG_PATH.toLowerCase())) {
+            startedChangelog = true;
+            continue;
+        }
+        if (startedChangelog) {
+            if (line.startsWith('diff')) {
+                break;
+            }
+            if (line.startsWith('+')) {
+                added.push(line.slice(1).trim());
+            }
+        }
+    }
+    return added.join('\n').trim();
+};
+
+/**
+ * Everything that changed since the last release (baseSha, exclusive) up to HEAD, or null when there
+ * are no new commits. Changed files come from diffing the range directly instead of from the commit
+ * list: with a merge commit, the oldest commit of the merged branch can be older than baseSha, and
+ * diffing from its parent would pull in already-released changes.
+ */
+export const getReleaseChanges = (baseSha: string) => {
+    if (spawnCommandInGhWorkspace('git rev-parse HEAD') === baseSha) {
+        return null;
+    }
+    const commits = fetchAllBranchCommits('HEAD', baseSha);
+    const changedFiles = spawnCommandInGhWorkspace(`git diff --name-only ${baseSha} HEAD`).split('\n').filter(Boolean);
+    const changelog = getChangelogAdditions(baseSha, changedFiles);
+    return { commits, changedFiles, changelog };
+};