apify-test-tools 0.10.2-beta.0 → 0.10.2-beta.2

package/README.md

@@ -57,7 +57,7 @@ When a PR is opened or code is pushed, the tool determines which actors need to
 2. **CHANGELOG classification** — a `CHANGELOG.md` file is always `cosmetic` (only triggers a release build, not tests), for every actor, regardless of context or folder. (See [issue #106](https://github.com/apify/apify-test-tools/issues/106).)
 3. **Context matching** — the file must fall within one of the actor's context paths (`dockerContextDir` from `actor.json` by default, or `overrideActorContext` from config if set). Files outside every context path are skipped.
 4. **Hardcoded ignore list, context-aware** — the file path is first "hoisted" relative to the context path it matched (e.g. a standalone actor's own `.eslintrc` is checked as just `.eslintrc`, not the full repo-root-relative path), then checked against repo-level dev file patterns (`.vscode/`, `.gitignore`, `.husky/`, `.eslintrc`, `eslint.config.mjs`, `.prettierrc`, `.editorconfig`). There's no hardcoded special-casing for legacy `code/`/`shared/` layouts — repos that need those directories treated as top-level must list them explicitly in `overrideActorContext`.
-5. **`.dockerignore` filtering** — if a `.dockerignore` exists at the root of the actor's `dockerContextDir`, matching files are ignored. Patterns are resolved relative to `dockerContextDir`, matching Docker's own behavior.
+5. **`.dockerignore` filtering** — if a `.dockerignore` exists at the root of the actor's `dockerContextDir`, matching files are ignored. Patterns are resolved relative to `dockerContextDir`, matching Docker's own behavior. Test files are the exception: `build --test-files-glob <glob>` (relative to the repo root) skips this check for matching files, so a changed test triggers a build, and with it its tests, even when `.dockerignore` lists it. The PR workflow passes `test/platform/**`. Context matching still applies, so tests outside an actor's `dockerContextDir` (e.g. a root `test/` with standalone actors) need to be listed in its `overrideActorContext`; `build` logs a warning for changed test files outside every actor's context.
 6. **README classification** — a `README.md` file is `cosmetic` (only triggers a release build, not tests) if it lives inside the actor's own `folder`; otherwise it's ignored entirely, since it isn't documentation for this actor.
 7. **Cosmetic JSON classification** — `.json` files inside the actor's own `.actor/` directory with only cosmetic schema changes (whitespace, key ordering) only trigger a release build.
 8. **Functional** — everything else triggers both build and tests.
@@ -308,48 +308,35 @@ Slack credentials in scope, so a postinstall script in the dependency tree canno
 
 ## Versioning and releases
 
-The workflows and the npm package live in one repo but ship on their own schedules. Two pointers
-decide what a consumer repo actually runs:
+The workflows and the npm package live in one repo but ship on their own schedules:
 
-| Pointer                             | What it selects                                | Moves when                                               |
-| ----------------------------------- | ---------------------------------------------- | -------------------------------------------------------- |
-| the `@workflows-v0` tag in `uses:`  | which workflows run                            | a master push, once the version below is published       |
-| `.github/workflows-package-version` | which `apify-test-tools` the workflows install | a stable release writes it; you may set it ahead of time |
+| What                                         | Consumers get it through                   | Ships when              |
+| -------------------------------------------- | ------------------------------------------ | ----------------------- |
+| the workflows and the CLI they call          | the `@workflows-v0` tag in `uses:`         | every master push       |
+| the lib (`testActor`, `describe`, matchers…) | `apify-test-tools` in their `package.json` | a stable release on npm |
 
-The setup action installs that **exact** version — not a range. A tag is therefore a complete
-statement: these workflows _and_ this library. The stable release writes the file into the same
-commit that bumps `package.json` and `CHANGELOG.md`, so a released commit always names the version
-it published, and rollout latency is unchanged: the release publishes and moves the tag in one run.
+The CLI is bundled with its dependencies into `.github/cli/apify-test-tools.mjs` (`npm run bundle`,
+run by the pre-commit hook and checked in CI), and the setup action runs it from there. Nothing is
+installed for it, so a tag is a complete statement: these workflows and this CLI, with the exact
+dependency versions from `package-lock.json` at that commit. The consumer's own `node_modules` is
+left alone, so their tests keep the lib version they locked.
 
-Prereleases are excluded. Master pushes publish a `-beta` that consumer repos must never install, so
-the pin only moves on a stable release; betas stay reachable through the lockfile path in the setup
-action, which is how branch testing works.
+- **Workflow or CLI change** — merge it. `workflows-v0` moves once checks and tests pass, and it goes
+  live. No release needed, even when a workflow starts using a new CLI feature: both are in the
+  same commit.
+- **Lib change** — merge it, then cut a release when you want it out. Consumers pick it up when
+  they update `apify-test-tools`.
 
-Nothing is coupled that doesn't need to be:
-
-- **Workflow-only change** — merge it. `workflows-v0` moves, it goes live, no release needed.
-- **Package-only change** — merge it, then cut a release when you want it out. The workflows are
-  unchanged, so consumers see nothing until the release lands.
-- **A workflow that calls a new CLI feature** — the one case that can break consumers, and the only
-  one with any ceremony. Put the package change, the workflow change, and the new pin in one PR. On
-  merge the tag is **held**: the CI job reports that the pinned version isn't on npm and leaves `workflows-v0`
-  where it is, so consumers keep running the previous workflows. Cut a stable release, and the tag
-  moves on its own. Run **Move major version tag** if you don't want to wait for the next master
-  push.
+The CLI and the lib in one job can be different versions now, so the formats between them are a
+contract: the `ACTOR_BUILDS` JSON that `build` writes and the lib reads, and the vitest JSON report
+`report-tests` reads. Add fields freely; renaming or removing one needs a new major tag.
 
 `workflows-v0` moving on every master push means `@workflows-v0` is as live as `@master` was —
-there's no staging step, just a gate on the package version. What the tag buys you is a
-`workflows-v1` for breaking workflow changes, so repos migrate one at a time instead of all at once,
-and a way to roll back by pointing the tag at an earlier commit. Bump `MAJOR_TAG` in
+there's no staging step. What the tag buys you is a `workflows-v1` for breaking workflow changes,
+so repos migrate one at a time instead of all at once, and a way to roll back by pointing the tag at
+an earlier commit (**Move major version tag** workflow). Bump `MAJOR_TAG` in
 `.github/workflows/_move_major_tag.yaml` to cut the next major; the old tag then freezes where it is
-and keeps working.
-
-Freezing is real, which is the whole reason the version is pinned rather than a floor. A frozen `workflows-v0`
-points at a commit whose pinned version never changes, so it keeps installing the library it was
-tested against no matter how far the package moves on. Had the workflows installed `>=<floor>`, a
-frozen `workflows-v0` would still resolve to whatever is newest — and since the release that enables
-`workflows-v1` is usually the same release that breaks `workflows-v0`, the migration window would
-have been zero.
+and keeps working, CLI included, since the bundle is part of the frozen commit.
 
 The tag tracks the **workflows'** contract, not the npm package version. They move independently on
 purpose, so `@workflows-v0` is expected to stay `@workflows-v0` after the package reaches 1.0 —
@@ -360,9 +347,8 @@ and those refs disagree, which is the mistake that would otherwise ship silently
 ### Testing workflow changes
 
 - Point [testing-repo-for-github-actions](https://github.com/apify-store/testing-repo-for-github-actions)
-  at your branch (`uses: ...@your-branch`). It has real attached Actors and tests. Because the
-  package lives here too, a master push publishes a `beta`, and the lockfile-beta path in the setup
-  action installs that exact version — so one branch tests both halves of a change together.
+  at your branch (`uses: ...@your-branch`). It has real attached Actors and tests. The branch carries
+  its own CLI bundle, so one branch tests the workflows and the CLI together.
 - To change the composite action itself, repoint the `uses:` refs inside the reusable workflows at
   your branch as well, and change them back before merging.
 - Make sure the shell code actually works on your laptop first.
@@ -580,6 +566,15 @@ describe('standby tests', () => {
 
 The package includes a CLI binary used by CI workflows to build Actors, detect changes, and report test results. You can also run it locally.
 
+### Dry runs (`--dry-run`)
+
+Every command that changes something (`build`, `build-from-local`, `release`, `delete-old-builds`, `report-tests`) takes `--dry-run`. A dry run:
+
+- **Changes nothing.** No versions or builds are created, nothing is uploaded or deleted, and no Slack message is posted.
+- **Does every read and check the real run does**, so it fails wherever the real run would: change detection, the Apify token and Actor lookup, the [repository check](#repository-check), collecting and zipping local sources, and the Slack token (verified with Slack's read-only `auth.test`).
+- **Needs the same tokens as the real run.** Read-only tokens are enough, since nothing is written. A dry run can't tell whether a token is also allowed to write, because the Apify API has no dry-run mode of its own.
+- **Prints a plan, not results.** Logs start with `[DRY RUN] Would …`. `build` and `build-from-local` write `{ "dryRun": true, "builds": [...] }` to stdout instead of the build array, so the plan can't be passed to tests as `ACTOR_BUILDS` by mistake.
+
 ### Running locally
 
 Running the testing library locally is useful when you only want to update the testing code in /test because you can iterate on it without pushing new code to the remote.
@@ -610,7 +605,7 @@ APIFY_TOKEN_JOHN_DOE=<token> \
     --dry-run
 ```
 
-Remove `--dry-run` to actually trigger builds and update the branch names/ The command outputs a JSON array of build objects to stdout:
+With `--dry-run`, it prints the plan (the version and Git URL each Actor would be built from). Remove `--dry-run` to actually trigger builds and update the branch names. The command outputs a JSON array of build objects to stdout:
 
 ```json
 [{ "buildId": "...", "actorRawId": "...", "buildNumber": "...", "actorFullName": "john.doe/my-actor" }]
@@ -625,7 +620,7 @@ APIFY_TOKEN_JOHN_DOE=<token> \
   npx apify-test-tools build-from-local --actors john.doe/my-actor
 ```
 
-Pass a hardcoded actor name via `--actors` to build only that Actor (comma-separate multiple names). Omit `--actors` to build all Actors in the repo, or add `--dry-run` to preview without building. It outputs the same JSON build array as `build`, so you run tests against it the same way as in step 5 below:
+Pass a hardcoded actor name via `--actors` to build only that Actor (comma-separate multiple names). Omit `--actors` to build all Actors in the repo, or add `--dry-run` to check the token, the Actor and the zipped sources without uploading or building. It outputs the same JSON build array as `build`, so you run tests against it the same way as in step 5 below:
 
 ```bash
 # Build from local source and capture output
@@ -649,12 +644,15 @@ TESTER_APIFY_TOKEN=<token> \
 
 #### Full example
 
+Tests only run for Actors in `ACTOR_BUILDS`, so a custom workflow has to build (and pass) the Actors even when only tests changed. Pass `--test-files-glob` so test changes count even when `.dockerignore` lists them.
+
 ```bash
 # Build and capture output
 BUILDS=$(APIFY_TOKEN_JOHN_DOE=apify_api_xxx \
     npx apify-test-tools build \
     --target-branch origin/master \
-    --source-branch origin/my-dummy-branch)
+    --source-branch origin/my-dummy-branch \
+    --test-files-glob 'test/platform/**')
 
 # Run tests with the builds
 ACTOR_BUILDS="$BUILDS" \

package/bin/build-from-local.ts

@@ -6,11 +6,11 @@ import AdmZip from 'adm-zip';
 import type * as ApifyClientTypes from 'apify-client';
 import { ActorSourceType } from 'apify-client';
 
-import { dryRunBuildData, LOCAL_SOURCE_VERSION_NUMBER, runAndSummarizeBuilds } from './build.js';
+import { ApifyBuilder, LOCAL_SOURCE_VERSION_NUMBER, runAndSummarizeBuilds } from './build.js';
 import { loadDockerIgnore } from './dockerignore.js';
 import { logger } from './logger.js';
 import { isPathWithinScope } from './path-utils.js';
-import type { ActorConfig, BuildData } from './types.js';
+import type { ActorConfig, BuildData, DryRunPlan } from './types.js';
 import type { SourceFile } from './utils.js';
 import { getGitignoredPaths, isOutsideDir, listRepoFilePaths, readSourceFile } from './utils.js';
 import type { RelativeDir } from './utils/path/repo-relative.js';
@@ -179,32 +179,66 @@ export const rewriteActorJsonPaths = async (
     await fs.writeFile(path.join(newActorDir, 'actor.json'), JSON.stringify(rewritten, null, 4));
 };
 
+type LocalBuildPlan = {
+    actorFullName: string;
+    versionNumber: string;
+    folder: string;
+    fileCount: number;
+    zipSizeBytes: number;
+};
+
 export const runBuildsFromLocal = async ({
     actorConfigs,
     dryRun,
 }: {
     actorConfigs: ActorConfig[];
     dryRun: boolean;
-}): Promise<BuildData[]> => {
+}): Promise<BuildData[] | DryRunPlan<LocalBuildPlan>> => {
+    // Read-only preparation shared with the real build, so a dry run fails on a missing token, an unknown
+    // Actor or unreadable sources just like the build would
+    const preparedBuilds = await Promise.all(
+        actorConfigs.map(async (actorConfig) => {
+            const actorInfo = await ApifyBuilder.fromActorConfig(actorConfig).getActorInfo();
+            // Zipped and uploaded like `apify push` does, since inline SOURCE_FILES are capped at ~9 MB.
+            const zip = new AdmZip();
+            const sourceFiles = await collectSourceFiles(actorConfig);
+            for (const { name, content } of sourceFiles) {
+                zip.addFile(name, content);
+            }
+            return { actorConfig, actorInfo, fileCount: sourceFiles.length, zipBuffer: zip.toBuffer() };
+        }),
+    );
+
     if (dryRun) {
         logger.info('[DRY RUN] Would build from local source:');
-        for (const { actorFullName, folder } of actorConfigs) {
-            logger.info(`  ${actorFullName} (${folder})`);
+        for (const { actorConfig, fileCount, zipBuffer } of preparedBuilds) {
+            logger.info(
+                `  ${actorConfig.actorFullName} (${actorConfig.folder}): ${fileCount} files, ${zipBuffer.length} bytes zipped`,
+            );
         }
-        return actorConfigs.map(({ actorFullName }) => dryRunBuildData(actorFullName, LOCAL_SOURCE_VERSION_NUMBER));
+        return {
+            dryRun: true,
+            builds: preparedBuilds.map(({ actorConfig, fileCount, zipBuffer }) => ({
+                actorFullName: actorConfig.actorFullName,
+                versionNumber: LOCAL_SOURCE_VERSION_NUMBER,
+                folder: actorConfig.folder.path,
+                fileCount,
+                zipSizeBytes: zipBuffer.length,
+            })),
+        };
     }
 
+    const preparedByActorFullName = new Map(
+        preparedBuilds.map((prepared) => [prepared.actorConfig.actorFullName, prepared]),
+    );
+
     return runAndSummarizeBuilds(actorConfigs, 'LOCAL BUILDS', async (actorConfig, builder) => {
-        // Zipped and uploaded like `apify push` does, since inline SOURCE_FILES are capped at ~9 MB.
-        const zip = new AdmZip();
-        for (const { name, content } of await collectSourceFiles(actorConfig)) {
-            zip.addFile(name, content);
-        }
+        const { actorInfo, zipBuffer } = preparedByActorFullName.get(actorConfig.actorFullName)!;
         const actorVersion: ApifyClientTypes.ActorVersion = {
             versionNumber: LOCAL_SOURCE_VERSION_NUMBER,
-            tarballUrl: await builder.uploadSourceZip(LOCAL_SOURCE_VERSION_NUMBER, zip.toBuffer()),
+            tarballUrl: await builder.uploadSourceZip(LOCAL_SOURCE_VERSION_NUMBER, zipBuffer),
             sourceType: ActorSourceType.Tarball,
         };
-        return builder.createVersionAndBuild(LOCAL_SOURCE_VERSION_NUMBER, actorVersion, false);
+        return builder.createVersionAndBuild(LOCAL_SOURCE_VERSION_NUMBER, actorVersion, false, actorInfo);
     });
 };

package/bin/build.ts

@@ -3,7 +3,7 @@ import { ActorSourceType, ApifyClient } from 'apify-client';
 
 import { normalizeRepoUrl } from './git.js';
 import { logger } from './logger.js';
-import type { ActorConfig, BuildData } from './types.js';
+import type { ActorConfig, BuildData, DryRunPlan } from './types.js';
 
 type BuildPrActorOptions = {
     buildTag?: string;
@@ -179,7 +179,7 @@ export class ApifyBuilder {
      * Deletes build of all versions. Apify API doesn't allow to delete default build and we explicitly skip it
      * We delete devel builds faster because we used the for every PR until recently so just to get rid of them faster
      */
-    async deleteOldBuilds(): Promise<void> {
+    async deleteOldBuilds({ dryRun }: { dryRun: boolean }): Promise<void> {
         // Even though we don't version our current Actors, if we ever such Actors to GitHub CI, we would accidentally delete old supported versions
         // This hardcoded solution is not ideal, but it should prevent most imaginable cases
         // All currently popular versioned Actors use `version-${number}` format
@@ -258,9 +258,15 @@ export class ApifyBuilder {
         });
 
         logger.info(
-            `[DELETE OLD BUILDS][${this.actorFullName}]: Deleting ${buildsToDelete.length} old builds that are non-default and ` +
-                `older than 30 days from total ${items.length}`,
+            `[DELETE OLD BUILDS][${this.actorFullName}]: ${dryRun ? '[DRY RUN] Would delete' : 'Deleting'} ${buildsToDelete.length} ` +
+                `old builds that are non-default and older than 30 days from total ${items.length}`,
         );
+        if (dryRun) {
+            for (const build of buildsToDelete) {
+                logger.info(`  ${build.buildNumber} (${build.id}, started ${build.startedAt.toISOString()})`);
+            }
+            return;
+        }
         for (const build of buildsToDelete) {
             await this.apifyClient.build(build.id).delete();
         }
@@ -310,13 +316,13 @@ export const runAndSummarizeBuilds = async (
     return waitAndSummarizeBuilds(startedBuilds, buildersByActorFullName, label);
 };
 
-// Placeholder BuildData for dry runs, since no real build was triggered.
-export const dryRunBuildData = (actorFullName: string, versionNumber: string): BuildData => ({
-    buildId: 'dry-run',
-    actorRawId: 'dry-run',
-    buildNumber: versionNumber,
-    actorFullName,
-});
+type GitBuildPlan = {
+    actorFullName: string;
+    versionNumber: string;
+    buildTag?: string;
+    gitRepoUrl: string;
+    useDockerCache: boolean;
+};
 
 type RunBuildsOptions = {
     actorConfigs: ActorConfig[];
@@ -337,7 +343,7 @@ export const runBuilds = async ({
     isLatest = false,
     dryRun,
     useDockerCache,
-}: RunBuildsOptions): Promise<BuildData[]> => {
+}: RunBuildsOptions): Promise<BuildData[] | DryRunPlan<GitBuildPlan>> => {
     // Fetched once per Actor and reused for the checks, the version numbers and the build itself
     const buildConfigs: BuildPrActorOptions[] = await Promise.all(
         actorConfigs.map(async (actorConfig) => {
@@ -366,14 +372,24 @@ export const runBuilds = async ({
         }),
     );
 
+    // Everything above only reads, so a dry run has checked what the real run would, short of writing
     if (dryRun) {
         logger.info('[DRY RUN] Would build:');
-        for (const { actorConfig, versionNumber } of buildConfigs) {
-            logger.info(`  ${actorConfig.actorFullName} (${versionNumber})`);
+        for (const { actorConfig, versionNumber, gitRepoUrl } of buildConfigs) {
+            logger.info(`  ${actorConfig.actorFullName} (${versionNumber}) from ${gitRepoUrl}`);
         }
-        return buildConfigs.map(({ actorConfig, versionNumber }) =>
-            dryRunBuildData(actorConfig.actorFullName, versionNumber),
-        );
+        return {
+            dryRun: true,
+            builds: buildConfigs.map(
+                ({ actorConfig, versionNumber, buildTag, gitRepoUrl, useDockerCache: useCache }) => ({
+                    actorFullName: actorConfig.actorFullName,
+                    versionNumber,
+                    buildTag,
+                    gitRepoUrl,
+                    useDockerCache: useCache,
+                }),
+            ),
+        };
     }
 
     const buildConfigsByActorFullName = new Map(
@@ -398,8 +414,8 @@ export const runBuilds = async ({
     });
 };
 
-export const deleteOldBuilds = async (actorConfigs: ActorConfig[]) => {
+export const deleteOldBuilds = async (actorConfigs: ActorConfig[], { dryRun }: { dryRun: boolean }) => {
     for (const actorConfig of actorConfigs) {
-        await ApifyBuilder.fromActorConfig(actorConfig).deleteOldBuilds();
+        await ApifyBuilder.fromActorConfig(actorConfig).deleteOldBuilds({ dryRun });
     }
 };

package/bin/diff-changes.ts

@@ -1,3 +1,5 @@
+import picomatch from 'picomatch';
+
 import { isCosmeticOnlyJsonSchemaChange } from './diff-json-schema.js';
 import { type DockerIgnoreMatcher, loadDockerIgnore } from './dockerignore.js';
 import { logger } from './logger.js';
@@ -9,6 +11,8 @@ interface ShouldBuildAndTestOptions {
     actorConfigs: ActorConfig[];
     isLatest?: boolean;
     commits: Commit[];
+    // Glob relative to the repo root. Matching files count as changes even when .dockerignore lists them.
+    testFilesGlob?: string;
 }
 
 const IGNORED_TOP_LEVEL_FILES = [
@@ -41,7 +45,7 @@ type FileChangeForActor =
  * 2. Context matching (actorConfig.contextPaths) → outside-context if no match
  * 3. Hardcoded ignore list, checked against the path hoisted relative to the matched context entry → ignored
  * 4. .dockerignore filtering (patterns relative to dockerContextDir), skipped for the actor's own `.actor/`
- *    dir → ignored if matched
+ *    dir and for test files → ignored if matched
  * 5. README.md by filename → cosmetic if inside the actor's own folder, otherwise ignored
  * 6. .json inside the actor's own `.actor/` dir with only cosmetic schema diffs → cosmetic (semantically verified)
  * 7. Everything else → functional
@@ -51,6 +55,7 @@ const classifyFileChange = async (
     actorConfig: ActorConfig,
     commits: Commit[],
     dockerIgnoreMatcher: DockerIgnoreMatcher,
+    isTestFile: (filePath: string) => boolean,
 ): Promise<FileChangeForActor> => {
     const lowercaseFilePath = changedFile.path.toLowerCase();
 
@@ -75,7 +80,8 @@ const classifyFileChange = async (
     // .actor/ can legitimately be listed in .dockerignore (the Apify platform evaluates it before
     // the Docker build, so excluding it from the build context is a valid caching optimization) —
     // that shouldn't cause changes to .actor/ itself to be ignored here.
-    if (!isUnderActorDotDir && dockerIgnoreMatcher(changedFile.path)) {
+    // Test files are often in .dockerignore too, but a change to them must still build and test.
+    if (!isUnderActorDotDir && !isTestFile(changedFile.path) && dockerIgnoreMatcher(changedFile.path)) {
         return { impact: 'ignored' };
     }
 
@@ -163,8 +169,11 @@ export const getChangedActors = async ({
     actorConfigs,
     isLatest = false,
     commits,
+    testFilesGlob,
 }: ShouldBuildAndTestOptions): Promise<ActorConfig[]> => {
     const actorsChangedMap = new Map<string, ActorChangeEntry>();
+    const testFilesInContext = new Set<string>();
+    const isTestFile = testFilesGlob ? picomatch(testFilesGlob, { nocase: true, dot: true }) : () => false;
 
     for (const actorConfig of actorConfigs) {
         const dockerIgnoreMatcher = loadDockerIgnore(actorConfig.dockerContextDir);
@@ -176,7 +185,10 @@ export const getChangedActors = async ({
                 continue;
             }
 
-            const change = await classifyFileChange(filePath, actorConfig, commits, dockerIgnoreMatcher);
+            const change = await classifyFileChange(filePath, actorConfig, commits, dockerIgnoreMatcher, isTestFile);
+            if (change.impact !== 'outside-context' && isTestFile(originalFilePath)) {
+                testFilesInContext.add(originalFilePath);
+            }
 
             if (change.impact === 'ignored' || change.impact === 'outside-context') continue;
             if (change.impact === 'cosmetic' && !isLatest) continue;
@@ -195,6 +207,15 @@ export const getChangedActors = async ({
     const groups = groupFilesByActorSet(fileToActors);
     logChangeGroups(groups);
 
+    for (const testFile of filepathsChanged.filter(
+        (filePath) => isTestFile(filePath) && !testFilesInContext.has(filePath),
+    )) {
+        logger.warn(
+            `[DIFF]: Test file ${testFile} is outside every Actor's context, so changing only it builds and tests nothing. ` +
+                `Add its folder to "overrideActorContext" of the Actors it tests.`,
+        );
+    }
+
     if (actorsChanged.length > 0) {
         const actors = actorsChanged.map((config) => config.actorFullName);
         logger.info(`[DIFF]: Actors to be built and tested: ${actors.join(', ')}`);

package/bin/main.ts

@@ -64,6 +64,19 @@ const actorSelectionOptions = {
     ignore: { type: 'string', array: true, default: [] as string[] },
 } as const satisfies Record<string, Options>;
 
+/**
+ * For commands that change something (builds, deletes, Slack posts). A dry run still does every read and check
+ * the real run does, credentials included, and only skips the writes. So it needs the same tokens, though
+ * read-only ones are enough. Read-only commands don't take it since they have nothing to skip.
+ */
+const dryRunOptions = {
+    'dry-run': {
+        type: 'boolean',
+        default: false,
+        describe: 'Do every read and check (tokens included) but change nothing; print what would be done',
+    },
+} as const satisfies Record<string, Options>;
+
 /** Flags for commands that trigger Apify builds. */
 const buildOptions = {
     'use-docker-cache': { type: 'boolean', default: false },
@@ -77,7 +90,6 @@ const globalOptions = {
         default: 'info',
         describe: 'Minimum diagnostic level written to stderr; silent disables logging',
     },
-    'dry-run': { type: 'boolean', default: false },
 } as const satisfies Record<string, Options>;
 
 const resolveChangedActors = async (config: Config, { isLatest }: { isLatest: boolean }) => {
@@ -99,6 +111,7 @@ const resolveChangedActors = async (config: Config, { isLatest }: { isLatest: bo
             filepathsChanged: branchOnlyFiles,
             actorConfigs,
             commits: allBranchCommits,
+            testFilesGlob: config.testFilesGlob,
         });
         if (branchOnlyActorsChanged.length === 0) {
             logger.info('[MERGE-FROM-TARGET-OPTIMIZATION]: Branch itself has no functional changes, skipping tests');
@@ -112,7 +125,13 @@ const resolveChangedActors = async (config: Config, { isLatest }: { isLatest: bo
     // If the optimization doesn't apply, we check all branch commits including merges for full coverage. We don't reuse the merge optimization results because here we can apply baseCommit and check merge commits (they might be functional or just cosmetic)
     const commits = await getCommits(config);
     const changedFiles = await getChangedFiles(commits);
-    return getChangedActors({ filepathsChanged: changedFiles, actorConfigs, isLatest, commits });
+    return getChangedActors({
+        filepathsChanged: changedFiles,
+        actorConfigs,
+        isLatest,
+        commits,
+        testFilesGlob: config.testFilesGlob,
+    });
 };
 
 await yargs()
@@ -172,6 +191,7 @@ await yargs()
         '',
         (y) =>
             y
+                .options(dryRunOptions)
                 .option('report-file', { type: 'string', demandOption: true })
                 .option('report-slack-channel', { type: 'string' })
                 .option('job-url', { type: 'string' })
@@ -183,7 +203,14 @@ await yargs()
     .command(
         'build',
         '',
-        (y) => y.options(gitRangeOptions).options(actorSelectionOptions).options(buildOptions).options(repoUrlOptions),
+        (y) =>
+            y
+                .options(gitRangeOptions)
+                .options(actorSelectionOptions)
+                .options(buildOptions)
+                .options(repoUrlOptions)
+                .options(dryRunOptions)
+                .option('test-files-glob', { type: 'string' }),
         async (config) => {
             const actorsChanged = await resolveChangedActors(config, { isLatest: false });
             const builds = await runBuilds({
@@ -204,6 +231,7 @@ await yargs()
                 .options(actorSelectionOptions)
                 .options(buildOptions)
                 .options(repoUrlOptions)
+                .options(dryRunOptions)
                 // The last commit that is already released, everything after it up to HEAD gets released.
                 // Required, see the README section "Releasing Actors" for how to set it for each merge strategy.
                 .option('base-commit', { type: 'string', demandOption: true })
@@ -256,7 +284,7 @@ await yargs()
     .command(
         'build-from-local',
         '',
-        (y) => y.options(actorSelectionOptions),
+        (y) => y.options(actorSelectionOptions).options(dryRunOptions),
         async ({ actors, ignore, dryRun }) => {
             const actorConfigs = await readConfigFile({ actors, ignore });
             const builds = await runBuildsFromLocal({ actorConfigs, dryRun });
@@ -266,10 +294,10 @@ await yargs()
     .command(
         'delete-old-builds',
         '',
-        (y) => y.options(actorSelectionOptions),
-        async ({ actors, ignore }) => {
+        (y) => y.options(actorSelectionOptions).options(dryRunOptions),
+        async ({ actors, ignore, dryRun }) => {
             const actorConfigs = await readConfigFile({ actors, ignore });
-            await deleteOldBuilds(actorConfigs);
+            await deleteOldBuilds(actorConfigs, { dryRun });
         },
     )
     .strictCommands()

package/bin/slack.ts

@@ -15,6 +15,25 @@ type NotifyToSlackOptions = {
     reportSlackChannel?: string;
 };
 
+/**
+ * Creates the client the real run posts with. A dry run posts nothing but still requires the token and checks
+ * Slack accepts it (`auth.test` is read-only), so it fails wherever the real run would.
+ */
+export const getSlackClient = async (tokenEnvVar: string, dryRun: boolean): Promise<WebClient> => {
+    const slack = new WebClient(getEnvVar(tokenEnvVar));
+    if (dryRun) {
+        try {
+            await slack.auth.test();
+        } catch (err) {
+            throw new Error(`${tokenEnvVar} was rejected by Slack: ${(err as Error).message}`);
+        }
+        logger.info(`[DRY RUN] ${tokenEnvVar} is valid`);
+    }
+    return slack;
+};
+
+export const sendingLabel = (dryRun: boolean) => (dryRun ? '[DRY RUN] Would send' : 'Sending');
+
 // Git authors come as "Name<email>", Slack only needs the name
 const displayName = (author: string) => author.replace(/\s*<[^>]*>$/, '');
 
@@ -28,19 +47,23 @@ export const notifyToSlack = async ({
     releaseSlackChannel,
     reportSlackChannel,
 }: NotifyToSlackOptions) => {
-    const slack = new WebClient(getEnvVar('SLACK_TOKEN_RELEASES_BOT'));
-
     if (!changelog) {
         logger.warn('No new changelog entries found, did you forget to update it?');
     }
 
+    if (!releaseSlackChannel && !reportSlackChannel) {
+        logger.info('Skipping slack notification, no --release-slack-channel or --report-slack-channel passed');
+        return;
+    }
+    const slack = await getSlackClient('SLACK_TOKEN_RELEASES_BOT', dryRun);
+
     let shortMessage = `*${repository}* – New release (by ${displayName(author)}):\n\n`;
 
     // This one is just for broader public that only cares about public facing changes
     if (changelog && releaseSlackChannel) {
         shortMessage += `*Additions to the changelog*:\n\n${changelog}\n`;
         logger.info(`=========================================`);
-        logger.info(`**Sending slack message to channel**: ${releaseSlackChannel}.\n\n${shortMessage}`);
+        logger.info(`**${sendingLabel(dryRun)} slack message to channel**: ${releaseSlackChannel}.\n\n${shortMessage}`);
         logger.info(`=========================================`);
         if (!dryRun) {
             await slack.chat.postMessage({
@@ -62,7 +85,7 @@ export const notifyToSlack = async ({
     // This one is for devs and project managers that need to know more details
     if (reportSlackChannel) {
         logger.info(`=========================================`);
-        logger.info(`Sending slack message to channel: ${reportSlackChannel}.\n\n${longMessage}`);
+        logger.info(`${sendingLabel(dryRun)} slack message to channel: ${reportSlackChannel}.\n\n${longMessage}`);
         logger.info(`=========================================`);
         if (!dryRun) {
             await slack.chat.postMessage({
@@ -73,8 +96,7 @@ export const notifyToSlack = async ({
     }
 };
 
-export const sendSlackMessage = async (channel: string, text: string, blocks: string[], token: string) => {
-    const slack = new WebClient(token);
+export const sendSlackMessage = async (slack: WebClient, channel: string, text: string, blocks: string[]) => {
     const { ts } = await slack.chat.postMessage({ text, channel });
     if (blocks.length > 0 && ts) {
         await slack.chat.postMessage({

package/bin/test-report.ts

@@ -1,8 +1,7 @@
 import fs from 'node:fs/promises';
 
 import { logger } from './logger.js';
-import { sendSlackMessage } from './slack.js';
-import { getEnvVar } from './utils.js';
+import { getSlackClient, sendingLabel, sendSlackMessage } from './slack.js';
 
 interface ReportTestResultsOptions {
     reportFile: string;
@@ -80,6 +79,9 @@ export const reportTestResults = async ({
         );
         return;
     }
+    // Checked before knowing if there's anything to send, so a broken token shows up on a green run
+    // (and in dry runs) rather than on the day tests fail
+    const slack = await getSlackClient('SLACK_TOKEN_TESTS_BOT', dryRun);
 
     if (failedAssertions.length === 0) {
         return;
@@ -95,16 +97,11 @@ export const reportTestResults = async ({
         .slice(1)
         .map(({ message, runLink, actorId }) => `• ${message} --- <${runLink}|${actorId}>`);
 
-    logger.info('SLACK:', slackMessage);
+    logger.info(`${sendingLabel(dryRun)} slack message to channel ${reportSlackChannel}:`, slackMessage);
     logger.info('\tblocks:', blocks.join('\n\t\t'));
 
-    if (!reportSlackChannel) {
-        return;
-    }
-
     if (!dryRun) {
-        const slackToken = getEnvVar('SLACK_TOKEN_TESTS_BOT');
-        await sendSlackMessage(reportSlackChannel, slackMessage, blocks, slackToken);
+        await sendSlackMessage(slack, reportSlackChannel, slackMessage, blocks);
     }
 };