apify-test-tools 0.9.1-beta.8 → 0.10.0

package/README.md

@@ -12,37 +12,15 @@ npm i -D apify-test-tools
 
 - Requires `vitest` version `3.2.0` or later (uses [annotate](https://vitest.dev/guide/test-context.html#annotate))
 - Make sure `target` and `module` in your `tsconfig.json`'s `compilerOptions` are set to `ES2022`
+- Every Actor must already have at least one build under its default build tag (usually `latest`), built
+  from a Git repository version whose URL points to this repository. Build it once manually on the
+  platform before CI takes over. See [Repository check](#repository-check).
 
 ### 2. Create the config file
 
-Every repo that uses `apify-test-tools` must have an `apify-test-tools.config.json` file at the root. This file tells the tool which actors live in the repo, how to identify them, and which token to use.
+Every repo that uses `apify-test-tools` must have an `apify-test-tools.config.json` file at the root. It lists the actors in the repo, their folders, their full names on the platform, and which env var holds the token to use for each. A top-level `mode` field picks the file's layout; leave it out to use the default `"legacy"` layout.
 
-```json
-{
-    "actors": [
-        {
-            "folder": "actors/web-scraper",
-            "actorFullName": "myteam/web-scraper",
-            "tokenEnvVar": "APIFY_TOKEN_MYTEAM"
-        },
-        {
-            "folder": "actors/email-sender",
-            "actorFullName": "myteam/email-sender",
-            "tokenEnvVar": "APIFY_TOKEN_MYTEAM",
-            "overrideActorContext": ["actors/email-sender", "packages/shared"]
-        }
-    ]
-}
-```
-
-Each entry has:
-
-| Field                  | Required | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
-| ---------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `folder`               | yes      | Relative path from repo root to the actor's own project directory — the folder that directly contains `.actor/actor.json` (i.e. `<folder>/.actor/actor.json`), the actor's README/CHANGELOG, and its source. Use `"."` for a single-actor repo where `.actor/` is at the root.                                                                                                                                                                                                                          |
-| `actorFullName`        | yes      | Full actor identifier in `owner/name` format (e.g. `"apify/web-scraper"`). This is the source of truth for the actor name — the `name` field in `actor.json` is not used.                                                                                                                                                                                                                                                                                                                               |
-| `tokenEnvVar`          | yes      | Name of the environment variable holding the Apify API token for this actor. No fallback — if the env var is not set at build time, the build fails.                                                                                                                                                                                                                                                                                                                                                    |
-| `overrideActorContext` | no       | Array of paths (relative to repo root) that define which files are relevant to this actor. When set, replaces the `dockerContextDir` from `actor.json` for change detection. Useful when an actor depends on shared packages outside its Docker build context. Entries must not be prefixes of one another (e.g. `["", "code"]` or `["actors", "actors/foo"]` are rejected). The actor's own `folder` is always part of its context — if none of the listed entries reach it, it's added automatically. |
+See [Config file](#config-file) for the available modes and every field.
 
 ### 3. Set up actor folders
 
@@ -97,11 +75,87 @@ mkdir -p test/platform/core
 
 See the [GitHub workflows](#github-worklows) section below.
 
+## Config file
+
+`apify-test-tools.config.json` lives at the repo root and tells the tool which actors live in the repo, how to identify them, and which token to use.
+
+### Actor fields
+
+Every actor in the config is described by these fields, whatever the mode:
+
+| Field                  | Required | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+| ---------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `folder`               | yes      | Relative path from repo root to the actor's own project directory — the folder that directly contains `.actor/actor.json` (i.e. `<folder>/.actor/actor.json`), the actor's README/CHANGELOG, and its source. Use `"."` for a single-actor repo where `.actor/` is at the root.                                                                                                                                                                                                                          |
+| `actorFullName`        | yes      | Full actor identifier in `owner/name` format (e.g. `"apify/web-scraper"`). This is the source of truth for the actor name — the `name` field in `actor.json` is not used.                                                                                                                                                                                                                                                                                                                               |
+| `tokenEnvVar`          | yes      | Name of the environment variable holding the Apify API token for this actor. In [grouped mode](#grouped-mode) it's inherited from the group unless the actor sets its own. No fallback — if the env var is not set at build time, the build fails.                                                                                                                                                                                                                                                      |
+| `overrideActorContext` | no       | Array of paths (relative to repo root) that define which files are relevant to this actor. When set, replaces the `dockerContextDir` from `actor.json` for change detection. Useful when an actor depends on shared packages outside its Docker build context. Entries must not be prefixes of one another (e.g. `["", "code"]` or `["actors", "actors/foo"]` are rejected). The actor's own `folder` is always part of its context — if none of the listed entries reach it, it's added automatically. |
+
+### Modes
+
+The file's layout is picked by the top-level `mode` field. Each mode is a different way of writing the same thing: every mode resolves to a list of actors with the fields above, and the same rules apply to all of them — no two actors may share a `folder`, and no two entries may point at the same `actorFullName`. If `mode` is left out, [`"legacy"`](#legacy-mode) is used, so existing config files keep working unchanged.
+
+### Legacy mode
+
+The default. A flat `actors` array where every entry carries all of its settings. Good for repos with a few actors, or where every actor has different settings.
+
+```json
+{
+    "mode": "legacy",
+    "actors": [
+        {
+            "folder": "actors/web-scraper",
+            "actorFullName": "myteam/web-scraper",
+            "tokenEnvVar": "APIFY_TOKEN_MYTEAM"
+        },
+        {
+            "folder": "actors/email-sender",
+            "actorFullName": "myteam/email-sender",
+            "tokenEnvVar": "APIFY_TOKEN_MYTEAM",
+            "overrideActorContext": ["actors/email-sender", "packages/shared"]
+        }
+    ]
+}
+```
+
+`actors` must hold at least one entry. Every entry takes all the [actor fields](#actor-fields) directly, and `tokenEnvVar` is required on each one.
+
+### Grouped mode
+
+Actors are organized into `groups`, and each group sets `tokenEnvVar` and `overrideActorContext` once for all of its actors. Good for repos where many actors share the same token or context.
+
+```json
+{
+    "mode": "grouped",
+    "groups": {
+        "myteam": {
+            "tokenEnvVar": "APIFY_TOKEN_MYTEAM",
+            "overrideActorContext": ["packages/shared"],
+            "actors": [
+                { "folder": "actors/web-scraper", "actorFullName": "myteam/web-scraper" },
+                {
+                    "folder": "actors/email-sender",
+                    "actorFullName": "myteam/email-sender",
+                    "overrideActorContext": ["actors/email-sender", "packages/mailer"]
+                }
+            ]
+        },
+        "partner": {
+            "tokenEnvVar": "APIFY_TOKEN_PARTNER",
+            "actors": [{ "folder": "actors/partner-crawler", "actorFullName": "partner/crawler" }]
+        }
+    }
+}
+```
+
+`groups` is an object with at least one group. The group key (`myteam`, `partner` above) is only a label for readability; it has no effect on the result, and it does not need to match any folder or username. Each group needs a `tokenEnvVar` and at least one actor, and can optionally set `overrideActorContext`.
+
+An actor inside a group can set its own `tokenEnvVar` or `overrideActorContext`, and that value replaces the group's for that actor only. Values are replaced, not merged: in the example above, `myteam/email-sender` gets `["actors/email-sender", "packages/mailer"]`, not `packages/shared` as well.
+
 ## Github worklows
 
 The reusable workflows live in this repo, alongside the package they call. They are the
 `public_`-prefixed files in `.github/workflows`; everything else there is this repo's own CI.
-Reference them at the `@v0` major tag, never at `@master` — see
+Reference them at the `@workflows-v0` major tag, never at `@master` — see
 [Versioning and releases](#versioning-and-releases).
 
 There should be 4 GH workflow files in `.github/workflows`, plus an optional fifth for Claude reviews.
@@ -119,7 +173,7 @@ on:
 
 jobs:
     platformTestsCore:
-        uses: apify/apify-test-tools/.github/workflows/public_platform-tests.yaml@v0
+        uses: apify/apify-test-tools/.github/workflows/public_platform-tests.yaml@workflows-v0
         with:
             subtest: core
         secrets: inherit
@@ -138,7 +192,7 @@ on:
 
 jobs:
     platformTestsDaily:
-        uses: apify/apify-test-tools/.github/workflows/public_platform-tests.yaml@v0
+        uses: apify/apify-test-tools/.github/workflows/public_platform-tests.yaml@workflows-v0
         secrets: inherit
 ```
 
@@ -153,7 +207,7 @@ on:
 
 jobs:
     buildDevelAndTest:
-        uses: apify/apify-test-tools/.github/workflows/public_pr-build-test.yaml@v0
+        uses: apify/apify-test-tools/.github/workflows/public_pr-build-test.yaml@workflows-v0
         secrets: inherit
 ```
 
@@ -168,10 +222,14 @@ on:
 
 jobs:
     buildLatest:
-        uses: apify/apify-test-tools/.github/workflows/public_push-build-latest.yaml@v0
+        uses: apify/apify-test-tools/.github/workflows/public_push-build-latest.yaml@workflows-v0
         secrets: inherit
 ```
 
+This workflow runs `apify-test-tools release --base-commit ${{ github.event.before }}`. See
+[Releasing Actors](#releasing-actors-release) for how the released range is determined and what to
+pass if you call the CLI from somewhere else.
+
 ### `claude-review.yaml`
 
 Optional. Reviews a PR against the shared guidelines when you add the trigger label, and again on
@@ -186,13 +244,13 @@ on:
 
 jobs:
     review:
-        uses: apify/apify-test-tools/.github/workflows/public_review.yaml@v0
+        uses: apify/apify-test-tools/.github/workflows/public_review.yaml@workflows-v0
         secrets: inherit
 ```
 
 The review instructions live in `.github/review-prompt.md` in this repo and are fetched at run time,
 because a reusable workflow doesn't get its own repo checked out. `prompt-ref` selects which ref to
-fetch them from and defaults to `v0`, so the instructions match the workflow you're calling — point
+fetch them from and defaults to `workflows-v0`, so the instructions match the workflow you're calling — point
 it at a branch only to test a prompt change.
 
 ### Secrets
@@ -256,7 +314,7 @@ decide what a consumer repo actually runs:
 
 | Pointer                             | What it selects                                | Moves when                                               |
 | ----------------------------------- | ---------------------------------------------- | -------------------------------------------------------- |
-| the `@v0` tag in `uses:`            | which workflows run                            | a master push, once the version below is published       |
+| the `@workflows-v0` tag in `uses:`  | which workflows run                            | a master push, once the version below is published       |
 | `.github/workflows-package-version` | which `apify-test-tools` the workflows install | a stable release writes it; you may set it ahead of time |
 
 The setup action installs that **exact** version — not a range. A tag is therefore a complete
@@ -270,31 +328,33 @@ action, which is how branch testing works.
 
 Nothing is coupled that doesn't need to be:
 
-- **Workflow-only change** — merge it. `v0` moves, it goes live, no release needed.
+- **Workflow-only change** — merge it. `workflows-v0` moves, it goes live, no release needed.
 - **Package-only change** — merge it, then cut a release when you want it out. The workflows are
   unchanged, so consumers see nothing until the release lands.
 - **A workflow that calls a new CLI feature** — the one case that can break consumers, and the only
   one with any ceremony. Put the package change, the workflow change, and the new pin in one PR. On
-  merge the tag is **held**: the CI job reports that the pinned version isn't on npm and leaves `v0`
+  merge the tag is **held**: the CI job reports that the pinned version isn't on npm and leaves `workflows-v0`
   where it is, so consumers keep running the previous workflows. Cut a stable release, and the tag
   moves on its own. Run **Move major version tag** if you don't want to wait for the next master
   push.
 
-`v0` moving on every master push means `@v0` is as live as `@master` was — there's no staging step,
-just a gate on the package version. What the tag buys you is a `v1` for breaking workflow changes,
-so repos migrate one at a time instead of all at once, and a way to roll back by pointing the tag at
-an earlier commit. Bump `MAJOR_TAG` in `.github/workflows/_move_major_tag.yaml` to cut the next
-major; the old tag then freezes where it is and keeps working.
+`workflows-v0` moving on every master push means `@workflows-v0` is as live as `@master` was —
+there's no staging step, just a gate on the package version. What the tag buys you is a
+`workflows-v1` for breaking workflow changes, so repos migrate one at a time instead of all at once,
+and a way to roll back by pointing the tag at an earlier commit. Bump `MAJOR_TAG` in
+`.github/workflows/_move_major_tag.yaml` to cut the next major; the old tag then freezes where it is
+and keeps working.
 
-Freezing is real, which is the whole reason the version is pinned rather than a floor. A frozen `v0`
+Freezing is real, which is the whole reason the version is pinned rather than a floor. A frozen `workflows-v0`
 points at a commit whose pinned version never changes, so it keeps installing the library it was
 tested against no matter how far the package moves on. Had the workflows installed `>=<floor>`, a
-frozen `v0` would still resolve to whatever is newest — and since the release that enables `v1` is
-usually the same release that breaks `v0`, the migration window would have been zero.
+frozen `workflows-v0` would still resolve to whatever is newest — and since the release that enables
+`workflows-v1` is usually the same release that breaks `workflows-v0`, the migration window would
+have been zero.
 
 The tag tracks the **workflows'** contract, not the npm package version. They move independently on
-purpose, so `@v0` is expected to stay `@v0` after the package reaches 1.0 — bump it when a workflow
-breaks its callers, not when the package does. Because `uses:` cannot take an expression, every ref
+purpose, so `@workflows-v0` is expected to stay `@workflows-v0` after the package reaches 1.0 —
+bump it when a workflow breaks its callers, not when the package does. Because `uses:` cannot take an expression, every ref
 into this repo repeats the tag literally; `check-major-tag-refs.mjs` fails the build if `MAJOR_TAG`
 and those refs disagree, which is the mistake that would otherwise ship silently during a bump.
 
@@ -535,7 +595,7 @@ The main local flow is:
 4. Build Actors on Apify (with your new code)
 5. Run tests against those builds. You can change tests and run on the same builds.
 
-`cd` into the actor repository you want to work with (or use `--workspace`).
+Run every command from the root of the actor repository you want to work with (or point `--workspace` at it).
 
 #### 4. Build affected Actors
 
@@ -545,7 +605,6 @@ Requires `APIFY_TOKEN_<USERNAME>` for all Apify users that own your Actors (e.g.
 
 ```bash
 APIFY_TOKEN_JOHN_DOE=<token> \
-GITHUB_WORKSPACE=. \
   npx apify-test-tools build \
     --target-branch origin/master \
     --source-branch origin/my-dummy-branch \
@@ -560,11 +619,10 @@ Remove `--dry-run` to actually trigger builds and update the branch names/ The c
 
 #### Build from local source (no push needed)
 
-If you don't want to push a dummy branch just to test a change and wait for all the tests to finish, `build-from-local` builds Actors directly from your local files (zipped and uploaded as `SOURCE_FILES`), skipping steps 1-4 above.
+If you don't want to push a dummy branch just to test a change and wait for all the tests to finish, `build-from-local` builds Actors directly from your local files (zipped into the Actor's `actor-<id>-source` key-value store and built as `TARBALL`, like `apify push`), skipping steps 1-4 above.
 
 ```bash
 APIFY_TOKEN_JOHN_DOE=<token> \
-GITHUB_WORKSPACE=. \
   npx apify-test-tools build-from-local --actors john.doe/my-actor
 ```
 
@@ -573,8 +631,7 @@ Pass a hardcoded actor name via `--actors` to build only that Actor (comma-separ
 ```bash
 # Build from local source and capture output
 BUILDS=$(APIFY_TOKEN_JOHN_DOE=apify_api_xxx \
-  GITHUB_WORKSPACE=. \
-  npx apify-test-tools build-from-local --actors apify/my-actor)
+    npx apify-test-tools build-from-local --actors apify/my-actor)
 ```
 
 Since you already scoped the build to just the Actor(s) you care about, point vitest at a specific test file (or a `-t` name filter) instead of the whole `test/platform` directory — you get feedback on that one test without waiting for the full suite to run.
@@ -596,8 +653,7 @@ TESTER_APIFY_TOKEN=<token> \
 ```bash
 # Build and capture output
 BUILDS=$(APIFY_TOKEN_JOHN_DOE=apify_api_xxx \
-  GITHUB_WORKSPACE=. \
-  npx apify-test-tools build \
+    npx apify-test-tools build \
     --target-branch origin/master \
     --source-branch origin/my-dummy-branch)
 
@@ -607,10 +663,67 @@ TESTER_APIFY_TOKEN=apify_api_yyy \
   npx vitest --run --maxConcurrency 20 --fileParallelism=true --maxWorkers 100 test/platform
 ```
 
+### Releasing Actors (`release`)
+
+`release` builds the `latest` (default) version of every Actor changed since the last release and
+posts release notes to Slack. It reads everything else from git: the branch from the checked-out
+`HEAD`, the commits and changed files from the range `<base-commit>..HEAD`, and the repository from
+the `origin` remote.
+
+```bash
+npx apify-test-tools release --base-commit <sha> --release-slack-channel "#releases" --dry-run
+```
+
+#### `--base-commit`
+
+The last commit that is **already released**. Everything after it, up to `HEAD`, is released. It is
+required: git alone can't tell which commits a push brought, and guessing wrong silently leaves Actors
+on old code.
+
+The changed files come from diffing `<base-commit>` and `HEAD` directly, so this works for every
+merge strategy as long as the base commit is the branch tip from **before** the push:
+
+| Merge strategy                 | What lands on the branch          | What to pass                                                           |
+| ------------------------------ | --------------------------------- | ---------------------------------------------------------------------- |
+| Squash and merge               | 1 new commit                      | the push's "before" commit (`HEAD~1` also works)                       |
+| Merge commit                   | the PR's commits + 1 merge commit | the push's "before" commit (`HEAD^1`, the first parent, also works)    |
+| Rebase and merge / direct push | N new commits                     | the push's "before" commit (`HEAD~1` would miss the first N-1 commits) |
+
+On GitHub, the push's "before" commit is `${{ github.event.before }}`, which the reusable workflow
+passes for you. On other CIs use the equivalent, e.g. `$CI_COMMIT_BEFORE_SHA` on GitLab. The checkout
+needs the full history (`fetch-depth: 0` in `actions/checkout`) so the base commit is available locally.
+
+`release` fails instead of guessing when:
+
+- the base commit is all zeros (GitHub sends this for a push that created the branch)
+- the base commit isn't in the local history (shallow checkout) or isn't an ancestor of `HEAD`
+  (the branch was force-pushed)
+
+In those cases, rerun `release` with a `--base-commit` you pick yourself: an ancestor of `HEAD` from
+before the changes you want to release (for a shallow checkout, fetch the full history instead).
+`--actors` only narrows which of the changed Actors get released; it never forces a build, so on its
+own it doesn't get past these errors.
+If `HEAD` is the base commit, there is nothing to release and the command exits successfully.
+
+A failed release isn't retried by the next push: that push's base commit is the failed one's head, so
+its changes are skipped. Re-run the failed workflow run, or release the Actors manually.
+
+#### Repository check
+
+`release` and `build` point each Actor's version at the `origin` remote. Because a build overwrites the
+version's Git URL, building from a fork or a mirror would repoint the Actor. So, before building, the
+remote is compared with the Git URL of the Actor's default version (the one its default build tag
+points to), and the command fails if they differ or if the default version isn't built from a Git
+repository. The URL form doesn't matter (`https://github.com/org/repo` equals
+`git@github.com:org/repo.git`).
+
+To build from a different repository on purpose, e.g. when moving an Actor, pass `--repo-url`. That
+skips the check.
+
 #### Dev mode
 
 For development on `apify-test-tools` itself, use `tsx` directly:
 
 ```bash
-GITHUB_WORKSPACE=local-clone tsx bin/main.ts get-actor-configs
+tsx bin/main.ts --workspace local-clone get-actor-configs
 ```

package/bin/build-from-local.ts

@@ -2,14 +2,16 @@ import fs from 'node:fs/promises';
 import os from 'node:os';
 import path from 'node:path';
 
-import type { ActorVersion, ActorVersionSourceFile } from 'apify-client';
+import AdmZip from 'adm-zip';
+import type * as ApifyClientTypes from 'apify-client';
 import { ActorSourceType } from 'apify-client';
 
 import { dryRunBuildData, LOCAL_SOURCE_VERSION_NUMBER, runAndSummarizeBuilds } from './build.js';
 import { buildDockerIgnoreMatcher } from './dockerignore.js';
 import { isPathWithinScope } from './path-utils.js';
 import type { ActorConfig, BuildData } from './types.js';
-import { getGitignoredPaths, isOutsideDir, listRepoFilePaths, toActorVersionSourceFile } from './utils.js';
+import type { SourceFile } from './utils.js';
+import { getGitignoredPaths, isOutsideDir, listRepoFilePaths, readSourceFile } from './utils.js';
 
 // JUST IN CASE. File patterns that commonly hold credentials — never ship these into a build, regardless
 // of sourceType or of whether the repo's .gitignore happens to list them. Everything else that should be
@@ -18,7 +20,7 @@ import { getGitignoredPaths, isOutsideDir, listRepoFilePaths, toActorVersionSour
 const SKIP_FILE_PATTERNS = [/^\.env(\..+)?$/, /\.pem$/, /\.key$/, /\.pfx$/, /\.p12$/];
 const isSecretFile = (fileName: string): boolean => SKIP_FILE_PATTERNS.some((pattern) => pattern.test(fileName));
 
-export const collectSourceFiles = async (actorName: string, actorDir: string): Promise<ActorVersionSourceFile[]> => {
+export const collectSourceFiles = async (actorName: string, actorDir: string): Promise<SourceFile[]> => {
     const repoRoot = process.cwd();
     const absActorDir = path.resolve(actorDir);
 
@@ -35,9 +37,7 @@ export const collectSourceFiles = async (actorName: string, actorDir: string): P
     const keptFilePaths = collectNonIgnoredFiles(dockerContextDirAbs, repoRoot);
 
     if (!isMonorepoActor) {
-        return Promise.all(
-            keptFilePaths.map(async (filePath) => toActorVersionSourceFile(filePath, dockerContextDirAbs)),
-        );
+        return Promise.all(keptFilePaths.map(async (filePath) => readSourceFile(filePath, dockerContextDirAbs)));
     }
 
     const { tempDir, filePaths } = await flattenMonorepoContext(
@@ -48,7 +48,7 @@ export const collectSourceFiles = async (actorName: string, actorDir: string): P
         keptFilePaths,
     );
     try {
-        return await Promise.all(filePaths.map(async (filePath) => toActorVersionSourceFile(filePath, tempDir)));
+        return await Promise.all(filePaths.map(async (filePath) => readSourceFile(filePath, tempDir)));
     } finally {
         // Only the flattened copy is temporary — never delete the actor's own directory.
         await fs.rm(tempDir, { recursive: true, force: true });
@@ -202,11 +202,15 @@ export const runBuildsFromLocal = async ({
     }
 
     return runAndSummarizeBuilds(actorConfigs, 'LOCAL BUILDS', async (actorConfig, builder) => {
-        const sourceFiles = await collectSourceFiles(actorConfig.actorFullName, actorConfig.folder);
-        const actorVersion: ActorVersion = {
+        // Zipped and uploaded like `apify push` does, since inline SOURCE_FILES are capped at ~9 MB.
+        const zip = new AdmZip();
+        for (const { name, content } of await collectSourceFiles(actorConfig.actorFullName, actorConfig.folder)) {
+            zip.addFile(name, content);
+        }
+        const actorVersion: ApifyClientTypes.ActorVersion = {
             versionNumber: LOCAL_SOURCE_VERSION_NUMBER,
-            sourceFiles,
-            sourceType: ActorSourceType.SourceFiles,
+            tarballUrl: await builder.uploadSourceZip(LOCAL_SOURCE_VERSION_NUMBER, zip.toBuffer()),
+            sourceType: ActorSourceType.Tarball,
         };
         return builder.createVersionAndBuild(LOCAL_SOURCE_VERSION_NUMBER, actorVersion, false);
     });