apify-test-tools 0.10.2-beta.1 → 0.10.2-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (187) hide show
  1. package/.gitattributes +2 -0
  2. package/.github/actions/checkout-restore-dependencies/action.yaml +64 -27
  3. package/.github/cli/apify-test-tools.mjs +360 -0
  4. package/.github/scripts/check-major-tag-refs.mjs +1 -1
  5. package/.github/scripts/mask-config-secrets.sh +17 -0
  6. package/.github/scripts/run-with-apify-tokens.mjs +37 -12
  7. package/.github/workflows/_check_code.yaml +25 -0
  8. package/.github/workflows/_move_major_tag.yaml +3 -81
  9. package/.github/workflows/_tests.yaml +34 -0
  10. package/.github/workflows/_update_release_metadata.yaml +0 -12
  11. package/.github/workflows/manual_move_major_tag.yaml +3 -3
  12. package/.github/workflows/manual_release_stable.yaml +3 -13
  13. package/.github/workflows/on_master.yaml +9 -5
  14. package/.github/workflows/on_pull_request.yaml +3 -48
  15. package/.github/workflows/public_platform-tests-claude-investigate-and-fix.yaml +13 -2
  16. package/.github/workflows/public_platform-tests.yaml +24 -1
  17. package/.github/workflows/public_pr-build-test.yaml +61 -25
  18. package/.github/workflows/public_push-build-latest.yaml +41 -8
  19. package/.husky/pre-commit +5 -0
  20. package/.prettierignore +3 -0
  21. package/CHANGELOG.md +16 -0
  22. package/CONTRIBUTING.md +105 -21
  23. package/README.md +218 -600
  24. package/bin/affected-actors.ts +198 -0
  25. package/bin/build-from-local.ts +65 -13
  26. package/bin/build.ts +123 -41
  27. package/bin/diff-changes.ts +32 -23
  28. package/bin/diff-json-schema.ts +5 -6
  29. package/bin/git.ts +69 -99
  30. package/bin/main.ts +96 -135
  31. package/bin/slack.ts +43 -10
  32. package/bin/stdin-input.ts +65 -0
  33. package/bin/test-report.ts +6 -9
  34. package/bin/types.ts +15 -0
  35. package/dist/bin/affected-actors.d.ts +46 -0
  36. package/dist/bin/affected-actors.d.ts.map +1 -0
  37. package/dist/bin/affected-actors.js +135 -0
  38. package/dist/bin/affected-actors.js.map +1 -0
  39. package/dist/bin/build-from-local.d.ts +12 -3
  40. package/dist/bin/build-from-local.d.ts.map +1 -1
  41. package/dist/bin/build-from-local.js +34 -12
  42. package/dist/bin/build-from-local.js.map +1 -1
  43. package/dist/bin/build.d.ts +30 -7
  44. package/dist/bin/build.d.ts.map +1 -1
  45. package/dist/bin/build.js +85 -26
  46. package/dist/bin/build.js.map +1 -1
  47. package/dist/bin/diff-changes.d.ts +11 -4
  48. package/dist/bin/diff-changes.d.ts.map +1 -1
  49. package/dist/bin/diff-changes.js +19 -19
  50. package/dist/bin/diff-changes.js.map +1 -1
  51. package/dist/bin/diff-json-schema.d.ts +2 -2
  52. package/dist/bin/diff-json-schema.d.ts.map +1 -1
  53. package/dist/bin/diff-json-schema.js +1 -4
  54. package/dist/bin/diff-json-schema.js.map +1 -1
  55. package/dist/bin/git.d.ts +23 -21
  56. package/dist/bin/git.d.ts.map +1 -1
  57. package/dist/bin/git.js +60 -83
  58. package/dist/bin/git.js.map +1 -1
  59. package/dist/bin/main.js +82 -105
  60. package/dist/bin/main.js.map +1 -1
  61. package/dist/bin/slack.d.ts +14 -4
  62. package/dist/bin/slack.d.ts.map +1 -1
  63. package/dist/bin/slack.js +39 -6
  64. package/dist/bin/slack.js.map +1 -1
  65. package/dist/bin/stdin-input.d.ts +17 -0
  66. package/dist/bin/stdin-input.d.ts.map +1 -0
  67. package/dist/bin/stdin-input.js +54 -0
  68. package/dist/bin/stdin-input.js.map +1 -0
  69. package/dist/bin/test-report.d.ts.map +1 -1
  70. package/dist/bin/test-report.js +6 -8
  71. package/dist/bin/test-report.js.map +1 -1
  72. package/dist/bin/types.d.ts +13 -0
  73. package/dist/bin/types.d.ts.map +1 -1
  74. package/dist/lib/extend-expect.d.ts.map +1 -1
  75. package/dist/lib/extend-expect.js +37 -218
  76. package/dist/lib/extend-expect.js.map +1 -1
  77. package/dist/lib/lib.d.ts +1 -2
  78. package/dist/lib/lib.d.ts.map +1 -1
  79. package/dist/lib/lib.js +15 -22
  80. package/dist/lib/lib.js.map +1 -1
  81. package/dist/lib/run-checks.d.ts +14 -0
  82. package/dist/lib/run-checks.d.ts.map +1 -0
  83. package/dist/lib/run-checks.js +82 -0
  84. package/dist/lib/run-checks.js.map +1 -0
  85. package/dist/lib/run-test-result.d.ts +3 -6
  86. package/dist/lib/run-test-result.d.ts.map +1 -1
  87. package/dist/lib/run-test-result.js +41 -54
  88. package/dist/lib/run-test-result.js.map +1 -1
  89. package/dist/test/bundle/cli-bundle.test.d.ts +2 -0
  90. package/dist/test/bundle/cli-bundle.test.d.ts.map +1 -0
  91. package/dist/test/bundle/cli-bundle.test.js +190 -0
  92. package/dist/test/bundle/cli-bundle.test.js.map +1 -0
  93. package/dist/test/e2e/cli-bundle.e2e.test.d.ts +2 -0
  94. package/dist/test/e2e/cli-bundle.e2e.test.d.ts.map +1 -0
  95. package/dist/test/e2e/cli-bundle.e2e.test.js +138 -0
  96. package/dist/test/e2e/cli-bundle.e2e.test.js.map +1 -0
  97. package/dist/test/e2e/git-build-release.e2e.test.d.ts +2 -0
  98. package/dist/test/e2e/git-build-release.e2e.test.d.ts.map +1 -0
  99. package/dist/test/e2e/git-build-release.e2e.test.js +87 -0
  100. package/dist/test/e2e/git-build-release.e2e.test.js.map +1 -0
  101. package/dist/test/e2e/helpers.d.ts +37 -0
  102. package/dist/test/e2e/helpers.d.ts.map +1 -0
  103. package/dist/test/e2e/helpers.js +57 -0
  104. package/dist/test/e2e/helpers.js.map +1 -0
  105. package/dist/test/unit/bin/affected-actors-cli.test.d.ts +2 -0
  106. package/dist/test/unit/bin/affected-actors-cli.test.d.ts.map +1 -0
  107. package/dist/test/unit/bin/affected-actors-cli.test.js +167 -0
  108. package/dist/test/unit/bin/affected-actors-cli.test.js.map +1 -0
  109. package/dist/test/unit/bin/affected-actors.test.d.ts +2 -0
  110. package/dist/test/unit/bin/affected-actors.test.d.ts.map +1 -0
  111. package/dist/test/unit/bin/affected-actors.test.js +141 -0
  112. package/dist/test/unit/bin/affected-actors.test.js.map +1 -0
  113. package/dist/test/unit/bin/build-cli.test.d.ts +2 -0
  114. package/dist/test/unit/bin/build-cli.test.d.ts.map +1 -0
  115. package/dist/test/unit/bin/build-cli.test.js +44 -0
  116. package/dist/test/unit/bin/build-cli.test.js.map +1 -0
  117. package/dist/test/unit/bin/build-from-local.test.js +65 -1
  118. package/dist/test/unit/bin/build-from-local.test.js.map +1 -1
  119. package/dist/test/unit/bin/build.test.js +223 -3
  120. package/dist/test/unit/bin/build.test.js.map +1 -1
  121. package/dist/test/unit/bin/compiled-cli.d.ts +6 -0
  122. package/dist/test/unit/bin/compiled-cli.d.ts.map +1 -0
  123. package/dist/test/unit/bin/compiled-cli.js +16 -0
  124. package/dist/test/unit/bin/compiled-cli.js.map +1 -0
  125. package/dist/test/unit/bin/diff-changes.test.js +130 -101
  126. package/dist/test/unit/bin/diff-changes.test.js.map +1 -1
  127. package/dist/test/unit/bin/diff.test.js +25 -28
  128. package/dist/test/unit/bin/diff.test.js.map +1 -1
  129. package/dist/test/unit/bin/git.test.js +69 -152
  130. package/dist/test/unit/bin/git.test.js.map +1 -1
  131. package/dist/test/unit/bin/slack.test.d.ts +2 -0
  132. package/dist/test/unit/bin/slack.test.d.ts.map +1 -0
  133. package/dist/test/unit/bin/slack.test.js +71 -0
  134. package/dist/test/unit/bin/slack.test.js.map +1 -0
  135. package/dist/test/unit/bin/stdin-input.test.d.ts +2 -0
  136. package/dist/test/unit/bin/stdin-input.test.d.ts.map +1 -0
  137. package/dist/test/unit/bin/stdin-input.test.js +50 -0
  138. package/dist/test/unit/bin/stdin-input.test.js.map +1 -0
  139. package/dist/test/unit/github-scripts/mask-config-secrets.test.d.ts +2 -0
  140. package/dist/test/unit/github-scripts/mask-config-secrets.test.d.ts.map +1 -0
  141. package/dist/test/unit/github-scripts/mask-config-secrets.test.js +50 -0
  142. package/dist/test/unit/github-scripts/mask-config-secrets.test.js.map +1 -0
  143. package/dist/test/unit/github-scripts/run-with-apify-tokens.test.d.ts +2 -0
  144. package/dist/test/unit/github-scripts/run-with-apify-tokens.test.d.ts.map +1 -0
  145. package/dist/test/unit/github-scripts/run-with-apify-tokens.test.js +102 -0
  146. package/dist/test/unit/github-scripts/run-with-apify-tokens.test.js.map +1 -0
  147. package/dist/test/unit/run-checks.test.d.ts +2 -0
  148. package/dist/test/unit/run-checks.test.d.ts.map +1 -0
  149. package/dist/test/unit/run-checks.test.js +62 -0
  150. package/dist/test/unit/run-checks.test.js.map +1 -0
  151. package/dist/test/unit/run-test-result.test.d.ts +2 -0
  152. package/dist/test/unit/run-test-result.test.d.ts.map +1 -0
  153. package/dist/test/unit/run-test-result.test.js +34 -0
  154. package/dist/test/unit/run-test-result.test.js.map +1 -0
  155. package/dist/test/unit/should-built-and-test.test.js +41 -38
  156. package/dist/test/unit/should-built-and-test.test.js.map +1 -1
  157. package/dist/tsconfig.tsbuildinfo +1 -1
  158. package/docs/cli.md +373 -0
  159. package/docs/lib.md +271 -0
  160. package/docs/workflows.md +367 -0
  161. package/lib/extend-expect.ts +48 -258
  162. package/lib/lib.ts +24 -28
  163. package/lib/run-checks.ts +101 -0
  164. package/lib/run-test-result.ts +45 -55
  165. package/package.json +13 -3
  166. package/test/bundle/cli-bundle.test.ts +235 -0
  167. package/test/e2e/cli-bundle.e2e.test.ts +207 -0
  168. package/test/e2e/git-build-release.e2e.test.ts +113 -0
  169. package/test/e2e/helpers.ts +82 -0
  170. package/test/unit/bin/affected-actors-cli.test.ts +201 -0
  171. package/test/unit/bin/affected-actors.test.ts +182 -0
  172. package/test/unit/bin/build-cli.test.ts +55 -0
  173. package/test/unit/bin/build-from-local.test.ts +71 -0
  174. package/test/unit/bin/build.test.ts +279 -3
  175. package/test/unit/bin/compiled-cli.ts +17 -0
  176. package/test/unit/bin/diff-changes.test.ts +139 -101
  177. package/test/unit/bin/diff.test.ts +25 -28
  178. package/test/unit/bin/git.test.ts +85 -176
  179. package/test/unit/bin/slack.test.ts +84 -0
  180. package/test/unit/bin/stdin-input.test.ts +61 -0
  181. package/test/unit/github-scripts/mask-config-secrets.test.ts +61 -0
  182. package/test/unit/github-scripts/run-with-apify-tokens.test.ts +115 -0
  183. package/test/unit/run-checks.test.ts +76 -0
  184. package/test/unit/run-test-result.test.ts +44 -0
  185. package/test/unit/should-built-and-test.test.ts +46 -39
  186. package/.github/scripts/check-package-version-bump.mjs +0 -74
  187. package/.github/workflows-package-version +0 -1
@@ -0,0 +1,198 @@
1
+ import { type ChangedActors, getChangedActors } from './diff-changes.js';
2
+ import {
3
+ findCommitSha,
4
+ getChangedFilesBetween,
5
+ getChangelogAdditions,
6
+ getCommitsBetween,
7
+ getMergeBase,
8
+ resolveCommitSha,
9
+ resolveReleaseBaseCommit,
10
+ } from './git.js';
11
+ import { logger } from './logger.js';
12
+ import type { ActorConfig, Commit, GitRange } from './types.js';
13
+ import { readConfigFile } from './utils/config/load-config.js';
14
+
15
+ // The range always ends at the checked-out commit: check out what you want to inspect
16
+ const HEAD = 'HEAD';
17
+
18
+ interface AffectedActorsOptions {
19
+ /**
20
+ * Base usually points to the master branch
21
+ * Only what HEAD changes compared to base counts, like `git diff base...HEAD` or GitHub's "Files changed" tab:
22
+ * the branch's own changes, not what it merged in from base.
23
+ */
24
+ base?: string;
25
+ /**
26
+ * With `base` it's the last validated commit of the branch: Actors that didn't change after it are left out.
27
+ * It's ignored when it is HEAD (a rerun) or isn't in the local history. Without `base` everything after it
28
+ * counts, and it must be an ancestor of HEAD (e.g. the last release).
29
+ */
30
+ since?: string;
31
+ /** Glob relative to the repo root. Matching files count as changes even when .dockerignore lists them. */
32
+ testFilesGlob?: string;
33
+ actors: string[];
34
+ ignore: string[];
35
+ }
36
+
37
+ /** The JSON document `get-affected-actors` prints. Versioned, since other commands and workflows consume it. */
38
+ interface AffectedActors {
39
+ version: 1;
40
+ head: string;
41
+ /** The commits, changed files and changelog describe the whole range, `--since` with `--base` only narrows the Actors. */
42
+ commits: Commit[];
43
+ changedFiles: string[];
44
+ changelog: string | null;
45
+ /**
46
+ * Actors with at least one functional change. They need a new build and tests. Only the names: the config file
47
+ * stays the source of truth for the rest.
48
+ */
49
+ functionallyChangedActors: { actorFullName: string }[];
50
+ /** Actors whose changes are all cosmetic (README, CHANGELOG, cosmetic .actor JSON). They only need a release build. */
51
+ cosmeticallyChangedActors: { actorFullName: string }[];
52
+ }
53
+
54
+ const names = (configs: ActorConfig[]) => configs.map(({ actorFullName }) => ({ actorFullName }));
55
+
56
+ /** The branch's last validated commit, unless it can't narrow anything down. */
57
+ const findValidatedSha = async (since: string, headSha: string): Promise<string | undefined> => {
58
+ const sinceSha = await findCommitSha(since);
59
+ if (!sinceSha) {
60
+ logger.info(
61
+ `Last validated commit ${since} is not in the local history (force-push?), checking the whole branch`,
62
+ );
63
+ return undefined;
64
+ }
65
+ if (sinceSha === headSha) {
66
+ logger.info('Last validated commit is HEAD, so this is a rerun: checking the whole branch from scratch');
67
+ return undefined;
68
+ }
69
+ return sinceSha;
70
+ };
71
+
72
+ /**
73
+ * This is Juan's magical algorithm where we need just 2 git diffs to determine functional changes on Actors.
74
+ *
75
+ * Naively, we could just check branch HEAD changes vs base or parent of first commit and be done but
76
+ * we do this jiu-jitsu code to apply 2 optimizations:
77
+ * 1. On our PR branch, we track last validated commit (since) that was tested and so we only need to test new commits.
78
+ * 2. We also merge changes from the base branch (usually master) and we can consider those already validated as well.
79
+ *
80
+ * But there is annoying problem: even if both our branch and base are validated, they could have novel interactions and
81
+ * we need to retest those.
82
+ *
83
+ * So we look at:
84
+ * 1. The changes introduced by our branch since the last validated commit (`since..HEAD`).
85
+ * 2. The changes introduced by the branch compared to the base branch (`base...HEAD`). The important part is that this
86
+ * diff does check the whole branch, discarding the last validated commit (since) so it checks those novel interactions problem.
87
+ *
88
+ * The magic is that both need to contain changes for an Actor to be retested:
89
+ * 1. If we have changes only in since..HEAD (and not in base...HEAD), it means those changes came from merging the base alone
90
+ * which is already validated and there were actually no real changes introduced by our branch alone.
91
+ * 2. If we have changes only in base...HEAD (and not in since..HEAD), it means there are no new changes on our branch
92
+ * since it was last validated. base...HEAD has changes that we already validated.
93
+ *
94
+ * This enables to simply do those 2 diffs for each Actor's context separately and only Actors that have changes
95
+ * in both diffs need to be retested.
96
+ *
97
+ * THIS ASSUMES BRANCH IS UP TO DATE WITH BASE WHEN IT MERGES, which we currently require. Only then is HEAD what lands on
98
+ * base, and base's commits validated. Otherwise what lands is HEAD combined with base's newer commits, which nobody
99
+ * tested, however the Actors are picked.
100
+ *
101
+ * In each example the branch's functional change to Actor A was validated at `since`, then one more push follows:
102
+ *
103
+ * Example scenarios:
104
+ * - Our own branch only: a commit changes A's README and B's code.
105
+ * `since..HEAD`: A cosmetic, B functional. `base...HEAD`: A and B functional.
106
+ * → A cosmetic, B functional: only B is tested, A's code is still the validated one.
107
+ * - Cosmetic merge: the branch merges base, which only changed READMEs and the CHANGELOG.
108
+ * `since..HEAD`: every Actor cosmetic (CHANGELOG counts for all). `base...HEAD`: A functional, since the merged
109
+ * files now match base and drop out of it.
110
+ * → A cosmetic, the other Actors left out: nothing is tested.
111
+ * - Functional merge: the branch merges base, which changed code of A and of C.
112
+ * `since..HEAD`: A and C functional. `base...HEAD`: A functional.
113
+ * → A functional, C left out: A is tested, since base's change can interact with the branch's, while C only has
114
+ * base's change, which base already validated.
115
+ */
116
+ const intersectChangedInBranchVsSince = (branch: ChangedActors, sinceValidated: ChangedActors): ChangedActors => {
117
+ const narrowed: ChangedActors = { functional: [], cosmetic: [] };
118
+ // All checks are done for each Actor separately, that's cleanest and most accurate
119
+ for (const actor of [...branch.functional, ...branch.cosmetic]) {
120
+ // base...HEAD
121
+ const changedFunctionallyOnBranch = branch.functional.includes(actor);
122
+ // since..HEAD
123
+ const changedFunctionallySince = sinceValidated.functional.includes(actor);
124
+
125
+ // changedSince is used for release. Release cares also about cosmetic.
126
+ const changedSince = changedFunctionallySince || sinceValidated.cosmetic.includes(actor);
127
+
128
+ // Nothing changed since the last validation, so the branch's change to this Actor is already validated
129
+ if (!changedSince) continue;
130
+
131
+ // As we said, only intersection counts as functional, otherwise it's cosmetic
132
+ if (changedFunctionallyOnBranch && changedFunctionallySince) {
133
+ narrowed.functional.push(actor);
134
+ } else {
135
+ narrowed.cosmetic.push(actor);
136
+ }
137
+ }
138
+ return narrowed;
139
+ };
140
+
141
+ /**
142
+ * Finds the Actors whose context changed in the given range. `base` selects the branch's own changes
143
+ * (a PR), `since` the changes after a known commit (a release, or the last validated commit of a PR).
144
+ */
145
+ export const getAffectedActors = async ({
146
+ base,
147
+ since: sinceOption,
148
+ testFilesGlob,
149
+ actors,
150
+ ignore,
151
+ }: AffectedActorsOptions): Promise<AffectedActors> => {
152
+ // An empty --since (e.g. an empty cache file) means the same as not passing it
153
+ const since = sinceOption || undefined;
154
+ if (!base && !since) {
155
+ throw new Error('Pass --base, --since, or both to select the commits to inspect.');
156
+ }
157
+
158
+ const actorConfigs = await readConfigFile({ actors, ignore });
159
+ const getChangedInRange = async (range: GitRange) => {
160
+ const changedFiles = await getChangedFilesBetween(range.fromRef, range.toRef);
161
+ return {
162
+ changedFiles,
163
+ changedActors: await getChangedActors({
164
+ filepathsChanged: changedFiles,
165
+ actorConfigs,
166
+ range,
167
+ testFilesGlob,
168
+ }),
169
+ };
170
+ };
171
+
172
+ const headSha = await resolveCommitSha(HEAD);
173
+ const rangeVsBase: GitRange = {
174
+ // merge base = PR, release base = push to master (release)
175
+ fromRef: base ? await getMergeBase(base, HEAD) : await resolveReleaseBaseCommit(since!),
176
+ toRef: headSha,
177
+ };
178
+
179
+ // Falls back to rangeVsBase.fromRef when there's no (usable) last validated commit — same as
180
+ // intersecting the branch's changes with themselves, which is a no-op (see this function's docs).
181
+ const validatedSha = base && since ? await findValidatedSha(since, headSha) : undefined;
182
+ const sinceFromRef = validatedSha ?? rangeVsBase.fromRef;
183
+
184
+ const { changedFiles, changedActors: changedVsBase } = await getChangedInRange(rangeVsBase);
185
+ const { changedActors: changedSinceActors } = await getChangedInRange({ fromRef: sinceFromRef, toRef: headSha });
186
+ // See this function's docs to understand the algorithm: only intersection counts as functional, otherwise it's cosmetic.
187
+ const changedActors = intersectChangedInBranchVsSince(changedVsBase, changedSinceActors);
188
+
189
+ return {
190
+ version: 1,
191
+ head: headSha,
192
+ commits: await getCommitsBetween(rangeVsBase.fromRef, rangeVsBase.toRef),
193
+ changedFiles,
194
+ changelog: await getChangelogAdditions(rangeVsBase.fromRef, rangeVsBase.toRef, changedFiles),
195
+ functionallyChangedActors: names(changedActors.functional),
196
+ cosmeticallyChangedActors: names(changedActors.cosmetic),
197
+ };
198
+ };
@@ -6,11 +6,17 @@ import AdmZip from 'adm-zip';
6
6
  import type * as ApifyClientTypes from 'apify-client';
7
7
  import { ActorSourceType } from 'apify-client';
8
8
 
9
- import { dryRunBuildData, LOCAL_SOURCE_VERSION_NUMBER, runAndSummarizeBuilds } from './build.js';
9
+ import {
10
+ ApifyBuilder,
11
+ assertEnvVarsMatchDefaultVersion,
12
+ LOCAL_SOURCE_VERSION_NUMBER,
13
+ resolveDefaultVersion,
14
+ runAndSummarizeBuilds,
15
+ } from './build.js';
10
16
  import { loadDockerIgnore } from './dockerignore.js';
11
17
  import { logger } from './logger.js';
12
18
  import { isPathWithinScope } from './path-utils.js';
13
- import type { ActorConfig, BuildData } from './types.js';
19
+ import type { ActorConfig, BuildData, DryRunPlan } from './types.js';
14
20
  import type { SourceFile } from './utils.js';
15
21
  import { getGitignoredPaths, isOutsideDir, listRepoFilePaths, readSourceFile } from './utils.js';
16
22
  import type { RelativeDir } from './utils/path/repo-relative.js';
@@ -179,32 +185,78 @@ export const rewriteActorJsonPaths = async (
179
185
  await fs.writeFile(path.join(newActorDir, 'actor.json'), JSON.stringify(rewritten, null, 4));
180
186
  };
181
187
 
188
+ type LocalBuildPlan = {
189
+ actorFullName: string;
190
+ versionNumber: string;
191
+ folder: string;
192
+ fileCount: number;
193
+ zipSizeBytes: number;
194
+ };
195
+
182
196
  export const runBuildsFromLocal = async ({
183
197
  actorConfigs,
184
198
  dryRun,
199
+ requireEnvVarsInSync = false,
185
200
  }: {
186
201
  actorConfigs: ActorConfig[];
187
202
  dryRun: boolean;
188
- }): Promise<BuildData[]> => {
203
+ // See assertEnvVarsMatchDefaultVersion
204
+ requireEnvVarsInSync?: boolean;
205
+ }): Promise<BuildData[] | DryRunPlan<LocalBuildPlan>> => {
206
+ // Read-only preparation shared with the real build, so a dry run fails on a missing token, an unknown
207
+ // Actor or unreadable sources just like the build would
208
+ const preparedBuilds = await Promise.all(
209
+ actorConfigs.map(async (actorConfig) => {
210
+ const actorInfo = await ApifyBuilder.fromActorConfig(actorConfig).getActorInfo();
211
+ if (requireEnvVarsInSync) {
212
+ const { defaultVersion } = resolveDefaultVersion(actorConfig.actorFullName, actorInfo);
213
+ assertEnvVarsMatchDefaultVersion(
214
+ actorConfig.actorFullName,
215
+ actorInfo,
216
+ defaultVersion,
217
+ LOCAL_SOURCE_VERSION_NUMBER,
218
+ );
219
+ }
220
+ // Zipped and uploaded like `apify push` does, since inline SOURCE_FILES are capped at ~9 MB.
221
+ const zip = new AdmZip();
222
+ const sourceFiles = await collectSourceFiles(actorConfig);
223
+ for (const { name, content } of sourceFiles) {
224
+ zip.addFile(name, content);
225
+ }
226
+ return { actorConfig, actorInfo, fileCount: sourceFiles.length, zipBuffer: zip.toBuffer() };
227
+ }),
228
+ );
229
+
189
230
  if (dryRun) {
190
231
  logger.info('[DRY RUN] Would build from local source:');
191
- for (const { actorFullName, folder } of actorConfigs) {
192
- logger.info(` ${actorFullName} (${folder})`);
232
+ for (const { actorConfig, fileCount, zipBuffer } of preparedBuilds) {
233
+ logger.info(
234
+ ` ${actorConfig.actorFullName} (${actorConfig.folder}): ${fileCount} files, ${zipBuffer.length} bytes zipped`,
235
+ );
193
236
  }
194
- return actorConfigs.map(({ actorFullName }) => dryRunBuildData(actorFullName, LOCAL_SOURCE_VERSION_NUMBER));
237
+ return {
238
+ dryRun: true,
239
+ builds: preparedBuilds.map(({ actorConfig, fileCount, zipBuffer }) => ({
240
+ actorFullName: actorConfig.actorFullName,
241
+ versionNumber: LOCAL_SOURCE_VERSION_NUMBER,
242
+ folder: actorConfig.folder.path,
243
+ fileCount,
244
+ zipSizeBytes: zipBuffer.length,
245
+ })),
246
+ };
195
247
  }
196
248
 
249
+ const preparedByActorFullName = new Map(
250
+ preparedBuilds.map((prepared) => [prepared.actorConfig.actorFullName, prepared]),
251
+ );
252
+
197
253
  return runAndSummarizeBuilds(actorConfigs, 'LOCAL BUILDS', async (actorConfig, builder) => {
198
- // Zipped and uploaded like `apify push` does, since inline SOURCE_FILES are capped at ~9 MB.
199
- const zip = new AdmZip();
200
- for (const { name, content } of await collectSourceFiles(actorConfig)) {
201
- zip.addFile(name, content);
202
- }
254
+ const { actorInfo, zipBuffer } = preparedByActorFullName.get(actorConfig.actorFullName)!;
203
255
  const actorVersion: ApifyClientTypes.ActorVersion = {
204
256
  versionNumber: LOCAL_SOURCE_VERSION_NUMBER,
205
- tarballUrl: await builder.uploadSourceZip(LOCAL_SOURCE_VERSION_NUMBER, zip.toBuffer()),
257
+ tarballUrl: await builder.uploadSourceZip(LOCAL_SOURCE_VERSION_NUMBER, zipBuffer),
206
258
  sourceType: ActorSourceType.Tarball,
207
259
  };
208
- return builder.createVersionAndBuild(LOCAL_SOURCE_VERSION_NUMBER, actorVersion, false);
260
+ return builder.createVersionAndBuild(LOCAL_SOURCE_VERSION_NUMBER, actorVersion, false, actorInfo);
209
261
  });
210
262
  };
package/bin/build.ts CHANGED
@@ -1,22 +1,21 @@
1
1
  import type * as ApifyClientTypes from 'apify-client';
2
2
  import { ActorSourceType, ApifyClient } from 'apify-client';
3
3
 
4
- import { normalizeRepoUrl } from './git.js';
4
+ import { getCurrentBranch, normalizeRepoUrl, resolveRepoUrl } from './git.js';
5
5
  import { logger } from './logger.js';
6
- import type { ActorConfig, BuildData } from './types.js';
6
+ import type { ActorConfig, BuildData, DryRunPlan } from './types.js';
7
7
 
8
8
  type BuildPrActorOptions = {
9
- buildTag?: string;
10
9
  versionNumber: string;
11
10
  gitRepoUrl: string;
12
11
  actorConfig: ActorConfig;
13
12
  actorInfo: ApifyClientTypes.Actor;
14
- useDockerCache: boolean;
15
13
  };
16
14
 
17
15
  // Fixed version number used to build from local source files, since there is no real version to track.
18
16
  export const LOCAL_SOURCE_VERSION_NUMBER = '0.98';
19
- const DEFAULT_TEST_VERSION_NUMBER = '0.99';
17
+ // Stands for each Actor's default version: the one its default build tag points to
18
+ export const DEFAULT_VERSION_KEYWORD = 'default';
20
19
 
21
20
  // Shown whenever an Actor isn't set up the way CI builds need it
22
21
  const ACTOR_SETUP_REQUIREMENT =
@@ -83,6 +82,44 @@ export const assertRepoUrlMatchesDefaultVersion = (
83
82
  }
84
83
  };
85
84
 
85
+ const getEnvVarNames = (version: ApifyClientTypes.ActorVersion | undefined): string[] =>
86
+ (version?.envVars ?? []).map(({ name }) => name).filter((name): name is string => !!name);
87
+
88
+ /**
89
+ * Guards against testing an Actor with different env vars than users run it with, e.g. a variable added to
90
+ * the default version but not to the test version, which would make the tests pass or fail for the wrong
91
+ * reason. Only names are compared because the API doesn't return secret values.
92
+ */
93
+ export const assertEnvVarsMatchDefaultVersion = (
94
+ actorFullName: string,
95
+ actorInfo: ApifyClientTypes.Actor,
96
+ defaultVersion: ApifyClientTypes.ActorVersion | undefined,
97
+ versionNumber: string,
98
+ ) => {
99
+ if (!defaultVersion) {
100
+ throw new Error(
101
+ `[${actorFullName}] The version of the default build no longer exists. ${ACTOR_SETUP_REQUIREMENT}`,
102
+ );
103
+ }
104
+ const version = actorInfo.versions.find((v) => v.versionNumber === versionNumber);
105
+ const defaultNames = getEnvVarNames(defaultVersion);
106
+ const names = getEnvVarNames(version);
107
+ const missing = defaultNames.filter((name) => !names.includes(name));
108
+ const extra = names.filter((name) => !defaultNames.includes(name));
109
+ if (missing.length === 0 && extra.length === 0) return;
110
+
111
+ const differences = [
112
+ missing.length > 0 ? `missing ${missing.join(', ')}` : undefined,
113
+ extra.length > 0 ? `extra ${extra.join(', ')}` : undefined,
114
+ ].filter(Boolean);
115
+ throw new Error(
116
+ `[${actorFullName}] Env vars of version ${versionNumber} are out of sync with default version ` +
117
+ `${defaultVersion.versionNumber} (${differences.join('; ')}). ` +
118
+ `${version ? '' : `Version ${versionNumber} doesn't exist yet, so create it on the platform first. `}` +
119
+ `Add or remove the env vars in version ${versionNumber} so their names match the default version's.`,
120
+ );
121
+ };
122
+
86
123
  export class ApifyBuilder {
87
124
  private constructor(
88
125
  private readonly apifyClient: ApifyClient,
@@ -179,7 +216,7 @@ export class ApifyBuilder {
179
216
  * Deletes build of all versions. Apify API doesn't allow to delete default build and we explicitly skip it
180
217
  * We delete devel builds faster because we used the for every PR until recently so just to get rid of them faster
181
218
  */
182
- async deleteOldBuilds(): Promise<void> {
219
+ async deleteOldBuilds({ dryRun }: { dryRun: boolean }): Promise<void> {
183
220
  // Even though we don't version our current Actors, if we ever such Actors to GitHub CI, we would accidentally delete old supported versions
184
221
  // This hardcoded solution is not ideal, but it should prevent most imaginable cases
185
222
  // All currently popular versioned Actors use `version-${number}` format
@@ -258,9 +295,15 @@ export class ApifyBuilder {
258
295
  });
259
296
 
260
297
  logger.info(
261
- `[DELETE OLD BUILDS][${this.actorFullName}]: Deleting ${buildsToDelete.length} old builds that are non-default and ` +
262
- `older than 30 days from total ${items.length}`,
298
+ `[DELETE OLD BUILDS][${this.actorFullName}]: ${dryRun ? '[DRY RUN] Would delete' : 'Deleting'} ${buildsToDelete.length} ` +
299
+ `old builds that are non-default and older than 30 days from total ${items.length}`,
263
300
  );
301
+ if (dryRun) {
302
+ for (const build of buildsToDelete) {
303
+ logger.info(` ${build.buildNumber} (${build.id}, started ${build.startedAt.toISOString()})`);
304
+ }
305
+ return;
306
+ }
264
307
  for (const build of buildsToDelete) {
265
308
  await this.apifyClient.build(build.id).delete();
266
309
  }
@@ -310,23 +353,23 @@ export const runAndSummarizeBuilds = async (
310
353
  return waitAndSummarizeBuilds(startedBuilds, buildersByActorFullName, label);
311
354
  };
312
355
 
313
- // Placeholder BuildData for dry runs, since no real build was triggered.
314
- export const dryRunBuildData = (actorFullName: string, versionNumber: string): BuildData => ({
315
- buildId: 'dry-run',
316
- actorRawId: 'dry-run',
317
- buildNumber: versionNumber,
318
- actorFullName,
319
- });
356
+ type GitBuildPlan = {
357
+ actorFullName: string;
358
+ versionNumber: string;
359
+ gitRepoUrl: string;
360
+ };
320
361
 
321
362
  type RunBuildsOptions = {
322
363
  actorConfigs: ActorConfig[];
323
- isLatest?: boolean;
364
+ // A version number like "0.99", or DEFAULT_VERSION_KEYWORD for each Actor's default version
365
+ versionNumber: string;
324
366
  repoUrl: string;
325
367
  // False when the repo URL was passed explicitly, see assertRepoUrlMatchesDefaultVersion
326
368
  shouldVerifyRepoUrl: boolean;
327
369
  branch: string;
328
370
  dryRun: boolean;
329
- useDockerCache: boolean;
371
+ // See assertEnvVarsMatchDefaultVersion
372
+ requireEnvVarsInSync?: boolean;
330
373
  };
331
374
 
332
375
  export const runBuilds = async ({
@@ -334,10 +377,10 @@ export const runBuilds = async ({
334
377
  shouldVerifyRepoUrl,
335
378
  branch,
336
379
  actorConfigs,
337
- isLatest = false,
380
+ versionNumber,
338
381
  dryRun,
339
- useDockerCache,
340
- }: RunBuildsOptions): Promise<BuildData[]> => {
382
+ requireEnvVarsInSync = false,
383
+ }: RunBuildsOptions): Promise<BuildData[] | DryRunPlan<GitBuildPlan>> => {
341
384
  // Fetched once per Actor and reused for the checks, the version numbers and the build itself
342
385
  const buildConfigs: BuildPrActorOptions[] = await Promise.all(
343
386
  actorConfigs.map(async (actorConfig) => {
@@ -346,9 +389,28 @@ export const runBuilds = async ({
346
389
  actorConfig.actorFullName,
347
390
  actorInfo,
348
391
  );
392
+ // Builds keep the tags their version gives them, so a default version without the default build tag
393
+ // would build fine but leave users on the previous build
394
+ if (versionNumber === DEFAULT_VERSION_KEYWORD && defaultVersion?.buildTag !== defaultBuildTag) {
395
+ logger.warn(
396
+ `[${actorConfig.actorFullName}] Default version ${defaultVersionNumber} has build tag ` +
397
+ `"${defaultVersion?.buildTag ?? 'none'}", not "${defaultBuildTag}", so users keep running the previous build. ` +
398
+ 'Set the build tag of the version on the platform to change that.',
399
+ );
400
+ }
349
401
  if (shouldVerifyRepoUrl) {
350
402
  assertRepoUrlMatchesDefaultVersion(actorConfig.actorFullName, defaultVersion, repoUrl);
351
403
  }
404
+ const targetVersionNumber =
405
+ versionNumber === DEFAULT_VERSION_KEYWORD ? defaultVersionNumber : versionNumber;
406
+ if (requireEnvVarsInSync) {
407
+ assertEnvVarsMatchDefaultVersion(
408
+ actorConfig.actorFullName,
409
+ actorInfo,
410
+ defaultVersion,
411
+ targetVersionNumber,
412
+ );
413
+ }
352
414
 
353
415
  // Depending on if these are miniactors or standaloneActors
354
416
  let gitRepoUrl = `${repoUrl}#${branch}`;
@@ -359,21 +421,23 @@ export const runBuilds = async ({
359
421
  actorConfig,
360
422
  actorInfo,
361
423
  gitRepoUrl,
362
- versionNumber: isLatest ? defaultVersionNumber : DEFAULT_TEST_VERSION_NUMBER,
363
- buildTag: isLatest ? defaultBuildTag : undefined,
364
- useDockerCache,
424
+ versionNumber: targetVersionNumber,
365
425
  };
366
426
  }),
367
427
  );
368
428
 
429
+ // Everything above only reads, so a dry run has checked what the real run would, short of writing
369
430
  if (dryRun) {
370
431
  logger.info('[DRY RUN] Would build:');
371
- for (const { actorConfig, versionNumber } of buildConfigs) {
372
- logger.info(` ${actorConfig.actorFullName} (${versionNumber})`);
432
+ const builds = buildConfigs.map((buildConfig) => ({
433
+ actorFullName: buildConfig.actorConfig.actorFullName,
434
+ versionNumber: buildConfig.versionNumber,
435
+ gitRepoUrl: buildConfig.gitRepoUrl,
436
+ }));
437
+ for (const build of builds) {
438
+ logger.info(` ${build.actorFullName} (${build.versionNumber}) from ${build.gitRepoUrl}`);
373
439
  }
374
- return buildConfigs.map(({ actorConfig, versionNumber }) =>
375
- dryRunBuildData(actorConfig.actorFullName, versionNumber),
376
- );
440
+ return { dryRun: true, builds };
377
441
  }
378
442
 
379
443
  const buildConfigsByActorFullName = new Map(
@@ -381,25 +445,43 @@ export const runBuilds = async ({
381
445
  );
382
446
 
383
447
  return runAndSummarizeBuilds(actorConfigs, 'BUILDS', async (actorConfig, builder) => {
384
- const {
385
- actorInfo,
386
- buildTag,
387
- versionNumber,
388
- gitRepoUrl,
389
- useDockerCache: useCache,
390
- } = buildConfigsByActorFullName.get(actorConfig.actorFullName)!;
448
+ const buildConfig = buildConfigsByActorFullName.get(actorConfig.actorFullName)!;
449
+ // No buildTag: the version keeps the one it has, so its build gets the tag the version already gives it
391
450
  const actorVersion: ApifyClientTypes.ActorVersion = {
392
- buildTag,
393
- versionNumber,
394
- gitRepoUrl,
451
+ versionNumber: buildConfig.versionNumber,
452
+ gitRepoUrl: buildConfig.gitRepoUrl,
395
453
  sourceType: ActorSourceType.GitRepo,
396
454
  };
397
- return builder.createVersionAndBuild(versionNumber, actorVersion, useCache, actorInfo);
455
+ // Always with the Docker cache: CI always asked for it, and a clean build can still be started on the platform
456
+ return builder.createVersionAndBuild(buildConfig.versionNumber, actorVersion, true, buildConfig.actorInfo);
457
+ });
458
+ };
459
+
460
+ type BuildActorsOptions = Omit<RunBuildsOptions, 'repoUrl' | 'shouldVerifyRepoUrl' | 'branch'> & {
461
+ // Defaults to the `origin` remote, see resolveRepoUrl
462
+ repoUrl?: string;
463
+ };
464
+
465
+ /**
466
+ * Builds exactly the given Actors from the checked-out Git branch, as it is on the remote. Fails on a detached HEAD,
467
+ * where there is no branch to build. Used by `build` and the commands built on top of it.
468
+ */
469
+ export const buildActors = async ({ actorConfigs, repoUrl, ...options }: BuildActorsOptions) => {
470
+ if (actorConfigs.length === 0) {
471
+ logger.info('No Actors to build');
472
+ // Still the dry-run shape in a dry run, so it can't pass for a real (empty) result
473
+ return options.dryRun ? { dryRun: true as const, builds: [] } : [];
474
+ }
475
+ return runBuilds({
476
+ ...options,
477
+ ...(await resolveRepoUrl(repoUrl)),
478
+ actorConfigs,
479
+ branch: await getCurrentBranch(),
398
480
  });
399
481
  };
400
482
 
401
- export const deleteOldBuilds = async (actorConfigs: ActorConfig[]) => {
483
+ export const deleteOldBuilds = async (actorConfigs: ActorConfig[], { dryRun }: { dryRun: boolean }) => {
402
484
  for (const actorConfig of actorConfigs) {
403
- await ApifyBuilder.fromActorConfig(actorConfig).deleteOldBuilds();
485
+ await ApifyBuilder.fromActorConfig(actorConfig).deleteOldBuilds({ dryRun });
404
486
  }
405
487
  };