apify-test-tools 0.9.1-beta.9 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/.github/actions/checkout-restore-dependencies/action.yaml +2 -28
  2. package/.github/workflows/public_platform-tests.yaml +0 -12
  3. package/.github/workflows/public_pr-build-test.yaml +2 -24
  4. package/.github/workflows/public_push-build-latest.yaml +3 -14
  5. package/.github/workflows/public_review.yaml +0 -7
  6. package/.github/workflows-package-version +1 -1
  7. package/CHANGELOG.md +4 -3
  8. package/CONTRIBUTING.md +2 -4
  9. package/README.md +147 -36
  10. package/bin/build-from-local.ts +15 -11
  11. package/bin/build.ts +127 -67
  12. package/bin/git.ts +140 -0
  13. package/bin/main.ts +128 -91
  14. package/bin/slack.ts +5 -2
  15. package/bin/types.ts +0 -73
  16. package/bin/utils/config/parser.ts +6 -5
  17. package/bin/utils/config/structures/base.ts +5 -1
  18. package/bin/utils/config/structures/grouped.ts +40 -0
  19. package/bin/utils/config/structures/legacy.ts +3 -1
  20. package/bin/utils.ts +7 -15
  21. package/dist/bin/build-from-local.d.ts +2 -2
  22. package/dist/bin/build-from-local.d.ts.map +1 -1
  23. package/dist/bin/build-from-local.js +11 -6
  24. package/dist/bin/build-from-local.js.map +1 -1
  25. package/dist/bin/build.d.ts +26 -4
  26. package/dist/bin/build.d.ts.map +1 -1
  27. package/dist/bin/build.js +90 -47
  28. package/dist/bin/build.js.map +1 -1
  29. package/dist/bin/git.d.ts +40 -0
  30. package/dist/bin/git.d.ts.map +1 -1
  31. package/dist/bin/git.js +118 -0
  32. package/dist/bin/git.js.map +1 -1
  33. package/dist/bin/main.d.ts +1 -20
  34. package/dist/bin/main.d.ts.map +1 -1
  35. package/dist/bin/main.js +65 -62
  36. package/dist/bin/main.js.map +1 -1
  37. package/dist/bin/slack.d.ts.map +1 -1
  38. package/dist/bin/slack.js +4 -2
  39. package/dist/bin/slack.js.map +1 -1
  40. package/dist/bin/types.d.ts +0 -64
  41. package/dist/bin/types.d.ts.map +1 -1
  42. package/dist/bin/utils/config/parser.d.ts.map +1 -1
  43. package/dist/bin/utils/config/parser.js +6 -5
  44. package/dist/bin/utils/config/parser.js.map +1 -1
  45. package/dist/bin/utils/config/structures/base.d.ts +3 -2
  46. package/dist/bin/utils/config/structures/base.d.ts.map +1 -1
  47. package/dist/bin/utils/config/structures/base.js +1 -0
  48. package/dist/bin/utils/config/structures/base.js.map +1 -1
  49. package/dist/bin/utils/config/structures/grouped.d.ts +6 -0
  50. package/dist/bin/utils/config/structures/grouped.d.ts.map +1 -0
  51. package/dist/bin/utils/config/structures/grouped.js +29 -0
  52. package/dist/bin/utils/config/structures/grouped.js.map +1 -0
  53. package/dist/bin/utils/config/structures/legacy.d.ts +1 -1
  54. package/dist/bin/utils/config/structures/legacy.d.ts.map +1 -1
  55. package/dist/bin/utils/config/structures/legacy.js +1 -1
  56. package/dist/bin/utils/config/structures/legacy.js.map +1 -1
  57. package/dist/bin/utils.d.ts +5 -2
  58. package/dist/bin/utils.d.ts.map +1 -1
  59. package/dist/bin/utils.js +6 -12
  60. package/dist/bin/utils.js.map +1 -1
  61. package/dist/test/unit/bin/build.test.d.ts +2 -0
  62. package/dist/test/unit/bin/build.test.d.ts.map +1 -0
  63. package/dist/test/unit/bin/build.test.js +46 -0
  64. package/dist/test/unit/bin/build.test.js.map +1 -0
  65. package/dist/test/unit/bin/git.test.js +104 -1
  66. package/dist/test/unit/bin/git.test.js.map +1 -1
  67. package/dist/test/unit/bin/utils/config/load-config.test.js +1 -0
  68. package/dist/test/unit/bin/utils/config/load-config.test.js.map +1 -1
  69. package/dist/test/unit/bin/utils/config/parser.test.js +23 -5
  70. package/dist/test/unit/bin/utils/config/parser.test.js.map +1 -1
  71. package/dist/test/unit/bin/utils/config/structures/grouped.test.d.ts +2 -0
  72. package/dist/test/unit/bin/utils/config/structures/grouped.test.d.ts.map +1 -0
  73. package/dist/test/unit/bin/utils/config/structures/grouped.test.js +75 -0
  74. package/dist/test/unit/bin/utils/config/structures/grouped.test.js.map +1 -0
  75. package/dist/tsconfig.tsbuildinfo +1 -1
  76. package/package.json +65 -63
  77. package/test/unit/bin/build.test.ts +68 -0
  78. package/test/unit/bin/git.test.ts +122 -0
  79. package/test/unit/bin/utils/config/load-config.test.ts +1 -0
  80. package/test/unit/bin/utils/config/parser.test.ts +26 -5
  81. package/test/unit/bin/utils/config/structures/grouped.test.ts +90 -0
  82. package/bin/github.ts +0 -69
  83. package/dist/bin/github.d.ts +0 -11
  84. package/dist/bin/github.d.ts.map +0 -1
  85. package/dist/bin/github.js +0 -59
  86. package/dist/bin/github.js.map +0 -1
package/bin/main.ts CHANGED
@@ -2,19 +2,28 @@
2
2
 
3
3
  import process from 'node:process';
4
4
 
5
- import yargs, { type Argv } from 'yargs';
5
+ import yargs, { type Options } from 'yargs';
6
6
  // eslint-disable-next-line import/extensions --- With .js, it cannot find types
7
7
  import { hideBin } from 'yargs/helpers';
8
8
 
9
9
  import { deleteOldBuilds, runBuilds } from './build.js';
10
10
  import { runBuildsFromLocal } from './build-from-local.js';
11
11
  import { getChangedActors } from './diff-changes.js';
12
- import { getBranchOnlyChangedFiles, getChangedFiles, getCommits, hasMergeFromTarget } from './git.js';
13
- import { getPushData } from './github.js';
12
+ import {
13
+ getBranchOnlyChangedFiles,
14
+ getChangedFiles,
15
+ getCommits,
16
+ getCurrentBranch,
17
+ getReleaseChanges,
18
+ getRepoName,
19
+ hasMergeFromTarget,
20
+ resolveReleaseBaseCommit,
21
+ resolveRepoUrl,
22
+ } from './git.js';
14
23
  import { notifyToSlack } from './slack.js';
15
24
  import { reportTestResults } from './test-report.js';
16
25
  import type { Config } from './types.js';
17
- import { setCwd, spawnCommandInGhWorkspace } from './utils.js';
26
+ import { setCwd } from './utils.js';
18
27
  import { readConfigFile } from './utils/config/load-config.js';
19
28
 
20
29
  /**
@@ -22,44 +31,48 @@ import { readConfigFile } from './utils/config/load-config.js';
22
31
  */
23
32
  const middlewares = [setCwd];
24
33
 
25
- export const buildOptions = <T>(y: Argv<T>) => {
26
- return y
27
- .option('target-branch', {
28
- type: 'string',
29
- demandOption: true,
30
- })
31
- .option('source-branch', {
32
- type: 'string',
33
- demandOption: true,
34
- })
35
- .option('use-docker-cache', {
36
- type: 'boolean',
37
- default: false,
38
- })
39
- .option('base-commit', {
40
- type: 'string',
41
- demandOption: false,
42
- });
43
- };
34
+ /*
35
+ * Option groups shared across commands. Each is a plain object passed to `.options()`, so a command's
36
+ * builder composes them by chaining (`y.options(gitRangeOptions).options(actorSelectionOptions)`)
37
+ * instead of nesting wrapper functions. `as const` keeps literals like `demandOption: true` so yargs
38
+ * can infer non-optional types; `satisfies` validates the shape without widening it.
39
+ */
40
+
41
+ /** Commit range to inspect: everything on `source-branch` since it diverged from `target-branch`. */
42
+ const gitRangeOptions = {
43
+ 'target-branch': { type: 'string', demandOption: true },
44
+ 'source-branch': { type: 'string', demandOption: true },
45
+ 'base-commit': { type: 'string', demandOption: false },
46
+ } as const satisfies Record<string, Options>;
47
+
48
+ /**
49
+ * Where to build the Actors from. Defaults to the `origin` remote, which is then checked against each
50
+ * Actor's default version. Passing it explicitly skips that check (see assertRepoUrlMatchesDefaultVersion).
51
+ */
52
+ const repoUrlOptions = {
53
+ 'repo-url': { type: 'string' },
54
+ } as const satisfies Record<string, Options>;
44
55
 
45
56
  /**
46
57
  * Actor-selection flags, applied to every command that reads the actor config so a caller can
47
58
  * narrow the set it operates on (e.g. two-stage releases: `--ignore X`, then `--actors X`).
48
- * Kept separate from `buildOptions` so the read-only git commands don't advertise flags they ignore.
59
+ * Kept separate from `gitRangeOptions` so the read-only git commands don't advertise flags they ignore.
49
60
  */
50
- export const actorSelectionOptions = <T>(y: Argv<T>) => {
51
- return y
52
- .option('actors', {
53
- type: 'string',
54
- array: true,
55
- default: [] as string[],
56
- })
57
- .option('ignore', {
58
- type: 'string',
59
- array: true,
60
- default: [] as string[],
61
- });
62
- };
61
+ const actorSelectionOptions = {
62
+ actors: { type: 'string', array: true, default: [] as string[] },
63
+ ignore: { type: 'string', array: true, default: [] as string[] },
64
+ } as const satisfies Record<string, Options>;
65
+
66
+ /** Flags for commands that trigger Apify builds. */
67
+ const buildOptions = {
68
+ 'use-docker-cache': { type: 'boolean', default: false },
69
+ } as const satisfies Record<string, Options>;
70
+
71
+ /** Global flags, available to every command. */
72
+ const globalOptions = {
73
+ 'dry-run': { type: 'boolean', default: false },
74
+ workspace: { type: 'string' },
75
+ } as const satisfies Record<string, Options>;
63
76
 
64
77
  const resolveChangedActors = async (config: Config, { isLatest }: { isLatest: boolean }) => {
65
78
  const actorConfigs = await readConfigFile(config);
@@ -98,37 +111,51 @@ const resolveChangedActors = async (config: Config, { isLatest }: { isLatest: bo
98
111
 
99
112
  await yargs()
100
113
  .scriptName('public-actors-utils')
101
- .option('dry-run', {
102
- type: 'boolean',
103
- default: false,
104
- })
105
- .option('workspace', {
106
- type: 'string',
107
- })
114
+ .options(globalOptions)
108
115
  .middleware(middlewares)
109
- .command('get-commits', '', buildOptions, (args) => {
110
- const commits = getCommits(args);
111
- console.log(JSON.stringify(commits));
112
- })
113
- .command('get-latest-commit', '', buildOptions, (args) => {
114
- const commits = getCommits(args);
115
- if (commits.length > 0) {
116
- console.log(JSON.stringify(commits[commits.length - 1]));
117
- }
118
- })
119
- .command('get-changed-files', '', buildOptions, (args) => {
120
- const commits = getCommits(args);
121
- const changedFiles = getChangedFiles(commits);
122
- console.log(JSON.stringify(changedFiles));
123
- })
124
- .command('get-actor-configs', '', actorSelectionOptions, async ({ actors, ignore }) => {
125
- const actorConfigs = await readConfigFile({ actors, ignore });
126
- console.log(JSON.stringify(actorConfigs));
127
- })
116
+ .command(
117
+ 'get-commits',
118
+ '',
119
+ (y) => y.options(gitRangeOptions),
120
+ (args) => {
121
+ const commits = getCommits(args);
122
+ console.log(JSON.stringify(commits));
123
+ },
124
+ )
125
+ .command(
126
+ 'get-latest-commit',
127
+ '',
128
+ (y) => y.options(gitRangeOptions),
129
+ (args) => {
130
+ const commits = getCommits(args);
131
+ if (commits.length > 0) {
132
+ console.log(JSON.stringify(commits[commits.length - 1]));
133
+ }
134
+ },
135
+ )
136
+ .command(
137
+ 'get-changed-files',
138
+ '',
139
+ (y) => y.options(gitRangeOptions),
140
+ (args) => {
141
+ const commits = getCommits(args);
142
+ const changedFiles = getChangedFiles(commits);
143
+ console.log(JSON.stringify(changedFiles));
144
+ },
145
+ )
146
+ .command(
147
+ 'get-actor-configs',
148
+ '',
149
+ (y) => y.options(actorSelectionOptions),
150
+ async ({ actors, ignore }) => {
151
+ const actorConfigs = await readConfigFile({ actors, ignore });
152
+ console.log(JSON.stringify(actorConfigs));
153
+ },
154
+ )
128
155
  .command(
129
156
  'get-affected-actors',
130
157
  '',
131
- (args) => actorSelectionOptions(buildOptions(args)),
158
+ (y) => y.options(gitRangeOptions).options(actorSelectionOptions),
132
159
  async (config) => {
133
160
  const actorsChanged = await resolveChangedActors(config, { isLatest: false });
134
161
  console.log(JSON.stringify(actorsChanged));
@@ -137,8 +164,8 @@ await yargs()
137
164
  .command(
138
165
  'report-tests',
139
166
  '',
140
- (args) =>
141
- args
167
+ (y) =>
168
+ y
142
169
  .option('report-file', { type: 'string', demandOption: true })
143
170
  .option('report-slack-channel', { type: 'string' })
144
171
  .option('job-url', { type: 'string' })
@@ -150,18 +177,11 @@ await yargs()
150
177
  .command(
151
178
  'build',
152
179
  '',
153
- (args) => actorSelectionOptions(buildOptions(args)).option('dry-run', { type: 'boolean', default: false }),
180
+ (y) => y.options(gitRangeOptions).options(actorSelectionOptions).options(buildOptions).options(repoUrlOptions),
154
181
  async (config) => {
155
182
  const actorsChanged = await resolveChangedActors(config, { isLatest: false });
156
- // https://github.com/apify-store/google-maps#:actors/lukaskrivka_google-maps-with-contact-details
157
- // git@github.com:apify-store/google-maps#:actors/lukaskrivka_google-maps-with-contact-details
158
- const repoUrl = spawnCommandInGhWorkspace(`git remote get-url origin`).replace(
159
- /^https:\/\/github\.com\//,
160
- 'git@github.com:',
161
- );
162
-
163
183
  const builds = await runBuilds({
164
- repoUrl,
184
+ ...resolveRepoUrl(config.repoUrl),
165
185
  actorConfigs: actorsChanged,
166
186
  branch: config.sourceBranch.replace('origin/', ''),
167
187
  dryRun: config.dryRun,
@@ -173,17 +193,27 @@ await yargs()
173
193
  .command(
174
194
  'release',
175
195
  '',
176
- (args) =>
177
- actorSelectionOptions(args)
178
- .option('push-event-path', { type: 'string', demandOption: true })
179
- .option('dry-run', { type: 'boolean', default: false })
196
+ (y) =>
197
+ y
198
+ .options(actorSelectionOptions)
199
+ .options(buildOptions)
200
+ .options(repoUrlOptions)
201
+ // The last commit that is already released, everything after it up to HEAD gets released.
202
+ // Required, see the README section "Releasing Actors" for how to set it for each merge strategy.
203
+ .option('base-commit', { type: 'string', demandOption: true })
180
204
  .option('report-slack-channel', { type: 'string' })
181
- .option('release-slack-channel', { type: 'string' })
182
- .option('use-docker-cache', { type: 'boolean', default: false }),
205
+ .option('release-slack-channel', { type: 'string' }),
183
206
  async (args) => {
184
- const { branch, changedFiles, repoUrl, commits, changelog, repository, author } = await getPushData(
185
- args.pushEventPath,
186
- );
207
+ const baseSha = resolveReleaseBaseCommit(args.baseCommit);
208
+ const branch = getCurrentBranch();
209
+ const changes = getReleaseChanges(baseSha);
210
+ if (!changes) {
211
+ console.error(`HEAD is the base commit ${baseSha}, there is nothing new to release`);
212
+ return;
213
+ }
214
+ const { commits, changedFiles, changelog } = changes;
215
+ const { repoUrl, shouldVerifyRepoUrl } = resolveRepoUrl(args.repoUrl);
216
+
187
217
  const isLatest = true;
188
218
  const actorConfigs = await readConfigFile(args);
189
219
  const actorsChanged = getChangedActors({
@@ -196,6 +226,7 @@ await yargs()
196
226
  const builds = await runBuilds({
197
227
  isLatest,
198
228
  repoUrl,
229
+ shouldVerifyRepoUrl,
199
230
  actorConfigs: actorsChanged,
200
231
  branch,
201
232
  dryRun,
@@ -207,9 +238,10 @@ await yargs()
207
238
  changedFiles,
208
239
  commits,
209
240
  changelog,
210
- repository,
241
+ repository: getRepoName(repoUrl),
211
242
  dryRun,
212
- author,
243
+ // The newest commit, same as the pushed head commit
244
+ author: commits[commits.length - 1].author,
213
245
  reportSlackChannel,
214
246
  releaseSlackChannel,
215
247
  });
@@ -218,17 +250,22 @@ await yargs()
218
250
  .command(
219
251
  'build-from-local',
220
252
  '',
221
- (args) => actorSelectionOptions(args).option('dry-run', { type: 'boolean', default: false }),
253
+ (y) => y.options(actorSelectionOptions),
222
254
  async ({ actors, ignore, dryRun }) => {
223
255
  const actorConfigs = await readConfigFile({ actors, ignore });
224
256
  const builds = await runBuildsFromLocal({ actorConfigs, dryRun });
225
257
  console.log(JSON.stringify(builds));
226
258
  },
227
259
  )
228
- .command('delete-old-builds', '', actorSelectionOptions, async ({ actors, ignore }) => {
229
- const actorConfigs = await readConfigFile({ actors, ignore });
230
- await deleteOldBuilds(actorConfigs);
231
- })
260
+ .command(
261
+ 'delete-old-builds',
262
+ '',
263
+ (y) => y.options(actorSelectionOptions),
264
+ async ({ actors, ignore }) => {
265
+ const actorConfigs = await readConfigFile({ actors, ignore });
266
+ await deleteOldBuilds(actorConfigs);
267
+ },
268
+ )
232
269
  .strictCommands()
233
270
  .demandCommand(1, 'Command is required')
234
271
  .fail((msg, err, yargsInstance) => {
package/bin/slack.ts CHANGED
@@ -14,6 +14,9 @@ type NotifyToSlackOptions = {
14
14
  reportSlackChannel?: string;
15
15
  };
16
16
 
17
+ // Git authors come as "Name<email>", Slack only needs the name
18
+ const displayName = (author: string) => author.replace(/\s*<[^>]*>$/, '');
19
+
17
20
  export const notifyToSlack = async ({
18
21
  changedFiles,
19
22
  commits,
@@ -30,7 +33,7 @@ export const notifyToSlack = async ({
30
33
  console.warn('No new changelog entries found, did you forget to update it?');
31
34
  }
32
35
 
33
- let shortMessage = `*${repository}* – New release (by ${author}):\n\n`;
36
+ let shortMessage = `*${repository}* – New release (by ${displayName(author)}):\n\n`;
34
37
 
35
38
  // This one is just for broader public that only cares about public facing changes
36
39
  if (changelog && releaseSlackChannel) {
@@ -49,7 +52,7 @@ export const notifyToSlack = async ({
49
52
  const commitsMessage = `${commits
50
53
  .map(
51
54
  ({ author: commitAuthor, message }, index) =>
52
- `${index + 1}. Commit message: ${message}\n\tAuthor: ${commitAuthor}.`,
55
+ `${index + 1}. Commit message: ${message}\n\tAuthor: ${displayName(commitAuthor)}.`,
53
56
  )
54
57
  .join('\n')}`;
55
58
  const changedFilesMessage = `*Files changed*: ${changedFiles.map((file) => `\`${file}\``).join(', ')}`;
package/bin/types.ts CHANGED
@@ -14,79 +14,6 @@ export type Commit = {
14
14
  message: string;
15
15
  };
16
16
 
17
- // NOTE: The GitHub types are incomplete, feel free to add fields/complete (not sure how stable GitHub API is)
18
- export interface GitHubHeadCommit {
19
- added: string[];
20
- removed: string[];
21
- modified: string[];
22
- author: {
23
- name: string;
24
- };
25
- message: string;
26
- id: string;
27
- }
28
-
29
- export interface Repository {
30
- full_name: string;
31
- name: string;
32
- ssh_url: string;
33
- owner: {
34
- login: string;
35
- };
36
- }
37
-
38
- export type GitHubEvent = GitHubEventPullRequest | GitHubEventPush;
39
-
40
- export interface GitHubEventPullRequest {
41
- // Type is our own variable we inject to have discriminated union
42
- // The rest of the properties are raw from GitHub API
43
- type: 'pull_request';
44
- pull_request: {
45
- number: number;
46
- base: {
47
- ref: string;
48
- sha: string;
49
- };
50
- head: {
51
- ref: string;
52
- sha: string;
53
- };
54
- };
55
- repository: Repository;
56
- }
57
-
58
- export interface GitHubEventPush {
59
- type: 'push';
60
- head_commit: GitHubHeadCommit;
61
- repository: Repository;
62
- ref: string;
63
- commits: GithubCommit[];
64
- before: string;
65
- after: string;
66
- }
67
-
68
- export interface GithubCommit {
69
- id: string;
70
- tree_id: string;
71
- distinct: boolean;
72
- message: string;
73
- timestamp: string;
74
- url: string;
75
- author: {
76
- name: string;
77
- email: string;
78
- username: string;
79
- };
80
- committer: {
81
- name: string;
82
- email: string;
83
- username: string;
84
- };
85
- added: string[];
86
- removed: string[];
87
- modified: string[];
88
- }
89
-
90
17
  export interface BuildData {
91
18
  buildId: string;
92
19
  actorRawId: string;
@@ -1,6 +1,7 @@
1
1
  import z from 'zod';
2
2
 
3
3
  import { CONFIG_FILE_STRATEGY, type ResolvedActorConfig, type StrategyParser } from './structures/base.js';
4
+ import { GROUPED_PARSER } from './structures/grouped.js';
4
5
  import { LEGACY_PARSER } from './structures/legacy.js';
5
6
 
6
7
  const CONFIG_FILE_STRATEGIES: {
@@ -8,9 +9,10 @@ const CONFIG_FILE_STRATEGIES: {
8
9
  [key in CONFIG_FILE_STRATEGY]: StrategyParser & { mode: key };
9
10
  } = {
10
11
  [CONFIG_FILE_STRATEGY.LEGACY]: LEGACY_PARSER,
12
+ [CONFIG_FILE_STRATEGY.GROUPED]: GROUPED_PARSER,
11
13
  } as const;
12
14
 
13
- const ModeSelectionSchema = z.enum(CONFIG_FILE_STRATEGY).default(CONFIG_FILE_STRATEGY.LEGACY);
15
+ const ModeSelectionSchema = z.object({ mode: z.enum(CONFIG_FILE_STRATEGY).default(CONFIG_FILE_STRATEGY.LEGACY) });
14
16
 
15
17
  function selectStrategy(mode: unknown): StrategyParser {
16
18
  const parsed = ModeSelectionSchema.safeParse(mode);
@@ -18,7 +20,7 @@ function selectStrategy(mode: unknown): StrategyParser {
18
20
  throw new Error(z.prettifyError(parsed.error));
19
21
  }
20
22
 
21
- return CONFIG_FILE_STRATEGIES[parsed.data];
23
+ return CONFIG_FILE_STRATEGIES[parsed.data.mode];
22
24
  }
23
25
 
24
26
  // Strips a trailing slash so config-declared paths ("actors/shopify/" vs "actors/shopify") compare equal.
@@ -79,10 +81,9 @@ export const _privates = {
79
81
  };
80
82
 
81
83
  export function parseConfigFile(body: Record<string, unknown>): ResolvedActorConfig[] {
82
- const { mode, ...rest } = body;
83
- const strategy = selectStrategy(mode);
84
+ const strategy = selectStrategy(body);
84
85
 
85
- const resolved = strategy.parse(rest);
86
+ const resolved = strategy.parse(body);
86
87
  const validated = verifyConfiguration(resolved);
87
88
  return validated;
88
89
  }
@@ -11,15 +11,19 @@ export const ACTOR_FULL_NAME_REGEX = new RegExp(
11
11
  USERNAME.REGEX.flags + ACTOR_NAME.REGEX.flags,
12
12
  );
13
13
 
14
+ // All fields must exist, even if their value can be undefined.
15
+ // This makes typescript enforce all configs to have resolution for EVERY field.
16
+ // So adding an extra field here creates TS errors in every config file that doesn't have it.
14
17
  export interface ResolvedActorConfig {
15
18
  actorFullName: string;
16
19
  folder: string;
17
20
  tokenEnvVar: string;
18
- overrideActorContext?: string[];
21
+ overrideActorContext: string[] | undefined;
19
22
  }
20
23
 
21
24
  export enum CONFIG_FILE_STRATEGY {
22
25
  LEGACY = 'legacy',
26
+ GROUPED = 'grouped',
23
27
  }
24
28
 
25
29
  export type StrategyParser = {
@@ -0,0 +1,40 @@
1
+ import z from 'zod';
2
+
3
+ import { ACTOR_FULL_NAME_REGEX, CONFIG_FILE_STRATEGY, defineStrategy } from './base.js';
4
+
5
+ const schema = z.object({
6
+ groups: z
7
+ .record(
8
+ z.string(),
9
+ z.object({
10
+ actors: z
11
+ .array(
12
+ z.object({
13
+ folder: z.string(),
14
+ actorFullName: z.string().regex(ACTOR_FULL_NAME_REGEX),
15
+ // these allow overrides from stuff set at the group level
16
+ tokenEnvVar: z.string().optional(),
17
+ overrideActorContext: z.array(z.string()).optional(),
18
+ }),
19
+ )
20
+ .min(1),
21
+ tokenEnvVar: z.string(),
22
+ overrideActorContext: z.array(z.string()).optional(),
23
+ }),
24
+ )
25
+ .refine((data) => Object.keys(data).length > 0, 'Invalid input: At least one configuration group is required.'),
26
+ });
27
+
28
+ type GroupedConfig = z.infer<typeof schema>;
29
+
30
+ export const GROUPED_PARSER = defineStrategy(CONFIG_FILE_STRATEGY.GROUPED, schema, (body: GroupedConfig) => {
31
+ const entries = Object.entries(body.groups);
32
+ return entries.flatMap(([_folder, entry]) =>
33
+ entry.actors.map((actor) => ({
34
+ tokenEnvVar: entry.tokenEnvVar,
35
+ overrideActorContext: entry.overrideActorContext,
36
+ // last so actor can override stuff from the group
37
+ ...actor,
38
+ })),
39
+ );
40
+ });
@@ -17,4 +17,6 @@ const schema = z.object({
17
17
 
18
18
  type LegacyConfig = z.infer<typeof schema>;
19
19
 
20
- export const LEGACY_PARSER = defineStrategy(CONFIG_FILE_STRATEGY.LEGACY, schema, (body: LegacyConfig) => body.actors);
20
+ export const LEGACY_PARSER = defineStrategy(CONFIG_FILE_STRATEGY.LEGACY, schema, (body: LegacyConfig) =>
21
+ body.actors.map((x) => ({ overrideActorContext: undefined, ...x })),
22
+ );
package/bin/utils.ts CHANGED
@@ -2,10 +2,6 @@ import { spawnSync } from 'node:child_process';
2
2
  import fs from 'node:fs/promises';
3
3
  import path from 'node:path';
4
4
 
5
- import type { ActorVersionSourceFile } from 'apify-client';
6
-
7
- import { SOURCE_FILE_FORMATS } from '@apify/consts';
8
-
9
5
  // Returns true when `childPath` is not inside `parentPath`.
10
6
  // Used to detect monorepo actors whose dockerContextDir escapes the actor directory.
11
7
  export const isOutsideDir = (childPath: string, parentPath: string): boolean =>
@@ -55,15 +51,12 @@ export const getGitignoredPaths = (relativePaths: string[]): Set<string> => {
55
51
  return new Set(result.stdout.toString().split('\n').filter(Boolean));
56
52
  };
57
53
 
58
- const isBinary = (buffer: Buffer): boolean => buffer.includes(0);
54
+ export type SourceFile = { name: string; content: Buffer };
59
55
 
60
- export const toActorVersionSourceFile = async (absPath: string, rootDir: string): Promise<ActorVersionSourceFile> => {
61
- const buffer = await fs.readFile(absPath);
62
- const name = path.relative(rootDir, absPath).split(path.sep).join('/');
63
- return isBinary(buffer)
64
- ? { name, format: SOURCE_FILE_FORMATS.BASE64, content: buffer.toString('base64') }
65
- : { name, format: SOURCE_FILE_FORMATS.TEXT, content: buffer.toString('utf8') };
66
- };
56
+ export const readSourceFile = async (absPath: string, rootDir: string): Promise<SourceFile> => ({
57
+ name: path.relative(rootDir, absPath).split(path.sep).join('/'),
58
+ content: await fs.readFile(absPath),
59
+ });
67
60
 
68
61
  export const spawnCommandInGhWorkspace = (command: string, args: string[] = []) => {
69
62
  console.error(command, args.join(' '));
@@ -91,11 +84,10 @@ export const getEnvVar = (varName: string, defaultValue?: string): string => {
91
84
  return value;
92
85
  };
93
86
 
87
+ // Commands expect to run from the repository root (config file and Actor folders are relative to it).
88
+ // That is the current directory by default, --workspace points elsewhere, e.g. for local development.
94
89
  export const setCwd = ({ workspace }: { workspace: string | undefined }) => {
95
90
  if (workspace) {
96
91
  process.chdir(workspace);
97
- return;
98
92
  }
99
- const ghWorkspace = getEnvVar('GITHUB_WORKSPACE', process.cwd());
100
- process.chdir(ghWorkspace);
101
93
  };
@@ -1,6 +1,6 @@
1
- import type { ActorVersionSourceFile } from 'apify-client';
2
1
  import type { ActorConfig, BuildData } from './types.js';
3
- export declare const collectSourceFiles: (actorName: string, actorDir: string) => Promise<ActorVersionSourceFile[]>;
2
+ import type { SourceFile } from './utils.js';
3
+ export declare const collectSourceFiles: (actorName: string, actorDir: string) => Promise<SourceFile[]>;
4
4
  export declare const collectNonIgnoredFiles: (dockerContextDir: string, repoRoot: string) => string[];
5
5
  export declare const flattenMonorepoContext: (actorName: string, absActorDir: string, contextAbsDir: string, actorJson: Record<string, unknown>, keptContextFiles: string[]) => Promise<{
6
6
  tempDir: string;
@@ -1 +1 @@
1
- {"version":3,"file":"build-from-local.d.ts","sourceRoot":"","sources":["../../bin/build-from-local.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAgB,sBAAsB,EAAE,MAAM,cAAc,CAAC;AAMzE,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAUzD,eAAO,MAAM,kBAAkB,GAAU,WAAW,MAAM,EAAE,UAAU,MAAM,KAAG,OAAO,CAAC,sBAAsB,EAAE,CAmC9G,CAAC;AAWF,eAAO,MAAM,sBAAsB,GAAI,kBAAkB,MAAM,EAAE,UAAU,MAAM,KAAG,MAAM,EAoBzF,CAAC;AAaF,eAAO,MAAM,sBAAsB,GAC/B,WAAW,MAAM,EACjB,aAAa,MAAM,EACnB,eAAe,MAAM,EACrB,WAAW,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAClC,kBAAkB,MAAM,EAAE,KAC3B,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,EAAE,CAAA;CAAE,CAyClD,CAAC;AAcF,eAAO,MAAM,qBAAqB,GAC9B,aAAa,MAAM,EACnB,eAAe,MAAM,EACrB,SAAS,MAAM,EACf,WAAW,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KACnC,OAAO,CAAC,IAAI,CAqBd,CAAC;AAEF,eAAO,MAAM,kBAAkB,GAAU,2BAGtC;IACC,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,MAAM,EAAE,OAAO,CAAC;CACnB,KAAG,OAAO,CAAC,SAAS,EAAE,CAkBtB,CAAC"}
1
+ {"version":3,"file":"build-from-local.d.ts","sourceRoot":"","sources":["../../bin/build-from-local.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACzD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAU7C,eAAO,MAAM,kBAAkB,GAAU,WAAW,MAAM,EAAE,UAAU,MAAM,KAAG,OAAO,CAAC,UAAU,EAAE,CAiClG,CAAC;AAWF,eAAO,MAAM,sBAAsB,GAAI,kBAAkB,MAAM,EAAE,UAAU,MAAM,KAAG,MAAM,EAoBzF,CAAC;AAaF,eAAO,MAAM,sBAAsB,GAC/B,WAAW,MAAM,EACjB,aAAa,MAAM,EACnB,eAAe,MAAM,EACrB,WAAW,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAClC,kBAAkB,MAAM,EAAE,KAC3B,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,EAAE,CAAA;CAAE,CAyClD,CAAC;AAcF,eAAO,MAAM,qBAAqB,GAC9B,aAAa,MAAM,EACnB,eAAe,MAAM,EACrB,SAAS,MAAM,EACf,WAAW,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KACnC,OAAO,CAAC,IAAI,CAqBd,CAAC;AAEF,eAAO,MAAM,kBAAkB,GAAU,2BAGtC;IACC,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,MAAM,EAAE,OAAO,CAAC;CACnB,KAAG,OAAO,CAAC,SAAS,EAAE,CAsBtB,CAAC"}
@@ -1,11 +1,12 @@
1
1
  import fs from 'node:fs/promises';
2
2
  import os from 'node:os';
3
3
  import path from 'node:path';
4
+ import AdmZip from 'adm-zip';
4
5
  import { ActorSourceType } from 'apify-client';
5
6
  import { dryRunBuildData, LOCAL_SOURCE_VERSION_NUMBER, runAndSummarizeBuilds } from './build.js';
6
7
  import { buildDockerIgnoreMatcher } from './dockerignore.js';
7
8
  import { isPathWithinScope } from './path-utils.js';
8
- import { getGitignoredPaths, isOutsideDir, listRepoFilePaths, toActorVersionSourceFile } from './utils.js';
9
+ import { getGitignoredPaths, isOutsideDir, listRepoFilePaths, readSourceFile } from './utils.js';
9
10
  // JUST IN CASE. File patterns that commonly hold credentials — never ship these into a build, regardless
10
11
  // of sourceType or of whether the repo's .gitignore happens to list them. Everything else that should be
11
12
  // excluded (build output, local overrides, project-specific secret files, ...) is expected to already be
@@ -26,11 +27,11 @@ export const collectSourceFiles = async (actorName, actorDir) => {
26
27
  const dockerContextDirAbs = isMonorepoActor ? contextAbsDir : absActorDir;
27
28
  const keptFilePaths = collectNonIgnoredFiles(dockerContextDirAbs, repoRoot);
28
29
  if (!isMonorepoActor) {
29
- return Promise.all(keptFilePaths.map(async (filePath) => toActorVersionSourceFile(filePath, dockerContextDirAbs)));
30
+ return Promise.all(keptFilePaths.map(async (filePath) => readSourceFile(filePath, dockerContextDirAbs)));
30
31
  }
31
32
  const { tempDir, filePaths } = await flattenMonorepoContext(actorName, absActorDir, contextAbsDir, actorJson, keptFilePaths);
32
33
  try {
33
- return await Promise.all(filePaths.map(async (filePath) => toActorVersionSourceFile(filePath, tempDir)));
34
+ return await Promise.all(filePaths.map(async (filePath) => readSourceFile(filePath, tempDir)));
34
35
  }
35
36
  finally {
36
37
  // Only the flattened copy is temporary — never delete the actor's own directory.
@@ -153,11 +154,15 @@ export const runBuildsFromLocal = async ({ actorConfigs, dryRun, }) => {
153
154
  return actorConfigs.map(({ actorFullName }) => dryRunBuildData(actorFullName, LOCAL_SOURCE_VERSION_NUMBER));
154
155
  }
155
156
  return runAndSummarizeBuilds(actorConfigs, 'LOCAL BUILDS', async (actorConfig, builder) => {
156
- const sourceFiles = await collectSourceFiles(actorConfig.actorFullName, actorConfig.folder);
157
+ // Zipped and uploaded like `apify push` does, since inline SOURCE_FILES are capped at ~9 MB.
158
+ const zip = new AdmZip();
159
+ for (const { name, content } of await collectSourceFiles(actorConfig.actorFullName, actorConfig.folder)) {
160
+ zip.addFile(name, content);
161
+ }
157
162
  const actorVersion = {
158
163
  versionNumber: LOCAL_SOURCE_VERSION_NUMBER,
159
- sourceFiles,
160
- sourceType: ActorSourceType.SourceFiles,
164
+ tarballUrl: await builder.uploadSourceZip(LOCAL_SOURCE_VERSION_NUMBER, zip.toBuffer()),
165
+ sourceType: ActorSourceType.Tarball,
161
166
  };
162
167
  return builder.createVersionAndBuild(LOCAL_SOURCE_VERSION_NUMBER, actorVersion, false);
163
168
  });