apify-test-tools 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/README.md +23 -0
  2. package/bin/build.ts +287 -0
  3. package/bin/consts.ts +6 -0
  4. package/bin/git.ts +56 -0
  5. package/bin/github.ts +75 -0
  6. package/bin/main.ts +145 -0
  7. package/bin/slack.ts +72 -0
  8. package/bin/test-report.ts +146 -0
  9. package/bin/types.ts +102 -0
  10. package/bin/utils.ts +204 -0
  11. package/dist/bin/build.d.ts +18 -0
  12. package/dist/bin/build.d.ts.map +1 -0
  13. package/dist/bin/build.js +222 -0
  14. package/dist/bin/build.js.map +1 -0
  15. package/dist/bin/consts.d.ts +4 -0
  16. package/dist/bin/consts.d.ts.map +1 -0
  17. package/dist/bin/consts.js +6 -0
  18. package/dist/bin/consts.js.map +1 -0
  19. package/dist/bin/git.d.ts +7 -0
  20. package/dist/bin/git.d.ts.map +1 -0
  21. package/dist/bin/git.js +47 -0
  22. package/dist/bin/git.js.map +1 -0
  23. package/dist/bin/github.d.ts +12 -0
  24. package/dist/bin/github.d.ts.map +1 -0
  25. package/dist/bin/github.js +64 -0
  26. package/dist/bin/github.js.map +1 -0
  27. package/dist/bin/main.d.ts +3 -0
  28. package/dist/bin/main.d.ts.map +1 -0
  29. package/dist/bin/main.js +117 -0
  30. package/dist/bin/main.js.map +1 -0
  31. package/dist/bin/slack.d.ts +12 -0
  32. package/dist/bin/slack.d.ts.map +1 -0
  33. package/dist/bin/slack.js +46 -0
  34. package/dist/bin/slack.js.map +1 -0
  35. package/dist/bin/test-report.d.ts +10 -0
  36. package/dist/bin/test-report.d.ts.map +1 -0
  37. package/dist/bin/test-report.js +74 -0
  38. package/dist/bin/test-report.js.map +1 -0
  39. package/dist/bin/types.d.ts +88 -0
  40. package/dist/bin/types.d.ts.map +1 -0
  41. package/dist/bin/types.js +2 -0
  42. package/dist/bin/types.js.map +1 -0
  43. package/dist/bin/utils.d.ts +26 -0
  44. package/dist/bin/utils.d.ts.map +1 -0
  45. package/dist/bin/utils.js +164 -0
  46. package/dist/bin/utils.js.map +1 -0
  47. package/dist/index.d.ts +3 -0
  48. package/dist/index.d.ts.map +1 -0
  49. package/dist/index.js +3 -0
  50. package/dist/index.js.map +1 -0
  51. package/dist/lib/consts.d.ts +3 -0
  52. package/dist/lib/consts.d.ts.map +1 -0
  53. package/dist/lib/consts.js +14 -0
  54. package/dist/lib/consts.js.map +1 -0
  55. package/dist/lib/extend-expect.d.ts +3 -0
  56. package/dist/lib/extend-expect.d.ts.map +1 -0
  57. package/dist/lib/extend-expect.js +228 -0
  58. package/dist/lib/extend-expect.js.map +1 -0
  59. package/dist/lib/lib.d.ts +16 -0
  60. package/dist/lib/lib.d.ts.map +1 -0
  61. package/dist/lib/lib.js +91 -0
  62. package/dist/lib/lib.js.map +1 -0
  63. package/dist/lib/run-test-result.d.ts +20 -0
  64. package/dist/lib/run-test-result.d.ts.map +1 -0
  65. package/dist/lib/run-test-result.js +62 -0
  66. package/dist/lib/run-test-result.js.map +1 -0
  67. package/dist/lib/types.d.ts +107 -0
  68. package/dist/lib/types.d.ts.map +1 -0
  69. package/dist/lib/types.js +2 -0
  70. package/dist/lib/types.js.map +1 -0
  71. package/dist/lib/utils.d.ts +7 -0
  72. package/dist/lib/utils.d.ts.map +1 -0
  73. package/dist/lib/utils.js +30 -0
  74. package/dist/lib/utils.js.map +1 -0
  75. package/dist/test/unit/custom-matchers.test.d.ts +2 -0
  76. package/dist/test/unit/custom-matchers.test.d.ts.map +1 -0
  77. package/dist/test/unit/custom-matchers.test.js +53 -0
  78. package/dist/test/unit/custom-matchers.test.js.map +1 -0
  79. package/dist/test/unit/should-built-and-test.test.d.ts +2 -0
  80. package/dist/test/unit/should-built-and-test.test.d.ts.map +1 -0
  81. package/dist/test/unit/should-built-and-test.test.js +136 -0
  82. package/dist/test/unit/should-built-and-test.test.js.map +1 -0
  83. package/dist/test/unit/test-actor.test.d.ts +2 -0
  84. package/dist/test/unit/test-actor.test.d.ts.map +1 -0
  85. package/dist/test/unit/test-actor.test.js +17 -0
  86. package/dist/test/unit/test-actor.test.js.map +1 -0
  87. package/dist/tsconfig.tsbuildinfo +1 -0
  88. package/example-github-events/pull-request.json +523 -0
  89. package/example-github-events/push.json +216 -0
  90. package/index.ts +2 -0
  91. package/lib/consts.ts +15 -0
  92. package/lib/extend-expect.ts +268 -0
  93. package/lib/lib.ts +134 -0
  94. package/lib/run-test-result.ts +70 -0
  95. package/lib/types.ts +131 -0
  96. package/lib/utils.ts +49 -0
  97. package/package.json +41 -0
  98. package/scripts/hook-job-started.sh +11 -0
  99. package/test/tsconfig.json +15 -0
  100. package/test/unit/custom-matchers.test.ts +61 -0
  101. package/test/unit/should-built-and-test.test.ts +166 -0
  102. package/test/unit/test-actor.test.ts +17 -0
  103. package/tsconfig.json +20 -0
package/README.md ADDED
@@ -0,0 +1,23 @@
1
+ This Actor has essentially 2 separate parts
2
+
3
+ 1. The Actor loads GitHub actions binary on build. When it is started, it will run `runner.ts` which spawns the GitHub actions server as a separate process. Then it only listens for migrations and does nothing else.
4
+ 2. The GitHub actions server binary (which we don't control) is then triggered by GitHub action and that subsequently calls `main.ts` (via calling it from `.github/workflows/apify.yaml` and `gha:start:build` command). This runs rest of the code in the files.
5
+
6
+ ## Run locally to test
7
+ 1. Choose if you want to emulate PR or push to master workflow.
8
+ 2. Copy appropriate example from `./example-github-events` to `INPUT.json`
9
+ 3. To mimic actual PR or push you care about, you should update the input
10
+ - PR: Update `pull_request.head.ref` (source branch of the PR), `repository.full_name`, `pull_request.base.sha`, `pull_request.head.sha`
11
+ - push: Update `ref`, `repository.full_name`
12
+ 4. Start the Actor with `TESTER_APIFY_TOKEN=<TOKEN> APIFY_TOKEN_${username}=<TOKEN> apify run -p` (possibly other apify users as well)
13
+
14
+ ## Testing real Actor
15
+ Do anything to https://github.com/apify-projects/store-testing-repo-for-github
16
+
17
+ ## Runner on Apify
18
+ We run GitHub Actions in Apify Actor that loads the GitHub Actions server binary. This server is then triggered by GitHub Actions and runs the rest of the code. There is one production forever running instance of this Actor (https://console.apify.com/admin/users/xRGg9iAfJSymqartk/actors/ANeZceuSfUd7i64qs/runs/hcTVcLB94TsOvPQFJ#log). If you need to update the build, just abort and resurrect the run with rebuilt `latest` version.
19
+
20
+ For testing on platform, build the beta version (you can change the branch) and then resurrect the test task of the runner. Don't forget to abort the run after the test is done.
21
+
22
+ ### Multiple production runners
23
+ It is possible to run multiple production runners at the same time. We might need this to scale up in the future since we are currently limited by only running one GitHub Action at a time.
package/bin/build.ts ADDED
@@ -0,0 +1,287 @@
1
+ import { ACTOR_SOURCE_TYPES } from '@apify/consts';
2
+ import { ApifyClient, Build } from 'apify-client';
3
+ import type { ActorConfig, BuildData } from './types.js';
4
+ import { getEnvVar } from './utils.js';
5
+
6
+ type BuildPrActorOptions = {
7
+ buildTag?: string
8
+ versionNumber: string
9
+ gitRepoUrl: string
10
+ actorName: string
11
+ }
12
+ class ApifyBuilder {
13
+ // eslint-disable-next-line no-empty-function
14
+ private constructor(private readonly apifyClient: ApifyClient, private readonly actorName: string) { }
15
+
16
+ // Usually 'latest' but not necessarily (can be e.g. 'version-0')
17
+ getDefaultVersionAndTag = async (): Promise<{ defaultBuildNumber: string, defaultVersionNumber: string, defaultBuildTag: string }> => {
18
+ const actorClient = this.apifyClient.actor(this.actorName);
19
+ const actorInfo = await actorClient.get();
20
+
21
+ if (!actorInfo) {
22
+ throw new Error(`[${this.actorName}] not found. It is not published or we are missing token to access it privately or its name is misspelled`);
23
+ }
24
+
25
+ const defaultBuildTag = actorInfo.defaultRunOptions.build;
26
+ console.error(`Default build tag for ${this.actorName} is ${defaultBuildTag}`);
27
+
28
+ // We could technically allow this but in most cases this is accidentally set wrongly and there is a workaround
29
+ if (defaultBuildTag.match(/\d+\.\d+\.\d+/)) {
30
+ throw new Error(`[${this.actorName}] Default build is a build number, not a tag. While this could work, `
31
+ + `we want to have a default as tag so this is often an accidental misconfiguration from the dev`);
32
+ }
33
+ // I reported that buildNumber should probably not be optional
34
+ const defaultBuildNumber = actorInfo.taggedBuilds![defaultBuildTag].buildNumber!;
35
+ const defaultVersionNumber = defaultBuildNumber.match(/(\d+\.\d+)\.\d+/)![1];
36
+ console.error(`Default version for ${this.actorName} is ${defaultVersionNumber}`);
37
+
38
+ return { defaultBuildNumber, defaultVersionNumber, defaultBuildTag };
39
+ };
40
+
41
+ startActorBuild = async ({
42
+ buildTag,
43
+ versionNumber,
44
+ gitRepoUrl,
45
+ }: BuildPrActorOptions): Promise<BuildData> => {
46
+ const actorClient = this.apifyClient.actor(this.actorName);
47
+ const actorInfo = await actorClient.get();
48
+ if (!actorInfo) {
49
+ throw new Error(`No actor named '${this.actorName}' was found on the platform. If this`
50
+ + ' is unexpected, make sure the actor you are targeting is spelled the'
51
+ + ' same as the folder in the repository.');
52
+ }
53
+
54
+ // NOTE: I couldn't find this type, so I had to extract it :(
55
+ type ActorVersion = Parameters<ReturnType<typeof actorClient.version>['update']>[0];
56
+ const actorVersion: ActorVersion = {
57
+ buildTag,
58
+ versionNumber,
59
+ gitRepoUrl,
60
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
61
+ // @ts-ignore: coudn't find this type either :(
62
+ sourceType: ACTOR_SOURCE_TYPES.GIT_REPO,
63
+ };
64
+
65
+ // Prepare version
66
+ const versionExists = !actorInfo.versions.find((version) => version.versionNumber === versionNumber);
67
+ if (versionExists) {
68
+ // create new version
69
+ await actorClient.versions().create(actorVersion);
70
+ } else {
71
+ const version = actorClient.version(versionNumber);
72
+ await version.update(actorVersion);
73
+ }
74
+
75
+ console.error(`[BUILD][${this.actorName}]: Will be built with version ${versionNumber}`);
76
+ console.error(`[BUILD][${this.actorName}]: ${JSON.stringify(actorVersion)}`);
77
+
78
+ // We also get back actId so the testing actor can both match by actor ID and name
79
+ const { id, actId, buildNumber } = await actorClient.build(versionNumber);
80
+
81
+ console.error(`[BUILD][${this.actorName}]: Build ${id} (${versionNumber}) has started`);
82
+ return { buildId: id, actorId: actId, buildNumber, actorName: this.actorName };
83
+ };
84
+
85
+ waitForBuildToFinish = async (buildId: string, actorName: string): Promise<Build> => {
86
+ console.error(`[BUILD][${actorName}]: Waiting for build ${buildId} to finish`);
87
+ const build = await this.apifyClient.build(buildId).waitForFinish();
88
+ const versionNumber = build.buildNumber;
89
+ if (build.status === 'FAILED' || build.status === 'TIMED-OUT') {
90
+ const message = `[BUILD][${actorName}]: Build ${buildId} (${versionNumber}) failed. `
91
+ + `Not continuing with other builds and tests.`;
92
+ console.warn(`D€LIV€RY_$L&CK: Webhook-to-build: ${message}`);
93
+ throw new Error(message);
94
+ }
95
+
96
+ console.error(`[BUILD][${actorName}]: Build ${build.id} (${versionNumber}) finished successfully.`);
97
+ return build;
98
+ };
99
+
100
+ /**
101
+ * Create ApifyBuilder with actor owner's token
102
+ */
103
+ static fromActorName = (actorName: string): ApifyBuilder => {
104
+ const username = actorName.split('/')[0];
105
+ // GitHib secrets only allow word characters (alphanum + underscore)
106
+ const usernameInGitHubSecretsFormat = username.replaceAll(/\W/g, '_').toUpperCase();
107
+ const usernameEnvVar = `APIFY_TOKEN_${usernameInGitHubSecretsFormat}`;
108
+ const token = process.env[usernameEnvVar];
109
+ if (!token) {
110
+ throw new Error(`Cannot find Apify API token for username: ${username}. `
111
+ + `Have you set secret env var to this GitHub repo with key: ${usernameEnvVar}?`);
112
+ }
113
+ const apifyClient = new ApifyClient({ token });
114
+ const builder = new ApifyBuilder(apifyClient, actorName);
115
+ return builder;
116
+ };
117
+
118
+ /**
119
+ * Deletes build of all versions. Apify API doesn't allow to delete default build and we explicitly skip it
120
+ * We delete devel builds faster because we used the for every PR until recently so just to get rid of them faster
121
+ */
122
+ async deleteOldBuilds(): Promise<void> {
123
+ // Even though we don't version our current Actors, if we ever such Actors to GitHub CI, we would accidentally delete old supported versions
124
+ // This hardcoded solution is not ideal, but it should prevent most imaginable cases
125
+ // All currently popular versioned Actors use `version-${number}` format
126
+ const PROTECTED_TAGS_PREFIX = ['latest', 'v-', 'version', 'v0', 'v1', 'v2', 'v3', 'v4', 'v5', 'v6', 'v7', 'v8', 'v9'];
127
+
128
+ // We don't want to be too short because we might to debug something
129
+ // but also not too long because it increases the risk of users using outdated versions
130
+ const DEFAULT_DAYS_BACK_PROD_VERSIONS = 30;
131
+ const DEFAULT_DAYS_BACK_DEVEL = 7;
132
+
133
+ const actorInfo = (await this.apifyClient.actor(this.actorName).get())!;
134
+
135
+ // 'devel' used to be hardcoded for testing version 0.99, once we get rid of this tag everywhere, we can remove this code
136
+ const taggedDevelBuildNumber: string | undefined = actorInfo.taggedBuilds!.devel?.buildNumber;
137
+
138
+ const allTags = Object.keys(actorInfo.taggedBuilds ?? {});
139
+ const protectedTags = allTags.filter((tag) => PROTECTED_TAGS_PREFIX.some((prefix) => tag.startsWith(prefix)));
140
+ const protectedBuildNumbers = protectedTags.map((tag) => ({ buildNumber: actorInfo.taggedBuilds![tag]!.buildNumber, tag }));
141
+
142
+ const { items } = (await this.apifyClient.actor(this.actorName).builds().list());
143
+
144
+ // Deleting default build throws an error, so we skip it
145
+ const { defaultBuildNumber, defaultBuildTag } = await ApifyBuilder.fromActorName(this.actorName).getDefaultVersionAndTag();
146
+
147
+ const daysAgoUnixProd = Date.now() - DEFAULT_DAYS_BACK_PROD_VERSIONS * 24 * 60 * 60 * 1000;
148
+ const daysAgoUnixDevel = Date.now() - DEFAULT_DAYS_BACK_DEVEL * 24 * 60 * 60 * 1000;
149
+
150
+ // Fixing API client missing buildNumber field
151
+ type CorrectBuildColletionItem = typeof items[0] & { buildNumber: string };
152
+ const buildsToDelete = (items as CorrectBuildColletionItem[]).filter((build) => {
153
+ if (build.buildNumber === defaultBuildNumber) {
154
+ console.error(`[DELETE OLD BUILDS][${this.actorName}]: Skipping default build ${defaultBuildNumber} (${defaultBuildTag}). `
155
+ + `We never delete default builds`);
156
+ return false;
157
+ }
158
+
159
+ const protectedTagFound = protectedBuildNumbers.find((protectedBuildNumber) => protectedBuildNumber.buildNumber === build.buildNumber);
160
+ if (protectedTagFound) {
161
+ console.error(`[DELETE OLD BUILDS][${this.actorName}]: Skipping protected build ${protectedTagFound.buildNumber} (${protectedTagFound.tag}).`);
162
+ return false;
163
+ }
164
+
165
+ if (taggedDevelBuildNumber && build.buildNumber === taggedDevelBuildNumber) {
166
+ const shouldDeleteDevelBuild = build.startedAt.getTime() < daysAgoUnixDevel;
167
+ if (shouldDeleteDevelBuild) {
168
+ console.error(`[DELETE OLD BUILDS][${this.actorName}]: Removing olf devel build ${taggedDevelBuildNumber}.`);
169
+ }
170
+ return shouldDeleteDevelBuild;
171
+ }
172
+ return build.startedAt.getTime() < daysAgoUnixProd;
173
+ });
174
+
175
+ console.error(`[DELETE OLD BUILDS][${this.actorName}]: Deleting ${buildsToDelete.length} old builds that are non-default and `
176
+ + `older than 30 days from total ${items.length}`);
177
+ for (const build of buildsToDelete) {
178
+ await this.apifyClient.build(build.id).delete();
179
+ }
180
+ }
181
+ }
182
+
183
+ type RunBuildsOptions = {
184
+ actorConfigs: ActorConfig[]
185
+ isLatest?: boolean
186
+ repoUrl: string
187
+ branch: string
188
+ dryRun: boolean
189
+ }
190
+
191
+ export const runBuilds = async ({
192
+ repoUrl,
193
+ branch,
194
+ actorConfigs,
195
+ isLatest = false,
196
+ dryRun,
197
+ }: RunBuildsOptions) => {
198
+ const buildConfigs: BuildPrActorOptions[] = [];
199
+
200
+ const circleActors = isLatest ? await findCircleApifyManaged(actorConfigs) : [];
201
+
202
+ for (const { actorName, folder } of actorConfigs.concat(circleActors)) {
203
+ let versionNumber: string;
204
+ let buildTag: string | undefined;
205
+
206
+ if (isLatest) {
207
+ const { defaultVersionNumber, defaultBuildTag } = await ApifyBuilder.fromActorName(actorName).getDefaultVersionAndTag();
208
+ versionNumber = defaultVersionNumber;
209
+ buildTag = defaultBuildTag;
210
+ } else {
211
+ versionNumber = '0.99';
212
+ }
213
+
214
+ // Depending on if these are miniactors or standaloneActors
215
+ let gitRepoUrl = `${repoUrl}#${branch}`;
216
+ if (folder) {
217
+ gitRepoUrl = `${gitRepoUrl}:${folder}`;
218
+ }
219
+ buildConfigs.push({ actorName, gitRepoUrl, versionNumber, buildTag });
220
+ }
221
+
222
+ if (dryRun) {
223
+ return buildConfigs;
224
+ }
225
+
226
+ const startedBuilds = await Promise.all(buildConfigs.map(async (buildConfig) => {
227
+ const builder = ApifyBuilder.fromActorName(buildConfig.actorName);
228
+ const buildData = await builder.startActorBuild(buildConfig);
229
+ return buildData;
230
+ }));
231
+
232
+ await Promise.all(startedBuilds.map(async (buildData) => {
233
+ const builder = ApifyBuilder.fromActorName(buildData.actorName);
234
+ await builder.waitForBuildToFinish(buildData.buildId, buildData.actorName);
235
+ }));
236
+
237
+ return startedBuilds;
238
+ };
239
+
240
+ export const deleteOldBuilds = async (actorConfigs: ActorConfig[]) => {
241
+ for (const { actorName } of actorConfigs) {
242
+ await ApifyBuilder.fromActorName(actorName).deleteOldBuilds();
243
+ }
244
+ };
245
+
246
+ /**
247
+ * We will read all Actors in the circ_le account and build those that match by name pattern
248
+ * There are many ways to approach this, a more robust one would be to have a map of Actors
249
+ * which would allow to have more than one special user per Actor
250
+ * But since that use-case might never be needed, I went with the simplest solution that doesn't require maintaining the map
251
+ * NOTE: One issue is that if any Actor is renamed, we will not match it in the circ_le account nor throw any error
252
+ */
253
+ const findCircleApifyManaged = async (actorConfigs: ActorConfig[]): Promise<ActorConfig[]> => {
254
+ // This token is hardcoded in the runner Actor, locally you have to inject it
255
+ const client = new ApifyClient({ token: getEnvVar('APIFY_TOKEN_CIRC_LE') });
256
+
257
+ const { items: circleActors } = await client.actors().list();
258
+
259
+ const actorsToBuild = circleActors.map<ActorConfig | undefined>((circleActor) => {
260
+ // They prefix all with apify-managed---, I communicated with Jacques to keep doing that
261
+ let actorConfigFound = actorConfigs.find((actorConfig) => circleActor.name.replace('apify-managed---', '') === actorConfig.actorName.split('/')[1]);
262
+
263
+ // Hack for bad naming of circ_le/apify-managed-google-search, we don't want to rename now to break customers
264
+ if (!actorConfigFound && circleActor.name === 'apify-managed-google-search') {
265
+ actorConfigFound = actorConfigs.find((actorConfig) => actorConfig.actorName.split('/')[1] === 'google-search-scraper');
266
+ }
267
+
268
+ if (actorConfigFound) {
269
+ return {
270
+ // We point the circle Actor to the repo folder
271
+ actorName: `${circleActor.username}/${circleActor.name}`,
272
+ folder: actorConfigFound.folder,
273
+ isStandalone: actorConfigFound.isStandalone,
274
+ };
275
+ }
276
+ return undefined;
277
+ }).filter((config) => config !== undefined);
278
+
279
+ console.error(`Found ${actorsToBuild.length} circ_le actors that match Actors we built out of total ${circleActors.length} circ_le actors`);
280
+ console.error(`All circ_le actors: ${circleActors.map((actor) => actor.name).join(', ')}`);
281
+ console.error(`circ_le Actors to build: ${actorsToBuild.map((actor) => actor.actorName).join(', ')}`);
282
+
283
+ if (actorsToBuild.length === 0) {
284
+ console.error('No circ_le actors to build');
285
+ }
286
+ return actorsToBuild;
287
+ };
package/bin/consts.ts ADDED
@@ -0,0 +1,6 @@
1
+ // Technically, upgrades in shared or packages might not need impact every actor
2
+ // but we cannot really know that
3
+ export const DEFAULT_BUILD_ALL_FOLDERS = ['code', 'shared', 'packages'];
4
+
5
+ export const PERSISTED_GH_JOBS_KVS_ID = 'XIBdO8EJePdD0KiAI';
6
+ export const MINIACTORS_LIST_STORE_ID = 'ftXklt2f2mN30Oc3y';
package/bin/git.ts ADDED
@@ -0,0 +1,56 @@
1
+ import { Commit, Config } from './types';
2
+ import { spawnCommandInGhWorkspace } from './utils.js';
3
+
4
+ const GIT_LOG_FORMAT = `{"sha":"%H","author":"%aN<%aE>","date":"%aD","message":"%s"}`;
5
+
6
+ export const getBranchLastCommit = (branch: string): Commit => {
7
+ const lastCommit = JSON.parse(spawnCommandInGhWorkspace(`git log -1 --pretty=format:'${GIT_LOG_FORMAT}' ${branch}`)) as Commit;
8
+ return lastCommit;
9
+ };
10
+
11
+ export const getChangedFiles = (commits: Commit[]) => {
12
+ const changedFiles = commits.length === 1
13
+ ? spawnCommandInGhWorkspace(`git show --pretty='format:' --name-only ${commits[0].sha}`)
14
+ : spawnCommandInGhWorkspace(`git diff --name-only ${commits[0].sha}..${commits[commits.length - 1].sha}`);
15
+ return changedFiles.split('\n');
16
+ };
17
+
18
+ export const getCommits = ({
19
+ sourceBranch,
20
+ targetBranch,
21
+ baseCommit: baseCommitSha,
22
+ }: Config): Commit[] => {
23
+ const targetBranchCommit = getBranchLastCommit(targetBranch);
24
+ const sourceBranchCommit = getBranchLastCommit(sourceBranch);
25
+ let baseCommit: Commit | undefined;
26
+ if (baseCommitSha) {
27
+ try {
28
+ baseCommit = getCommitInfo(baseCommitSha);
29
+ } catch (err) {
30
+ console.warn(`Failed to get base commit "${baseCommitSha}", it will not be used: ${err}`);
31
+ }
32
+ }
33
+ // const baseCommit = baseCommitSha !== undefined ? : undefined;
34
+ // const gitOutputFormat = '{"sha":"%H","author":"%aN<%aE>","date":"%aD","message":"%f"}';
35
+ // console.log(`git log --pretty=format:"${gitOutputFormat}" ${targetBranch}..${sourceBranch}`);
36
+ // return [];
37
+ const base = getBaseCommit(sourceBranchCommit, targetBranchCommit);
38
+ const commitsStrings = spawnCommandInGhWorkspace(`git log --pretty=format:'${GIT_LOG_FORMAT}' ${base.sha}..${sourceBranchCommit.sha}`)
39
+ .split('\n');
40
+ const commits = JSON.parse(`[${commitsStrings.join(',')}]`) as Commit[];
41
+ commits.reverse();
42
+ const baseCommitIndex = baseCommit
43
+ ? commits.findIndex(({ sha }) => sha === baseCommit.sha)
44
+ : -1;
45
+ return commits.slice(baseCommitIndex + 1);
46
+ };
47
+
48
+ export const getBaseCommit = (target: Commit, source: Commit): Commit => {
49
+ const baseCommitSha = spawnCommandInGhWorkspace(`git merge-base ${target.sha} ${source.sha}`);
50
+ const baseCommit = getCommitInfo(baseCommitSha);
51
+ return baseCommit;
52
+ };
53
+
54
+ export const getCommitInfo = (commitSha: string): Commit => {
55
+ return JSON.parse(spawnCommandInGhWorkspace(`git log -1 --pretty=format:'${GIT_LOG_FORMAT}' ${commitSha}`));
56
+ };
package/bin/github.ts ADDED
@@ -0,0 +1,75 @@
1
+ import fs from 'fs/promises';
2
+ import { Commit, GitHubEventPush } from './types';
3
+ import { spawnCommandInGhWorkspace } from './utils.js';
4
+
5
+ export const getPushData = async (path: string) => {
6
+ const event = await loadGitHubEvent(path);
7
+ const branch = event.ref.replace('refs/heads/', '');
8
+ const {
9
+ commits: ghCommits,
10
+ repository: { ssh_url: repoUrl, name },
11
+ } = event;
12
+ const commits: Commit[] = ghCommits.map(({ author, message, id, timestamp }) => ({
13
+ author: author.username,
14
+ sha: id,
15
+ message,
16
+ date: timestamp,
17
+ }));
18
+ const changedFiles = getChangedFiles(event);
19
+ const changelog = getChangelogChanges(changedFiles, event);
20
+ return {
21
+ branch,
22
+ commits,
23
+ changedFiles,
24
+ repoUrl,
25
+ changelog,
26
+ repository: name,
27
+ };
28
+ };
29
+
30
+ export const loadGitHubEvent = async (path: string): Promise<GitHubEventPush> => {
31
+ const pushEvent = JSON.parse((await fs.readFile(path)).toString()) as GitHubEventPush;
32
+ return pushEvent;
33
+ };
34
+
35
+ const getChangedFiles = ({ after, before }: GitHubEventPush): string[] => {
36
+ const changedFiles = spawnCommandInGhWorkspace(`git diff --name-only ${after}..${before}`);
37
+ return changedFiles.split('\n');
38
+ };
39
+
40
+ export const getChangelogChanges = (
41
+ changedFiles: string[],
42
+ event: GitHubEventPush,
43
+ ): string | null => {
44
+ const changelogPath = 'CHANGELOG.md';
45
+ if (!changedFiles.includes(changelogPath)) {
46
+ return null;
47
+ }
48
+ const { after, before } = event;
49
+ const diff = spawnCommandInGhWorkspace('git', [
50
+ 'diff',
51
+ before,
52
+ after,
53
+ '--',
54
+ changelogPath,
55
+ ]);
56
+
57
+ const added: string[] = [];
58
+ let startedChangelog = false;
59
+ for (const line of diff.split('\n')) {
60
+ // We should already get only files we care about from getLastCommitDiffForFile but better to double check
61
+ if (line.startsWith('+++') && line.toLowerCase().includes(changelogPath.toLowerCase())) {
62
+ startedChangelog = true;
63
+ continue;
64
+ }
65
+ if (startedChangelog) {
66
+ if (line.startsWith('diff')) {
67
+ break;
68
+ }
69
+ if (line.startsWith('+')) {
70
+ added.push(line.slice(1).trim());
71
+ }
72
+ }
73
+ }
74
+ return added.join('\n').trim();
75
+ };
package/bin/main.ts ADDED
@@ -0,0 +1,145 @@
1
+ #!/usr/bin/env node
2
+
3
+ import process from 'process';
4
+ import yargs from 'yargs';
5
+ import { hideBin } from 'yargs/helpers';
6
+
7
+ import {
8
+ getRepoActors,
9
+ getChangedActors,
10
+ spawnCommandInGhWorkspace,
11
+ } from './utils.js';
12
+ import { runBuilds } from './build.js';
13
+ import { getChangedFiles, getCommits } from './git.js';
14
+ import { getPushData } from './github.js';
15
+ import { notifyToSlack } from './slack.js';
16
+ import { reportTestRestuls } from './test-report.js';
17
+
18
+ const buildOptions = (y: yargs.Argv) => {
19
+ return y
20
+ .option('target-branch', {
21
+ type: 'string',
22
+ demandOption: true,
23
+ })
24
+ .option('source-branch', {
25
+ type: 'string',
26
+ demandOption: true,
27
+ })
28
+ .option('base-commit', {
29
+ type: 'string',
30
+ });
31
+ };
32
+
33
+ await yargs()
34
+ .scriptName('public-actors-utils')
35
+ .option('dry-run', {
36
+ type: 'boolean',
37
+ default: false,
38
+ })
39
+ .command('get-commits', '', buildOptions, (args) => {
40
+ const commits = getCommits(args);
41
+ console.log(JSON.stringify(commits));
42
+ })
43
+ .command('get-latest-commit', '', buildOptions, (args) => {
44
+ const commits = getCommits(args);
45
+ if (commits.length > 0) {
46
+ console.log(JSON.stringify(commits[commits.length - 1]));
47
+ }
48
+ })
49
+ .command('get-changed-files', '', buildOptions, (args) => {
50
+ const commits = getCommits(args);
51
+ const changedFiles = getChangedFiles(commits);
52
+ console.log(JSON.stringify(changedFiles));
53
+ })
54
+ .command('get-actor-configs', '', (_) => _, async () => {
55
+ const actorConfigs = await getRepoActors();
56
+ console.log(JSON.stringify(actorConfigs));
57
+ })
58
+ .command('get-affected-actors', '', buildOptions, async (args) => {
59
+ const commits = getCommits(args);
60
+ const changedFiles = getChangedFiles(commits);
61
+ const actorConfigs = await getRepoActors();
62
+ const { actorsChanged } = getChangedActors({ filepathsChanged: changedFiles, actorConfigs, isLatest: false });
63
+ console.log(JSON.stringify(actorsChanged));
64
+ })
65
+ .command(
66
+ 'report-tests',
67
+ '',
68
+ (args) => args
69
+ .option('report-file', { type: 'string', demandOption: true })
70
+ .option('job-url', { type: 'string' })
71
+ .option('workflow-name', { type: 'string' })
72
+ .option('repository', { type: 'string' }),
73
+ async (args) => {
74
+ await reportTestRestuls(args);
75
+ })
76
+ .command(
77
+ 'build',
78
+ '',
79
+ (args) => buildOptions(args)
80
+ .option('dry-run', { type: 'boolean', default: false }),
81
+ async (args) => {
82
+ const commits = getCommits(args);
83
+ const changedFiles = getChangedFiles(commits);
84
+ const actorConfigs = await getRepoActors();
85
+ const { actorsChanged } = getChangedActors({
86
+ filepathsChanged: changedFiles,
87
+ actorConfigs,
88
+ });
89
+ // https://github.com/apify-store/google-maps#:actors/lukaskrivka_google-maps-with-contact-details
90
+ // git@github.com:apify-store/google-maps#:actors/lukaskrivka_google-maps-with-contact-details
91
+ const repoUrl = spawnCommandInGhWorkspace(`git remote get-url origin`)
92
+ .replace(/^https:\/\/github\.com\//, 'git@github.com:');
93
+
94
+ const builds = await runBuilds({
95
+ repoUrl,
96
+ actorConfigs: actorsChanged,
97
+ branch: args.sourceBranch.replace('origin/', ''),
98
+ dryRun: args.dryRun,
99
+ });
100
+ console.log(JSON.stringify(builds));
101
+ })
102
+ .command(
103
+ 'release',
104
+ '',
105
+ (args) => args
106
+ .option('push-event-path', { type: 'string', demandOption: true })
107
+ .option('dry-run', { type: 'boolean', default: false }),
108
+ async (args) => {
109
+ const {
110
+ branch,
111
+ changedFiles,
112
+ repoUrl,
113
+ commits,
114
+ changelog,
115
+ repository,
116
+ } = await getPushData(args.pushEventPath);
117
+ const isLatest = true;
118
+ const actorConfigs = await getRepoActors();
119
+ const { actorsChanged } = getChangedActors({
120
+ filepathsChanged: changedFiles,
121
+ actorConfigs,
122
+ isLatest,
123
+ });
124
+ const { dryRun } = args;
125
+ const builds = await runBuilds({
126
+ isLatest,
127
+ repoUrl,
128
+ actorConfigs: actorsChanged,
129
+ branch,
130
+ dryRun,
131
+ });
132
+ console.error(JSON.stringify(builds));
133
+
134
+ // TODO: build circle actors
135
+ await notifyToSlack({
136
+ changedFiles,
137
+ commits,
138
+ changelog,
139
+ repository,
140
+ dryRun,
141
+ });
142
+ })
143
+ .strictCommands()
144
+ .demandCommand(1, 'Command is required')
145
+ .parse(hideBin(process.argv));
package/bin/slack.ts ADDED
@@ -0,0 +1,72 @@
1
+ import { WebClient } from '@slack/web-api';
2
+ import { Commit } from './types';
3
+ import { getEnvVar } from './utils.js';
4
+
5
+ type NotifyToSlackOptions = {
6
+ repository: string
7
+ changedFiles: string[]
8
+ changelog: string | null
9
+ commits: Commit[]
10
+ dryRun: boolean
11
+ }
12
+
13
+ export const notifyToSlack = async ({
14
+ changedFiles,
15
+ commits,
16
+ changelog,
17
+ repository,
18
+ dryRun,
19
+ }: NotifyToSlackOptions) => {
20
+ const slack = new WebClient(getEnvVar('SLACK_TOKEN_RELEASES_BOT'));
21
+
22
+ if (!changelog) {
23
+ console.warn('No new changelog entries found, haven\'t you forgotten to update it?');
24
+ }
25
+
26
+ let shortMessage = `${repository}: new release:\n`;
27
+
28
+ // This one is just for broader public that only cares about public facing changes
29
+ if (changelog) {
30
+ console.error(`New changelog entries: ${changelog}`);
31
+ shortMessage += `Changelog:\n\n${changelog}\n`;
32
+ const channel = '#delivery-public-actors';
33
+ console.error(`Sending slack message to channel: ${channel}. Message: ${shortMessage}`);
34
+ if (!dryRun) {
35
+ await slack.chat.postMessage({
36
+ channel,
37
+ text: shortMessage,
38
+ });
39
+ }
40
+ }
41
+
42
+ const commitsMessage = `${commits.map(({ author, message }, index) => ` ${index + 1} Commit message: ${message}\n Author: ${author}.`).join('\n')}`;
43
+ const changedFilesMessage = `Files changed: ${changedFiles.join(', ')}`;
44
+ const longMessage = `${shortMessage}\nCommits:\n\n${commitsMessage}\n\n${changedFilesMessage}`;
45
+
46
+ // This one is for devs and project managers that need to know more details
47
+ const notifChannel = `#notif-${repository.toLowerCase()}`;
48
+ console.error(`Sending slack message to channel: ${notifChannel}. Message: ${longMessage}`);
49
+ if (!dryRun) {
50
+ await slack.chat.postMessage({
51
+ text: longMessage,
52
+ channel: notifChannel,
53
+ });
54
+ }
55
+ };
56
+
57
+ export const sendSlackMessage = async (
58
+ channel: string,
59
+ text: string,
60
+ blocks: string[],
61
+ token: string,
62
+ ) => {
63
+ const slack = new WebClient(token);
64
+ const { ts } = await slack.chat.postMessage({ text, channel });
65
+ if (blocks.length > 0 && ts) {
66
+ await slack.chat.postMessage({
67
+ channel,
68
+ thread_ts: ts,
69
+ blocks: blocks.map((t) => ({ text: { type: 'mrkdwn', text: t }, type: 'section' })),
70
+ });
71
+ }
72
+ };