apify-test-tools 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -0
- package/bin/build.ts +287 -0
- package/bin/consts.ts +6 -0
- package/bin/git.ts +56 -0
- package/bin/github.ts +75 -0
- package/bin/main.ts +145 -0
- package/bin/slack.ts +72 -0
- package/bin/test-report.ts +146 -0
- package/bin/types.ts +102 -0
- package/bin/utils.ts +204 -0
- package/dist/bin/build.d.ts +18 -0
- package/dist/bin/build.d.ts.map +1 -0
- package/dist/bin/build.js +222 -0
- package/dist/bin/build.js.map +1 -0
- package/dist/bin/consts.d.ts +4 -0
- package/dist/bin/consts.d.ts.map +1 -0
- package/dist/bin/consts.js +6 -0
- package/dist/bin/consts.js.map +1 -0
- package/dist/bin/git.d.ts +7 -0
- package/dist/bin/git.d.ts.map +1 -0
- package/dist/bin/git.js +47 -0
- package/dist/bin/git.js.map +1 -0
- package/dist/bin/github.d.ts +12 -0
- package/dist/bin/github.d.ts.map +1 -0
- package/dist/bin/github.js +64 -0
- package/dist/bin/github.js.map +1 -0
- package/dist/bin/main.d.ts +3 -0
- package/dist/bin/main.d.ts.map +1 -0
- package/dist/bin/main.js +117 -0
- package/dist/bin/main.js.map +1 -0
- package/dist/bin/slack.d.ts +12 -0
- package/dist/bin/slack.d.ts.map +1 -0
- package/dist/bin/slack.js +46 -0
- package/dist/bin/slack.js.map +1 -0
- package/dist/bin/test-report.d.ts +10 -0
- package/dist/bin/test-report.d.ts.map +1 -0
- package/dist/bin/test-report.js +74 -0
- package/dist/bin/test-report.js.map +1 -0
- package/dist/bin/types.d.ts +88 -0
- package/dist/bin/types.d.ts.map +1 -0
- package/dist/bin/types.js +2 -0
- package/dist/bin/types.js.map +1 -0
- package/dist/bin/utils.d.ts +26 -0
- package/dist/bin/utils.d.ts.map +1 -0
- package/dist/bin/utils.js +164 -0
- package/dist/bin/utils.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -0
- package/dist/lib/consts.d.ts +3 -0
- package/dist/lib/consts.d.ts.map +1 -0
- package/dist/lib/consts.js +14 -0
- package/dist/lib/consts.js.map +1 -0
- package/dist/lib/extend-expect.d.ts +3 -0
- package/dist/lib/extend-expect.d.ts.map +1 -0
- package/dist/lib/extend-expect.js +228 -0
- package/dist/lib/extend-expect.js.map +1 -0
- package/dist/lib/lib.d.ts +16 -0
- package/dist/lib/lib.d.ts.map +1 -0
- package/dist/lib/lib.js +91 -0
- package/dist/lib/lib.js.map +1 -0
- package/dist/lib/run-test-result.d.ts +20 -0
- package/dist/lib/run-test-result.d.ts.map +1 -0
- package/dist/lib/run-test-result.js +62 -0
- package/dist/lib/run-test-result.js.map +1 -0
- package/dist/lib/types.d.ts +107 -0
- package/dist/lib/types.d.ts.map +1 -0
- package/dist/lib/types.js +2 -0
- package/dist/lib/types.js.map +1 -0
- package/dist/lib/utils.d.ts +7 -0
- package/dist/lib/utils.d.ts.map +1 -0
- package/dist/lib/utils.js +30 -0
- package/dist/lib/utils.js.map +1 -0
- package/dist/test/unit/custom-matchers.test.d.ts +2 -0
- package/dist/test/unit/custom-matchers.test.d.ts.map +1 -0
- package/dist/test/unit/custom-matchers.test.js +53 -0
- package/dist/test/unit/custom-matchers.test.js.map +1 -0
- package/dist/test/unit/should-built-and-test.test.d.ts +2 -0
- package/dist/test/unit/should-built-and-test.test.d.ts.map +1 -0
- package/dist/test/unit/should-built-and-test.test.js +136 -0
- package/dist/test/unit/should-built-and-test.test.js.map +1 -0
- package/dist/test/unit/test-actor.test.d.ts +2 -0
- package/dist/test/unit/test-actor.test.d.ts.map +1 -0
- package/dist/test/unit/test-actor.test.js +17 -0
- package/dist/test/unit/test-actor.test.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -0
- package/example-github-events/pull-request.json +523 -0
- package/example-github-events/push.json +216 -0
- package/index.ts +2 -0
- package/lib/consts.ts +15 -0
- package/lib/extend-expect.ts +268 -0
- package/lib/lib.ts +134 -0
- package/lib/run-test-result.ts +70 -0
- package/lib/types.ts +131 -0
- package/lib/utils.ts +49 -0
- package/package.json +41 -0
- package/scripts/hook-job-started.sh +11 -0
- package/test/tsconfig.json +15 -0
- package/test/unit/custom-matchers.test.ts +61 -0
- package/test/unit/should-built-and-test.test.ts +166 -0
- package/test/unit/test-actor.test.ts +17 -0
- package/tsconfig.json +20 -0
package/README.md
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
This Actor has essentially 2 separate parts
|
|
2
|
+
|
|
3
|
+
1. The Actor loads GitHub actions binary on build. When it is started, it will run `runner.ts` which spawns the GitHub actions server as a separate process. Then it only listens for migrations and does nothing else.
|
|
4
|
+
2. The GitHub actions server binary (which we don't control) is then triggered by GitHub action and that subsequently calls `main.ts` (via calling it from `.github/workflows/apify.yaml` and `gha:start:build` command). This runs rest of the code in the files.
|
|
5
|
+
|
|
6
|
+
## Run locally to test
|
|
7
|
+
1. Choose if you want to emulate PR or push to master workflow.
|
|
8
|
+
2. Copy appropriate example from `./example-github-events` to `INPUT.json`
|
|
9
|
+
3. To mimic actual PR or push you care about, you should update the input
|
|
10
|
+
- PR: Update `pull_request.head.ref` (source branch of the PR), `repository.full_name`, `pull_request.base.sha`, `pull_request.head.sha`
|
|
11
|
+
- push: Update `ref`, `repository.full_name`
|
|
12
|
+
4. Start the Actor with `TESTER_APIFY_TOKEN=<TOKEN> APIFY_TOKEN_${username}=<TOKEN> apify run -p` (possibly other apify users as well)
|
|
13
|
+
|
|
14
|
+
## Testing real Actor
|
|
15
|
+
Do anything to https://github.com/apify-projects/store-testing-repo-for-github
|
|
16
|
+
|
|
17
|
+
## Runner on Apify
|
|
18
|
+
We run GitHub Actions in Apify Actor that loads the GitHub Actions server binary. This server is then triggered by GitHub Actions and runs the rest of the code. There is one production forever running instance of this Actor (https://console.apify.com/admin/users/xRGg9iAfJSymqartk/actors/ANeZceuSfUd7i64qs/runs/hcTVcLB94TsOvPQFJ#log). If you need to update the build, just abort and resurrect the run with rebuilt `latest` version.
|
|
19
|
+
|
|
20
|
+
For testing on platform, build the beta version (you can change the branch) and then resurrect the test task of the runner. Don't forget to abort the run after the test is done.
|
|
21
|
+
|
|
22
|
+
### Multiple production runners
|
|
23
|
+
It is possible to run multiple production runners at the same time. We might need this to scale up in the future since we are currently limited by only running one GitHub Action at a time.
|
package/bin/build.ts
ADDED
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
import { ACTOR_SOURCE_TYPES } from '@apify/consts';
|
|
2
|
+
import { ApifyClient, Build } from 'apify-client';
|
|
3
|
+
import type { ActorConfig, BuildData } from './types.js';
|
|
4
|
+
import { getEnvVar } from './utils.js';
|
|
5
|
+
|
|
6
|
+
type BuildPrActorOptions = {
|
|
7
|
+
buildTag?: string
|
|
8
|
+
versionNumber: string
|
|
9
|
+
gitRepoUrl: string
|
|
10
|
+
actorName: string
|
|
11
|
+
}
|
|
12
|
+
class ApifyBuilder {
|
|
13
|
+
// eslint-disable-next-line no-empty-function
|
|
14
|
+
private constructor(private readonly apifyClient: ApifyClient, private readonly actorName: string) { }
|
|
15
|
+
|
|
16
|
+
// Usually 'latest' but not necessarily (can be e.g. 'version-0')
|
|
17
|
+
getDefaultVersionAndTag = async (): Promise<{ defaultBuildNumber: string, defaultVersionNumber: string, defaultBuildTag: string }> => {
|
|
18
|
+
const actorClient = this.apifyClient.actor(this.actorName);
|
|
19
|
+
const actorInfo = await actorClient.get();
|
|
20
|
+
|
|
21
|
+
if (!actorInfo) {
|
|
22
|
+
throw new Error(`[${this.actorName}] not found. It is not published or we are missing token to access it privately or its name is misspelled`);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const defaultBuildTag = actorInfo.defaultRunOptions.build;
|
|
26
|
+
console.error(`Default build tag for ${this.actorName} is ${defaultBuildTag}`);
|
|
27
|
+
|
|
28
|
+
// We could technically allow this but in most cases this is accidentally set wrongly and there is a workaround
|
|
29
|
+
if (defaultBuildTag.match(/\d+\.\d+\.\d+/)) {
|
|
30
|
+
throw new Error(`[${this.actorName}] Default build is a build number, not a tag. While this could work, `
|
|
31
|
+
+ `we want to have a default as tag so this is often an accidental misconfiguration from the dev`);
|
|
32
|
+
}
|
|
33
|
+
// I reported that buildNumber should probably not be optional
|
|
34
|
+
const defaultBuildNumber = actorInfo.taggedBuilds![defaultBuildTag].buildNumber!;
|
|
35
|
+
const defaultVersionNumber = defaultBuildNumber.match(/(\d+\.\d+)\.\d+/)![1];
|
|
36
|
+
console.error(`Default version for ${this.actorName} is ${defaultVersionNumber}`);
|
|
37
|
+
|
|
38
|
+
return { defaultBuildNumber, defaultVersionNumber, defaultBuildTag };
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
startActorBuild = async ({
|
|
42
|
+
buildTag,
|
|
43
|
+
versionNumber,
|
|
44
|
+
gitRepoUrl,
|
|
45
|
+
}: BuildPrActorOptions): Promise<BuildData> => {
|
|
46
|
+
const actorClient = this.apifyClient.actor(this.actorName);
|
|
47
|
+
const actorInfo = await actorClient.get();
|
|
48
|
+
if (!actorInfo) {
|
|
49
|
+
throw new Error(`No actor named '${this.actorName}' was found on the platform. If this`
|
|
50
|
+
+ ' is unexpected, make sure the actor you are targeting is spelled the'
|
|
51
|
+
+ ' same as the folder in the repository.');
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// NOTE: I couldn't find this type, so I had to extract it :(
|
|
55
|
+
type ActorVersion = Parameters<ReturnType<typeof actorClient.version>['update']>[0];
|
|
56
|
+
const actorVersion: ActorVersion = {
|
|
57
|
+
buildTag,
|
|
58
|
+
versionNumber,
|
|
59
|
+
gitRepoUrl,
|
|
60
|
+
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
61
|
+
// @ts-ignore: coudn't find this type either :(
|
|
62
|
+
sourceType: ACTOR_SOURCE_TYPES.GIT_REPO,
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
// Prepare version
|
|
66
|
+
const versionExists = !actorInfo.versions.find((version) => version.versionNumber === versionNumber);
|
|
67
|
+
if (versionExists) {
|
|
68
|
+
// create new version
|
|
69
|
+
await actorClient.versions().create(actorVersion);
|
|
70
|
+
} else {
|
|
71
|
+
const version = actorClient.version(versionNumber);
|
|
72
|
+
await version.update(actorVersion);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
console.error(`[BUILD][${this.actorName}]: Will be built with version ${versionNumber}`);
|
|
76
|
+
console.error(`[BUILD][${this.actorName}]: ${JSON.stringify(actorVersion)}`);
|
|
77
|
+
|
|
78
|
+
// We also get back actId so the testing actor can both match by actor ID and name
|
|
79
|
+
const { id, actId, buildNumber } = await actorClient.build(versionNumber);
|
|
80
|
+
|
|
81
|
+
console.error(`[BUILD][${this.actorName}]: Build ${id} (${versionNumber}) has started`);
|
|
82
|
+
return { buildId: id, actorId: actId, buildNumber, actorName: this.actorName };
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
waitForBuildToFinish = async (buildId: string, actorName: string): Promise<Build> => {
|
|
86
|
+
console.error(`[BUILD][${actorName}]: Waiting for build ${buildId} to finish`);
|
|
87
|
+
const build = await this.apifyClient.build(buildId).waitForFinish();
|
|
88
|
+
const versionNumber = build.buildNumber;
|
|
89
|
+
if (build.status === 'FAILED' || build.status === 'TIMED-OUT') {
|
|
90
|
+
const message = `[BUILD][${actorName}]: Build ${buildId} (${versionNumber}) failed. `
|
|
91
|
+
+ `Not continuing with other builds and tests.`;
|
|
92
|
+
console.warn(`D€LIV€RY_$L&CK: Webhook-to-build: ${message}`);
|
|
93
|
+
throw new Error(message);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
console.error(`[BUILD][${actorName}]: Build ${build.id} (${versionNumber}) finished successfully.`);
|
|
97
|
+
return build;
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Create ApifyBuilder with actor owner's token
|
|
102
|
+
*/
|
|
103
|
+
static fromActorName = (actorName: string): ApifyBuilder => {
|
|
104
|
+
const username = actorName.split('/')[0];
|
|
105
|
+
// GitHib secrets only allow word characters (alphanum + underscore)
|
|
106
|
+
const usernameInGitHubSecretsFormat = username.replaceAll(/\W/g, '_').toUpperCase();
|
|
107
|
+
const usernameEnvVar = `APIFY_TOKEN_${usernameInGitHubSecretsFormat}`;
|
|
108
|
+
const token = process.env[usernameEnvVar];
|
|
109
|
+
if (!token) {
|
|
110
|
+
throw new Error(`Cannot find Apify API token for username: ${username}. `
|
|
111
|
+
+ `Have you set secret env var to this GitHub repo with key: ${usernameEnvVar}?`);
|
|
112
|
+
}
|
|
113
|
+
const apifyClient = new ApifyClient({ token });
|
|
114
|
+
const builder = new ApifyBuilder(apifyClient, actorName);
|
|
115
|
+
return builder;
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Deletes build of all versions. Apify API doesn't allow to delete default build and we explicitly skip it
|
|
120
|
+
* We delete devel builds faster because we used the for every PR until recently so just to get rid of them faster
|
|
121
|
+
*/
|
|
122
|
+
async deleteOldBuilds(): Promise<void> {
|
|
123
|
+
// Even though we don't version our current Actors, if we ever such Actors to GitHub CI, we would accidentally delete old supported versions
|
|
124
|
+
// This hardcoded solution is not ideal, but it should prevent most imaginable cases
|
|
125
|
+
// All currently popular versioned Actors use `version-${number}` format
|
|
126
|
+
const PROTECTED_TAGS_PREFIX = ['latest', 'v-', 'version', 'v0', 'v1', 'v2', 'v3', 'v4', 'v5', 'v6', 'v7', 'v8', 'v9'];
|
|
127
|
+
|
|
128
|
+
// We don't want to be too short because we might to debug something
|
|
129
|
+
// but also not too long because it increases the risk of users using outdated versions
|
|
130
|
+
const DEFAULT_DAYS_BACK_PROD_VERSIONS = 30;
|
|
131
|
+
const DEFAULT_DAYS_BACK_DEVEL = 7;
|
|
132
|
+
|
|
133
|
+
const actorInfo = (await this.apifyClient.actor(this.actorName).get())!;
|
|
134
|
+
|
|
135
|
+
// 'devel' used to be hardcoded for testing version 0.99, once we get rid of this tag everywhere, we can remove this code
|
|
136
|
+
const taggedDevelBuildNumber: string | undefined = actorInfo.taggedBuilds!.devel?.buildNumber;
|
|
137
|
+
|
|
138
|
+
const allTags = Object.keys(actorInfo.taggedBuilds ?? {});
|
|
139
|
+
const protectedTags = allTags.filter((tag) => PROTECTED_TAGS_PREFIX.some((prefix) => tag.startsWith(prefix)));
|
|
140
|
+
const protectedBuildNumbers = protectedTags.map((tag) => ({ buildNumber: actorInfo.taggedBuilds![tag]!.buildNumber, tag }));
|
|
141
|
+
|
|
142
|
+
const { items } = (await this.apifyClient.actor(this.actorName).builds().list());
|
|
143
|
+
|
|
144
|
+
// Deleting default build throws an error, so we skip it
|
|
145
|
+
const { defaultBuildNumber, defaultBuildTag } = await ApifyBuilder.fromActorName(this.actorName).getDefaultVersionAndTag();
|
|
146
|
+
|
|
147
|
+
const daysAgoUnixProd = Date.now() - DEFAULT_DAYS_BACK_PROD_VERSIONS * 24 * 60 * 60 * 1000;
|
|
148
|
+
const daysAgoUnixDevel = Date.now() - DEFAULT_DAYS_BACK_DEVEL * 24 * 60 * 60 * 1000;
|
|
149
|
+
|
|
150
|
+
// Fixing API client missing buildNumber field
|
|
151
|
+
type CorrectBuildColletionItem = typeof items[0] & { buildNumber: string };
|
|
152
|
+
const buildsToDelete = (items as CorrectBuildColletionItem[]).filter((build) => {
|
|
153
|
+
if (build.buildNumber === defaultBuildNumber) {
|
|
154
|
+
console.error(`[DELETE OLD BUILDS][${this.actorName}]: Skipping default build ${defaultBuildNumber} (${defaultBuildTag}). `
|
|
155
|
+
+ `We never delete default builds`);
|
|
156
|
+
return false;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const protectedTagFound = protectedBuildNumbers.find((protectedBuildNumber) => protectedBuildNumber.buildNumber === build.buildNumber);
|
|
160
|
+
if (protectedTagFound) {
|
|
161
|
+
console.error(`[DELETE OLD BUILDS][${this.actorName}]: Skipping protected build ${protectedTagFound.buildNumber} (${protectedTagFound.tag}).`);
|
|
162
|
+
return false;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
if (taggedDevelBuildNumber && build.buildNumber === taggedDevelBuildNumber) {
|
|
166
|
+
const shouldDeleteDevelBuild = build.startedAt.getTime() < daysAgoUnixDevel;
|
|
167
|
+
if (shouldDeleteDevelBuild) {
|
|
168
|
+
console.error(`[DELETE OLD BUILDS][${this.actorName}]: Removing olf devel build ${taggedDevelBuildNumber}.`);
|
|
169
|
+
}
|
|
170
|
+
return shouldDeleteDevelBuild;
|
|
171
|
+
}
|
|
172
|
+
return build.startedAt.getTime() < daysAgoUnixProd;
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
console.error(`[DELETE OLD BUILDS][${this.actorName}]: Deleting ${buildsToDelete.length} old builds that are non-default and `
|
|
176
|
+
+ `older than 30 days from total ${items.length}`);
|
|
177
|
+
for (const build of buildsToDelete) {
|
|
178
|
+
await this.apifyClient.build(build.id).delete();
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
type RunBuildsOptions = {
|
|
184
|
+
actorConfigs: ActorConfig[]
|
|
185
|
+
isLatest?: boolean
|
|
186
|
+
repoUrl: string
|
|
187
|
+
branch: string
|
|
188
|
+
dryRun: boolean
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
export const runBuilds = async ({
|
|
192
|
+
repoUrl,
|
|
193
|
+
branch,
|
|
194
|
+
actorConfigs,
|
|
195
|
+
isLatest = false,
|
|
196
|
+
dryRun,
|
|
197
|
+
}: RunBuildsOptions) => {
|
|
198
|
+
const buildConfigs: BuildPrActorOptions[] = [];
|
|
199
|
+
|
|
200
|
+
const circleActors = isLatest ? await findCircleApifyManaged(actorConfigs) : [];
|
|
201
|
+
|
|
202
|
+
for (const { actorName, folder } of actorConfigs.concat(circleActors)) {
|
|
203
|
+
let versionNumber: string;
|
|
204
|
+
let buildTag: string | undefined;
|
|
205
|
+
|
|
206
|
+
if (isLatest) {
|
|
207
|
+
const { defaultVersionNumber, defaultBuildTag } = await ApifyBuilder.fromActorName(actorName).getDefaultVersionAndTag();
|
|
208
|
+
versionNumber = defaultVersionNumber;
|
|
209
|
+
buildTag = defaultBuildTag;
|
|
210
|
+
} else {
|
|
211
|
+
versionNumber = '0.99';
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
// Depending on if these are miniactors or standaloneActors
|
|
215
|
+
let gitRepoUrl = `${repoUrl}#${branch}`;
|
|
216
|
+
if (folder) {
|
|
217
|
+
gitRepoUrl = `${gitRepoUrl}:${folder}`;
|
|
218
|
+
}
|
|
219
|
+
buildConfigs.push({ actorName, gitRepoUrl, versionNumber, buildTag });
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
if (dryRun) {
|
|
223
|
+
return buildConfigs;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
const startedBuilds = await Promise.all(buildConfigs.map(async (buildConfig) => {
|
|
227
|
+
const builder = ApifyBuilder.fromActorName(buildConfig.actorName);
|
|
228
|
+
const buildData = await builder.startActorBuild(buildConfig);
|
|
229
|
+
return buildData;
|
|
230
|
+
}));
|
|
231
|
+
|
|
232
|
+
await Promise.all(startedBuilds.map(async (buildData) => {
|
|
233
|
+
const builder = ApifyBuilder.fromActorName(buildData.actorName);
|
|
234
|
+
await builder.waitForBuildToFinish(buildData.buildId, buildData.actorName);
|
|
235
|
+
}));
|
|
236
|
+
|
|
237
|
+
return startedBuilds;
|
|
238
|
+
};
|
|
239
|
+
|
|
240
|
+
export const deleteOldBuilds = async (actorConfigs: ActorConfig[]) => {
|
|
241
|
+
for (const { actorName } of actorConfigs) {
|
|
242
|
+
await ApifyBuilder.fromActorName(actorName).deleteOldBuilds();
|
|
243
|
+
}
|
|
244
|
+
};
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* We will read all Actors in the circ_le account and build those that match by name pattern
|
|
248
|
+
* There are many ways to approach this, a more robust one would be to have a map of Actors
|
|
249
|
+
* which would allow to have more than one special user per Actor
|
|
250
|
+
* But since that use-case might never be needed, I went with the simplest solution that doesn't require maintaining the map
|
|
251
|
+
* NOTE: One issue is that if any Actor is renamed, we will not match it in the circ_le account nor throw any error
|
|
252
|
+
*/
|
|
253
|
+
const findCircleApifyManaged = async (actorConfigs: ActorConfig[]): Promise<ActorConfig[]> => {
|
|
254
|
+
// This token is hardcoded in the runner Actor, locally you have to inject it
|
|
255
|
+
const client = new ApifyClient({ token: getEnvVar('APIFY_TOKEN_CIRC_LE') });
|
|
256
|
+
|
|
257
|
+
const { items: circleActors } = await client.actors().list();
|
|
258
|
+
|
|
259
|
+
const actorsToBuild = circleActors.map<ActorConfig | undefined>((circleActor) => {
|
|
260
|
+
// They prefix all with apify-managed---, I communicated with Jacques to keep doing that
|
|
261
|
+
let actorConfigFound = actorConfigs.find((actorConfig) => circleActor.name.replace('apify-managed---', '') === actorConfig.actorName.split('/')[1]);
|
|
262
|
+
|
|
263
|
+
// Hack for bad naming of circ_le/apify-managed-google-search, we don't want to rename now to break customers
|
|
264
|
+
if (!actorConfigFound && circleActor.name === 'apify-managed-google-search') {
|
|
265
|
+
actorConfigFound = actorConfigs.find((actorConfig) => actorConfig.actorName.split('/')[1] === 'google-search-scraper');
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
if (actorConfigFound) {
|
|
269
|
+
return {
|
|
270
|
+
// We point the circle Actor to the repo folder
|
|
271
|
+
actorName: `${circleActor.username}/${circleActor.name}`,
|
|
272
|
+
folder: actorConfigFound.folder,
|
|
273
|
+
isStandalone: actorConfigFound.isStandalone,
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
return undefined;
|
|
277
|
+
}).filter((config) => config !== undefined);
|
|
278
|
+
|
|
279
|
+
console.error(`Found ${actorsToBuild.length} circ_le actors that match Actors we built out of total ${circleActors.length} circ_le actors`);
|
|
280
|
+
console.error(`All circ_le actors: ${circleActors.map((actor) => actor.name).join(', ')}`);
|
|
281
|
+
console.error(`circ_le Actors to build: ${actorsToBuild.map((actor) => actor.actorName).join(', ')}`);
|
|
282
|
+
|
|
283
|
+
if (actorsToBuild.length === 0) {
|
|
284
|
+
console.error('No circ_le actors to build');
|
|
285
|
+
}
|
|
286
|
+
return actorsToBuild;
|
|
287
|
+
};
|
package/bin/consts.ts
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
// Technically, upgrades in shared or packages might not need impact every actor
|
|
2
|
+
// but we cannot really know that
|
|
3
|
+
export const DEFAULT_BUILD_ALL_FOLDERS = ['code', 'shared', 'packages'];
|
|
4
|
+
|
|
5
|
+
export const PERSISTED_GH_JOBS_KVS_ID = 'XIBdO8EJePdD0KiAI';
|
|
6
|
+
export const MINIACTORS_LIST_STORE_ID = 'ftXklt2f2mN30Oc3y';
|
package/bin/git.ts
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { Commit, Config } from './types';
|
|
2
|
+
import { spawnCommandInGhWorkspace } from './utils.js';
|
|
3
|
+
|
|
4
|
+
const GIT_LOG_FORMAT = `{"sha":"%H","author":"%aN<%aE>","date":"%aD","message":"%s"}`;
|
|
5
|
+
|
|
6
|
+
export const getBranchLastCommit = (branch: string): Commit => {
|
|
7
|
+
const lastCommit = JSON.parse(spawnCommandInGhWorkspace(`git log -1 --pretty=format:'${GIT_LOG_FORMAT}' ${branch}`)) as Commit;
|
|
8
|
+
return lastCommit;
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
export const getChangedFiles = (commits: Commit[]) => {
|
|
12
|
+
const changedFiles = commits.length === 1
|
|
13
|
+
? spawnCommandInGhWorkspace(`git show --pretty='format:' --name-only ${commits[0].sha}`)
|
|
14
|
+
: spawnCommandInGhWorkspace(`git diff --name-only ${commits[0].sha}..${commits[commits.length - 1].sha}`);
|
|
15
|
+
return changedFiles.split('\n');
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
export const getCommits = ({
|
|
19
|
+
sourceBranch,
|
|
20
|
+
targetBranch,
|
|
21
|
+
baseCommit: baseCommitSha,
|
|
22
|
+
}: Config): Commit[] => {
|
|
23
|
+
const targetBranchCommit = getBranchLastCommit(targetBranch);
|
|
24
|
+
const sourceBranchCommit = getBranchLastCommit(sourceBranch);
|
|
25
|
+
let baseCommit: Commit | undefined;
|
|
26
|
+
if (baseCommitSha) {
|
|
27
|
+
try {
|
|
28
|
+
baseCommit = getCommitInfo(baseCommitSha);
|
|
29
|
+
} catch (err) {
|
|
30
|
+
console.warn(`Failed to get base commit "${baseCommitSha}", it will not be used: ${err}`);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
// const baseCommit = baseCommitSha !== undefined ? : undefined;
|
|
34
|
+
// const gitOutputFormat = '{"sha":"%H","author":"%aN<%aE>","date":"%aD","message":"%f"}';
|
|
35
|
+
// console.log(`git log --pretty=format:"${gitOutputFormat}" ${targetBranch}..${sourceBranch}`);
|
|
36
|
+
// return [];
|
|
37
|
+
const base = getBaseCommit(sourceBranchCommit, targetBranchCommit);
|
|
38
|
+
const commitsStrings = spawnCommandInGhWorkspace(`git log --pretty=format:'${GIT_LOG_FORMAT}' ${base.sha}..${sourceBranchCommit.sha}`)
|
|
39
|
+
.split('\n');
|
|
40
|
+
const commits = JSON.parse(`[${commitsStrings.join(',')}]`) as Commit[];
|
|
41
|
+
commits.reverse();
|
|
42
|
+
const baseCommitIndex = baseCommit
|
|
43
|
+
? commits.findIndex(({ sha }) => sha === baseCommit.sha)
|
|
44
|
+
: -1;
|
|
45
|
+
return commits.slice(baseCommitIndex + 1);
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
export const getBaseCommit = (target: Commit, source: Commit): Commit => {
|
|
49
|
+
const baseCommitSha = spawnCommandInGhWorkspace(`git merge-base ${target.sha} ${source.sha}`);
|
|
50
|
+
const baseCommit = getCommitInfo(baseCommitSha);
|
|
51
|
+
return baseCommit;
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
export const getCommitInfo = (commitSha: string): Commit => {
|
|
55
|
+
return JSON.parse(spawnCommandInGhWorkspace(`git log -1 --pretty=format:'${GIT_LOG_FORMAT}' ${commitSha}`));
|
|
56
|
+
};
|
package/bin/github.ts
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import fs from 'fs/promises';
|
|
2
|
+
import { Commit, GitHubEventPush } from './types';
|
|
3
|
+
import { spawnCommandInGhWorkspace } from './utils.js';
|
|
4
|
+
|
|
5
|
+
export const getPushData = async (path: string) => {
|
|
6
|
+
const event = await loadGitHubEvent(path);
|
|
7
|
+
const branch = event.ref.replace('refs/heads/', '');
|
|
8
|
+
const {
|
|
9
|
+
commits: ghCommits,
|
|
10
|
+
repository: { ssh_url: repoUrl, name },
|
|
11
|
+
} = event;
|
|
12
|
+
const commits: Commit[] = ghCommits.map(({ author, message, id, timestamp }) => ({
|
|
13
|
+
author: author.username,
|
|
14
|
+
sha: id,
|
|
15
|
+
message,
|
|
16
|
+
date: timestamp,
|
|
17
|
+
}));
|
|
18
|
+
const changedFiles = getChangedFiles(event);
|
|
19
|
+
const changelog = getChangelogChanges(changedFiles, event);
|
|
20
|
+
return {
|
|
21
|
+
branch,
|
|
22
|
+
commits,
|
|
23
|
+
changedFiles,
|
|
24
|
+
repoUrl,
|
|
25
|
+
changelog,
|
|
26
|
+
repository: name,
|
|
27
|
+
};
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
export const loadGitHubEvent = async (path: string): Promise<GitHubEventPush> => {
|
|
31
|
+
const pushEvent = JSON.parse((await fs.readFile(path)).toString()) as GitHubEventPush;
|
|
32
|
+
return pushEvent;
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
const getChangedFiles = ({ after, before }: GitHubEventPush): string[] => {
|
|
36
|
+
const changedFiles = spawnCommandInGhWorkspace(`git diff --name-only ${after}..${before}`);
|
|
37
|
+
return changedFiles.split('\n');
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
export const getChangelogChanges = (
|
|
41
|
+
changedFiles: string[],
|
|
42
|
+
event: GitHubEventPush,
|
|
43
|
+
): string | null => {
|
|
44
|
+
const changelogPath = 'CHANGELOG.md';
|
|
45
|
+
if (!changedFiles.includes(changelogPath)) {
|
|
46
|
+
return null;
|
|
47
|
+
}
|
|
48
|
+
const { after, before } = event;
|
|
49
|
+
const diff = spawnCommandInGhWorkspace('git', [
|
|
50
|
+
'diff',
|
|
51
|
+
before,
|
|
52
|
+
after,
|
|
53
|
+
'--',
|
|
54
|
+
changelogPath,
|
|
55
|
+
]);
|
|
56
|
+
|
|
57
|
+
const added: string[] = [];
|
|
58
|
+
let startedChangelog = false;
|
|
59
|
+
for (const line of diff.split('\n')) {
|
|
60
|
+
// We should already get only files we care about from getLastCommitDiffForFile but better to double check
|
|
61
|
+
if (line.startsWith('+++') && line.toLowerCase().includes(changelogPath.toLowerCase())) {
|
|
62
|
+
startedChangelog = true;
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
if (startedChangelog) {
|
|
66
|
+
if (line.startsWith('diff')) {
|
|
67
|
+
break;
|
|
68
|
+
}
|
|
69
|
+
if (line.startsWith('+')) {
|
|
70
|
+
added.push(line.slice(1).trim());
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return added.join('\n').trim();
|
|
75
|
+
};
|
package/bin/main.ts
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import process from 'process';
|
|
4
|
+
import yargs from 'yargs';
|
|
5
|
+
import { hideBin } from 'yargs/helpers';
|
|
6
|
+
|
|
7
|
+
import {
|
|
8
|
+
getRepoActors,
|
|
9
|
+
getChangedActors,
|
|
10
|
+
spawnCommandInGhWorkspace,
|
|
11
|
+
} from './utils.js';
|
|
12
|
+
import { runBuilds } from './build.js';
|
|
13
|
+
import { getChangedFiles, getCommits } from './git.js';
|
|
14
|
+
import { getPushData } from './github.js';
|
|
15
|
+
import { notifyToSlack } from './slack.js';
|
|
16
|
+
import { reportTestRestuls } from './test-report.js';
|
|
17
|
+
|
|
18
|
+
const buildOptions = (y: yargs.Argv) => {
|
|
19
|
+
return y
|
|
20
|
+
.option('target-branch', {
|
|
21
|
+
type: 'string',
|
|
22
|
+
demandOption: true,
|
|
23
|
+
})
|
|
24
|
+
.option('source-branch', {
|
|
25
|
+
type: 'string',
|
|
26
|
+
demandOption: true,
|
|
27
|
+
})
|
|
28
|
+
.option('base-commit', {
|
|
29
|
+
type: 'string',
|
|
30
|
+
});
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
await yargs()
|
|
34
|
+
.scriptName('public-actors-utils')
|
|
35
|
+
.option('dry-run', {
|
|
36
|
+
type: 'boolean',
|
|
37
|
+
default: false,
|
|
38
|
+
})
|
|
39
|
+
.command('get-commits', '', buildOptions, (args) => {
|
|
40
|
+
const commits = getCommits(args);
|
|
41
|
+
console.log(JSON.stringify(commits));
|
|
42
|
+
})
|
|
43
|
+
.command('get-latest-commit', '', buildOptions, (args) => {
|
|
44
|
+
const commits = getCommits(args);
|
|
45
|
+
if (commits.length > 0) {
|
|
46
|
+
console.log(JSON.stringify(commits[commits.length - 1]));
|
|
47
|
+
}
|
|
48
|
+
})
|
|
49
|
+
.command('get-changed-files', '', buildOptions, (args) => {
|
|
50
|
+
const commits = getCommits(args);
|
|
51
|
+
const changedFiles = getChangedFiles(commits);
|
|
52
|
+
console.log(JSON.stringify(changedFiles));
|
|
53
|
+
})
|
|
54
|
+
.command('get-actor-configs', '', (_) => _, async () => {
|
|
55
|
+
const actorConfigs = await getRepoActors();
|
|
56
|
+
console.log(JSON.stringify(actorConfigs));
|
|
57
|
+
})
|
|
58
|
+
.command('get-affected-actors', '', buildOptions, async (args) => {
|
|
59
|
+
const commits = getCommits(args);
|
|
60
|
+
const changedFiles = getChangedFiles(commits);
|
|
61
|
+
const actorConfigs = await getRepoActors();
|
|
62
|
+
const { actorsChanged } = getChangedActors({ filepathsChanged: changedFiles, actorConfigs, isLatest: false });
|
|
63
|
+
console.log(JSON.stringify(actorsChanged));
|
|
64
|
+
})
|
|
65
|
+
.command(
|
|
66
|
+
'report-tests',
|
|
67
|
+
'',
|
|
68
|
+
(args) => args
|
|
69
|
+
.option('report-file', { type: 'string', demandOption: true })
|
|
70
|
+
.option('job-url', { type: 'string' })
|
|
71
|
+
.option('workflow-name', { type: 'string' })
|
|
72
|
+
.option('repository', { type: 'string' }),
|
|
73
|
+
async (args) => {
|
|
74
|
+
await reportTestRestuls(args);
|
|
75
|
+
})
|
|
76
|
+
.command(
|
|
77
|
+
'build',
|
|
78
|
+
'',
|
|
79
|
+
(args) => buildOptions(args)
|
|
80
|
+
.option('dry-run', { type: 'boolean', default: false }),
|
|
81
|
+
async (args) => {
|
|
82
|
+
const commits = getCommits(args);
|
|
83
|
+
const changedFiles = getChangedFiles(commits);
|
|
84
|
+
const actorConfigs = await getRepoActors();
|
|
85
|
+
const { actorsChanged } = getChangedActors({
|
|
86
|
+
filepathsChanged: changedFiles,
|
|
87
|
+
actorConfigs,
|
|
88
|
+
});
|
|
89
|
+
// https://github.com/apify-store/google-maps#:actors/lukaskrivka_google-maps-with-contact-details
|
|
90
|
+
// git@github.com:apify-store/google-maps#:actors/lukaskrivka_google-maps-with-contact-details
|
|
91
|
+
const repoUrl = spawnCommandInGhWorkspace(`git remote get-url origin`)
|
|
92
|
+
.replace(/^https:\/\/github\.com\//, 'git@github.com:');
|
|
93
|
+
|
|
94
|
+
const builds = await runBuilds({
|
|
95
|
+
repoUrl,
|
|
96
|
+
actorConfigs: actorsChanged,
|
|
97
|
+
branch: args.sourceBranch.replace('origin/', ''),
|
|
98
|
+
dryRun: args.dryRun,
|
|
99
|
+
});
|
|
100
|
+
console.log(JSON.stringify(builds));
|
|
101
|
+
})
|
|
102
|
+
.command(
|
|
103
|
+
'release',
|
|
104
|
+
'',
|
|
105
|
+
(args) => args
|
|
106
|
+
.option('push-event-path', { type: 'string', demandOption: true })
|
|
107
|
+
.option('dry-run', { type: 'boolean', default: false }),
|
|
108
|
+
async (args) => {
|
|
109
|
+
const {
|
|
110
|
+
branch,
|
|
111
|
+
changedFiles,
|
|
112
|
+
repoUrl,
|
|
113
|
+
commits,
|
|
114
|
+
changelog,
|
|
115
|
+
repository,
|
|
116
|
+
} = await getPushData(args.pushEventPath);
|
|
117
|
+
const isLatest = true;
|
|
118
|
+
const actorConfigs = await getRepoActors();
|
|
119
|
+
const { actorsChanged } = getChangedActors({
|
|
120
|
+
filepathsChanged: changedFiles,
|
|
121
|
+
actorConfigs,
|
|
122
|
+
isLatest,
|
|
123
|
+
});
|
|
124
|
+
const { dryRun } = args;
|
|
125
|
+
const builds = await runBuilds({
|
|
126
|
+
isLatest,
|
|
127
|
+
repoUrl,
|
|
128
|
+
actorConfigs: actorsChanged,
|
|
129
|
+
branch,
|
|
130
|
+
dryRun,
|
|
131
|
+
});
|
|
132
|
+
console.error(JSON.stringify(builds));
|
|
133
|
+
|
|
134
|
+
// TODO: build circle actors
|
|
135
|
+
await notifyToSlack({
|
|
136
|
+
changedFiles,
|
|
137
|
+
commits,
|
|
138
|
+
changelog,
|
|
139
|
+
repository,
|
|
140
|
+
dryRun,
|
|
141
|
+
});
|
|
142
|
+
})
|
|
143
|
+
.strictCommands()
|
|
144
|
+
.demandCommand(1, 'Command is required')
|
|
145
|
+
.parse(hideBin(process.argv));
|
package/bin/slack.ts
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { WebClient } from '@slack/web-api';
|
|
2
|
+
import { Commit } from './types';
|
|
3
|
+
import { getEnvVar } from './utils.js';
|
|
4
|
+
|
|
5
|
+
type NotifyToSlackOptions = {
|
|
6
|
+
repository: string
|
|
7
|
+
changedFiles: string[]
|
|
8
|
+
changelog: string | null
|
|
9
|
+
commits: Commit[]
|
|
10
|
+
dryRun: boolean
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export const notifyToSlack = async ({
|
|
14
|
+
changedFiles,
|
|
15
|
+
commits,
|
|
16
|
+
changelog,
|
|
17
|
+
repository,
|
|
18
|
+
dryRun,
|
|
19
|
+
}: NotifyToSlackOptions) => {
|
|
20
|
+
const slack = new WebClient(getEnvVar('SLACK_TOKEN_RELEASES_BOT'));
|
|
21
|
+
|
|
22
|
+
if (!changelog) {
|
|
23
|
+
console.warn('No new changelog entries found, haven\'t you forgotten to update it?');
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
let shortMessage = `${repository}: new release:\n`;
|
|
27
|
+
|
|
28
|
+
// This one is just for broader public that only cares about public facing changes
|
|
29
|
+
if (changelog) {
|
|
30
|
+
console.error(`New changelog entries: ${changelog}`);
|
|
31
|
+
shortMessage += `Changelog:\n\n${changelog}\n`;
|
|
32
|
+
const channel = '#delivery-public-actors';
|
|
33
|
+
console.error(`Sending slack message to channel: ${channel}. Message: ${shortMessage}`);
|
|
34
|
+
if (!dryRun) {
|
|
35
|
+
await slack.chat.postMessage({
|
|
36
|
+
channel,
|
|
37
|
+
text: shortMessage,
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const commitsMessage = `${commits.map(({ author, message }, index) => ` ${index + 1} Commit message: ${message}\n Author: ${author}.`).join('\n')}`;
|
|
43
|
+
const changedFilesMessage = `Files changed: ${changedFiles.join(', ')}`;
|
|
44
|
+
const longMessage = `${shortMessage}\nCommits:\n\n${commitsMessage}\n\n${changedFilesMessage}`;
|
|
45
|
+
|
|
46
|
+
// This one is for devs and project managers that need to know more details
|
|
47
|
+
const notifChannel = `#notif-${repository.toLowerCase()}`;
|
|
48
|
+
console.error(`Sending slack message to channel: ${notifChannel}. Message: ${longMessage}`);
|
|
49
|
+
if (!dryRun) {
|
|
50
|
+
await slack.chat.postMessage({
|
|
51
|
+
text: longMessage,
|
|
52
|
+
channel: notifChannel,
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
export const sendSlackMessage = async (
|
|
58
|
+
channel: string,
|
|
59
|
+
text: string,
|
|
60
|
+
blocks: string[],
|
|
61
|
+
token: string,
|
|
62
|
+
) => {
|
|
63
|
+
const slack = new WebClient(token);
|
|
64
|
+
const { ts } = await slack.chat.postMessage({ text, channel });
|
|
65
|
+
if (blocks.length > 0 && ts) {
|
|
66
|
+
await slack.chat.postMessage({
|
|
67
|
+
channel,
|
|
68
|
+
thread_ts: ts,
|
|
69
|
+
blocks: blocks.map((t) => ({ text: { type: 'mrkdwn', text: t }, type: 'section' })),
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
};
|