apify-test-tools 0.6.0 → 0.6.2-beta.0

package/bin/github.ts

@@ -1,5 +1,6 @@
-import fs from 'fs/promises';
-import { Commit, GitHubEventPush } from './types.js';
+import fs from 'node:fs/promises';
+
+import type { Commit, GitHubEventPush } from './types.js';
 import { spawnCommandInGhWorkspace } from './utils.js';
 
 export const getPushData = async (path: string) => {
@@ -10,8 +11,8 @@ export const getPushData = async (path: string) => {
         repository: { ssh_url: repoUrl, name },
         head_commit: { author },
     } = event;
-    const commits: Commit[] = ghCommits.map(({ author, message, id, timestamp }) => ({
-        author: author.username,
+    const commits: Commit[] = ghCommits.map(({ author: commitAuthor, message, id, timestamp }) => ({
+        author: commitAuthor.username,
         sha: id,
         message,
         date: timestamp,
@@ -29,7 +30,7 @@ export const getPushData = async (path: string) => {
     };
 };
 
-export const loadGitHubEvent = async (path: string): Promise<GitHubEventPush> => {
+const loadGitHubEvent = async (path: string): Promise<GitHubEventPush> => {
     const pushEvent = JSON.parse((await fs.readFile(path)).toString()) as GitHubEventPush;
     return pushEvent;
 };
@@ -39,7 +40,7 @@ const getChangedFiles = ({ after, before }: GitHubEventPush): string[] => {
     return changedFiles.split('\n');
 };
 
-export const getChangelogChanges = (changedFiles: string[], event: GitHubEventPush): string | null => {
+const getChangelogChanges = (changedFiles: string[], event: GitHubEventPush): string | null => {
     const changelogPath = 'CHANGELOG.md';
     if (!changedFiles.includes(changelogPath)) {
         return null;

package/bin/main.ts

@@ -1,15 +1,18 @@
 #!/usr/bin/env node
 
-import process from 'process';
+import process from 'node:process';
+
 import yargs, { type Argv } from 'yargs';
+// eslint-disable-next-line import/extensions --- With .js, it cannot find types
 import { hideBin } from 'yargs/helpers';
 
-import { getRepoActors, getChangedActors, spawnCommandInGhWorkspace, setCwd } from './utils.js';
-import { runBuilds } from './build.js';
+import { deleteOldBuilds, runBuilds } from './build.js';
+import { getChangedActors } from './diff-changes.js';
 import { getChangedFiles, getCommits } from './git.js';
 import { getPushData } from './github.js';
 import { notifyToSlack } from './slack.js';
 import { reportTestResults } from './test-report.js';
+import { getRepoActors, setCwd, spawnCommandInGhWorkspace } from './utils.js';
 
 /**
  * Middlewares to be run before every command execution
@@ -63,13 +66,18 @@ await yargs()
         async () => {
             const actorConfigs = await getRepoActors();
             console.log(JSON.stringify(actorConfigs));
-        }
+        },
     )
     .command('get-affected-actors', '', buildOptions, async (args) => {
         const commits = getCommits(args);
         const changedFiles = getChangedFiles(commits);
         const actorConfigs = await getRepoActors();
-        const { actorsChanged } = getChangedActors({ filepathsChanged: changedFiles, actorConfigs, isLatest: false });
+        const actorsChanged = getChangedActors({
+            filepathsChanged: changedFiles,
+            actorConfigs,
+            isLatest: false,
+            commits,
+        });
         console.log(JSON.stringify(actorsChanged));
     })
     .command(
@@ -83,7 +91,7 @@ await yargs()
                 .option('workflow-name', { type: 'string' }),
         async (args) => {
             await reportTestResults(args);
-        }
+        },
     )
     .command(
         'build',
@@ -93,15 +101,16 @@ await yargs()
             const commits = getCommits(args);
             const changedFiles = getChangedFiles(commits);
             const actorConfigs = await getRepoActors();
-            const { actorsChanged } = getChangedActors({
+            const actorsChanged = getChangedActors({
                 filepathsChanged: changedFiles,
                 actorConfigs,
+                commits,
             });
             // https://github.com/apify-store/google-maps#:actors/lukaskrivka_google-maps-with-contact-details
             // git@github.com:apify-store/google-maps#:actors/lukaskrivka_google-maps-with-contact-details
             const repoUrl = spawnCommandInGhWorkspace(`git remote get-url origin`).replace(
                 /^https:\/\/github\.com\//,
-                'git@github.com:'
+                'git@github.com:',
             );
 
             const builds = await runBuilds({
@@ -111,7 +120,7 @@ await yargs()
                 dryRun: args.dryRun,
             });
             console.log(JSON.stringify(builds));
-        }
+        },
     )
     .command(
         'release',
@@ -124,14 +133,15 @@ await yargs()
                 .option('release-slack-channel', { type: 'string' }),
         async (args) => {
             const { branch, changedFiles, repoUrl, commits, changelog, repository, author } = await getPushData(
-                args.pushEventPath
+                args.pushEventPath,
             );
             const isLatest = true;
             const actorConfigs = await getRepoActors();
-            const { actorsChanged } = getChangedActors({
+            const actorsChanged = getChangedActors({
                 filepathsChanged: changedFiles,
                 actorConfigs,
                 isLatest,
+                commits,
             });
             const { dryRun, reportSlackChannel, releaseSlackChannel } = args;
             const builds = await runBuilds({
@@ -143,7 +153,6 @@ await yargs()
             });
             console.error(JSON.stringify(builds));
 
-            // TODO: build circle actors
             await notifyToSlack({
                 changedFiles,
                 commits,
@@ -154,7 +163,16 @@ await yargs()
                 reportSlackChannel,
                 releaseSlackChannel,
             });
-        }
+        },
+    )
+    .command(
+        'delete-old-builds',
+        '',
+        (_) => _,
+        async () => {
+            const actorConfigs = await getRepoActors();
+            await deleteOldBuilds(actorConfigs);
+        },
     )
     .strictCommands()
     .demandCommand(1, 'Command is required')

package/bin/slack.ts

@@ -1,5 +1,6 @@
 import { WebClient } from '@slack/web-api';
-import { Commit } from './types.js';
+
+import type { Commit } from './types.js';
 import { getEnvVar } from './utils.js';
 
 type NotifyToSlackOptions = {
@@ -46,7 +47,10 @@ export const notifyToSlack = async ({
     }
 
     const commitsMessage = `${commits
-        .map(({ author, message }, index) => `${index + 1}. Commit message: ${message}\n\tAuthor: ${author}.`)
+        .map(
+            ({ author: commitAuthor, message }, index) =>
+                `${index + 1}. Commit message: ${message}\n\tAuthor: ${commitAuthor}.`,
+        )
         .join('\n')}`;
     const changedFilesMessage = `**Files changed**: ${changedFiles.join(', ')}`;
     const longMessage = `${shortMessage}\n**Commit list**:\n${commitsMessage}\n\n${changedFilesMessage}`;

package/bin/test-report.ts

@@ -1,13 +1,14 @@
-import fs from 'fs/promises';
+import fs from 'node:fs/promises';
+
 import { sendSlackMessage } from './slack.js';
 import { getEnvVar } from './utils.js';
 
 interface ReportTestResultsOptions {
-    reportFile: string
-    dryRun: boolean
-    reportSlackChannel?: string
-    jobUrl?: string
-    workflowName?: string
+    reportFile: string;
+    dryRun: boolean;
+    reportSlackChannel?: string;
+    jobUrl?: string;
+    workflowName?: string;
 }
 
 export const reportTestResults = async ({
@@ -35,7 +36,7 @@ export const reportTestResults = async ({
         }
     }
 
-    const failedAssertions: { message: string; runLink: string, actorName: string }[] = [];
+    const failedAssertions: { message: string; runLink: string; actorName: string }[] = [];
 
     console.error();
     console.error(`PASSED: ${passed.length}, FAILED: ${failed.length}`);
@@ -57,7 +58,13 @@ export const reportTestResults = async ({
     for (const [i, aResult] of failed.entries()) {
         const { failureMessages, fullName, meta } = aResult;
         if (failureMessages) {
-            failedAssertions.push(...failureMessages.map(message => ({ message: message.split('\n')?.[0], runLink: meta.runLink, actorName: meta.actorName })));
+            failedAssertions.push(
+                ...failureMessages.map((message) => ({
+                    message: message.split('\n')?.[0],
+                    runLink: meta.runLink,
+                    actorName: meta.actorName,
+                })),
+            );
         }
         console.error(`${i + 1}) ${fullName} ... ${meta.runLink}`);
         console.error();
@@ -65,9 +72,11 @@ export const reportTestResults = async ({
     console.error();
     console.error(`PASSED: ${passed.length}, FAILED: ${failed.length}`);
     console.error();
-    
+
     if (!reportSlackChannel) {
-        console.error(`Skipping slack notification. If you want to enable it, add --report-slack-channel flag and make sure SLACK_TOKEN_TESTS_BOT env variable is set.`);
+        console.error(
+            `Skipping slack notification. If you want to enable it, add --report-slack-channel flag and make sure SLACK_TOKEN_TESTS_BOT env variable is set.`,
+        );
         return;
     }
 
@@ -81,7 +90,9 @@ export const reportTestResults = async ({
     let slackMessage = `\`${workflowName ?? '-'}\``;
     slackMessage += `: has ${failedAssertions.length} failed assertions. Failing test suites: ${failed.length}/${total}.${jobLink}`;
     slackMessage += `\n\n${failedAssertions[0].message} --- <${failedAssertions[0].runLink}|${failedAssertions[0].actorName}>`;
-    const blocks = failedAssertions.slice(1).map(({ message, runLink, actorName }) => `• ${message} --- <${runLink}|${actorName}>`);
+    const blocks = failedAssertions
+        .slice(1)
+        .map(({ message, runLink, actorName }) => `• ${message} --- <${runLink}|${actorName}>`);
 
     console.error('SLACK:', slackMessage);
     console.error('\tblocks:', blocks.join('\n\t\t'));
@@ -96,52 +107,52 @@ export const reportTestResults = async ({
     }
 };
 
-type Status = 'passed' | 'failed' | 'skipped' | 'pending' | 'todo' | 'disabled'
-type Milliseconds = number
+type Status = 'passed' | 'failed' | 'skipped' | 'pending' | 'todo' | 'disabled';
+type Milliseconds = number;
 interface Callsite {
-    line: number
-    column: number
+    line: number;
+    column: number;
 }
 
 interface JsonAssertionResult {
-    ancestorTitles: Array<string>
-    fullName: string
-    status: Status
-    title: string
+    ancestorTitles: string[];
+    fullName: string;
+    status: Status;
+    title: string;
     meta: {
-        runId: string
-        runLink: string
-        actorName: string
-    }
-    duration?: Milliseconds | null
-    failureMessages: Array<string> | null
-    location?: Callsite | null
+        runId: string;
+        runLink: string;
+        actorName: string;
+    };
+    duration?: Milliseconds | null;
+    failureMessages: string[] | null;
+    location?: Callsite | null;
 }
 
 interface JsonTestResult {
-    message: string
-    name: string
-    status: 'failed' | 'passed'
-    startTime: number
-    endTime: number
-    assertionResults: Array<JsonAssertionResult>
+    message: string;
+    name: string;
+    status: 'failed' | 'passed';
+    startTime: number;
+    endTime: number;
+    assertionResults: JsonAssertionResult[];
     // summary: string
     // coverage: unknown
 }
 
 interface JsonTestResults {
-    numFailedTests: number
-    numFailedTestSuites: number
-    numPassedTests: number
-    numPassedTestSuites: number
-    numPendingTests: number
-    numPendingTestSuites: number
-    numTodoTests: number
-    numTotalTests: number
-    numTotalTestSuites: number
-    startTime: number
-    success: boolean
-    testResults: Array<JsonTestResult>
+    numFailedTests: number;
+    numFailedTestSuites: number;
+    numPassedTests: number;
+    numPassedTestSuites: number;
+    numPendingTests: number;
+    numPendingTestSuites: number;
+    numTodoTests: number;
+    numTotalTests: number;
+    numTotalTestSuites: number;
+    startTime: number;
+    success: boolean;
+    testResults: JsonTestResult[];
     // snapshot: SnapshotSummary
     // coverageMap?: CoverageMap | null | undefined
     // numRuntimeErrorTestSuites: number

package/bin/types.ts

@@ -1,91 +1,88 @@
 export interface Config {
-    targetBranch: string
-    sourceBranch: string
-    baseCommit?: string
-    workspace?: string
+    targetBranch: string;
+    sourceBranch: string;
+    baseCommit?: string;
+    workspace?: string;
 }
 
 export type Commit = {
-    sha: string
-    author: string
-    date: string
-    message: string
-}
+    sha: string;
+    author: string;
+    date: string;
+    message: string;
+};
 
 // NOTE: The GitHub types are incomplete, feel free to add fields/complete (not sure how stable GitHub API is)
 export interface GitHubHeadCommit {
-    added: string[]
-    removed: string[]
-    modified: string[],
+    added: string[];
+    removed: string[];
+    modified: string[];
     author: {
-        name: string
-    },
-    message: string,
-    id: string,
+        name: string;
+    };
+    message: string;
+    id: string;
 }
 
 export interface Repository {
-    full_name: string
-    name: string
-    ssh_url: string
+    full_name: string;
+    name: string;
+    ssh_url: string;
     owner: {
-        login: string
-    }
+        login: string;
+    };
 }
 
 export type GitHubEvent = GitHubEventPullRequest | GitHubEventPush;
 
-// All optional type so we check that we have all required fields for each event type
-export type RawGitHubEvent = Partial<Omit<GitHubEventPullRequest, 'type'> & Omit<GitHubEventPush, 'type'>>;
-
 export interface GitHubEventPullRequest {
     // Type is our own variable we inject to have discriminated union
     // The rest of the properties are raw from GitHub API
-    type: 'pull_request',
+    type: 'pull_request';
     pull_request: {
-        number: number
+        number: number;
         base: {
-            ref: string,
-            sha: string
-        }
+            ref: string;
+            sha: string;
+        };
         head: {
-            ref: string
-            sha: string
-        }
-    }
-    repository: Repository,
+            ref: string;
+            sha: string;
+        };
+    };
+    repository: Repository;
 }
 
 export interface GitHubEventPush {
-    type: 'push',
-    head_commit: GitHubHeadCommit,
-    repository: Repository,
-    ref: string
-    commits: GithubCommit[]
-    before: string
-    after: string
+    type: 'push';
+    head_commit: GitHubHeadCommit;
+    repository: Repository;
+    ref: string;
+    commits: GithubCommit[];
+    before: string;
+    after: string;
 }
 
 export interface GithubCommit {
-    id: string
-    tree_id: string
-    distinct: boolean
-    message: string
-    timestamp: string
-    url: string
+    id: string;
+    tree_id: string;
+    distinct: boolean;
+    message: string;
+    timestamp: string;
+    url: string;
     author: {
-        name: string
-        email: string
-        username: string
-    }
+        name: string;
+        email: string;
+        username: string;
+    };
     committer: {
-        name: string
-        email: string
-        username: string
-    }
-    added: string[]
-    removed: string[]
-    modified: string[]
+        name: string;
+        email: string;
+        username: string;
+    };
+    added: string[];
+    removed: string[];
+    modified: string[];
 }
 
 export interface BuildData {
@@ -99,5 +96,5 @@ export interface BuildData {
 export interface ActorConfig {
     actorName: string;
     folder: string;
-    isStandalone: boolean
+    isStandalone: boolean;
 }

package/bin/utils.ts

@@ -1,10 +1,7 @@
-import fs from 'node:fs/promises';
 import { spawnSync } from 'node:child_process';
-import type {
-    ActorConfig,
-    Commit,
-    GitHubEvent,
-} from './types.js';
+import fs from 'node:fs/promises';
+
+import type { ActorConfig, GitHubEvent } from './types.js';
 
 export const spawnCommandInGhWorkspace = (command: string, args: string[] = []) => {
     console.error(command, args.join(' '));
@@ -49,14 +46,14 @@ export const getRepoActors = async (): Promise<ActorConfig[]> => {
     let actorDirs: string[];
     try {
         actorDirs = (await fs.readdir(`./actors`)).map((dir) => `actors/${dir}`);
-    } catch (err) {
+    } catch {
         console.warn(`No /actors directory found in repo`);
         actorDirs = [];
     }
     let standaloneActorDirs: string[];
     try {
         standaloneActorDirs = (await fs.readdir(`./standalone-actors`)).map((dir) => `standalone-actors/${dir}`);
-    } catch (err) {
+    } catch {
         console.warn(`No /standalone-actors directory found in repo`);
         standaloneActorDirs = [];
     }
@@ -73,8 +70,18 @@ export const getRepoActors = async (): Promise<ActorConfig[]> => {
             isStandalone: folderType === 'standalone-actors',
         });
     }
-    console.error(`Actors in repo: ${actorConfigs.filter(({ isStandalone }) => !isStandalone).map(({ actorName }) => actorName).join(', ')}`);
-    console.error(`Standalone actors in repo: ${actorConfigs.filter(({ isStandalone }) => !!isStandalone).map(({ actorName }) => actorName).join(', ')}`);
+    console.error(
+        `Actors in repo: ${actorConfigs
+            .filter(({ isStandalone }) => !isStandalone)
+            .map(({ actorName }) => actorName)
+            .join(', ')}`,
+    );
+    console.error(
+        `Standalone actors in repo: ${actorConfigs
+            .filter(({ isStandalone }) => !!isStandalone)
+            .map(({ actorName }) => actorName)
+            .join(', ')}`,
+    );
     return actorConfigs;
 };
 
@@ -85,127 +92,8 @@ export const setCwd = ({ workspace }: { workspace: string | undefined }) => {
     }
     const ghWorkspace = getEnvVar('GITHUB_WORKSPACE', process.cwd());
     process.chdir(ghWorkspace);
-}
-
-export const getHeadCommitSha = (githubEvent: GitHubEvent) => {
-    return githubEvent.type === 'pull_request'
-        ? githubEvent.pull_request.head.sha
-        : githubEvent.head_commit.id;
 };
 
-export interface GetChangedActorsResult {
-    actorsChanged: ActorConfig[];
-    codeChanged: boolean;
-}
-
-interface ShouldBuildAndTestOptions {
-    filepathsChanged: string[];
-    actorConfigs: ActorConfig[];
-    // Just for logging
-    isLatest?: boolean;
-}
-
-/**
- * Also works for folders
- */
-const isIgnoredTopLevelFile = (lowercaseFilePath: string) => {
-    // On top level, we should only have dev-only readme and .actor/ is just for apify push CLI (real Actor configs are in /actors)
-    const IGNORED_TOP_LEVEL_FILES = ['.vscode/', '.gitignore', 'readme.md', '.husky/', '.eslintrc', '.editorconfig', '.actor/'];
-    // Strip out deprecated /code and /shared folders, treat them as top-level code
-    const sanitizedLowercaseFilePath = lowercaseFilePath.replace(/^code\//, '').replace(/^shared\//, '');
-
-    return IGNORED_TOP_LEVEL_FILES.some((ignoredFile) => sanitizedLowercaseFilePath.startsWith(ignoredFile));
-};
-
-const isLatestBuildOnlyFile = (lowercaseFilePath: string) => {
-    if (lowercaseFilePath.endsWith('changelog.md')) {
-        return true;
-    }
-
-    // Either in /actors or /standalone-actors, we need to rebuild readme but we don't rebuild top-level dev-only readme
-    if ((lowercaseFilePath.startsWith('actors/') || lowercaseFilePath.startsWith('standalone-actors/')) && lowercaseFilePath.endsWith('readme.md')) {
-        return true;
-    }
-
-    return false;
-};
-
-/**
- * Latest and devel are the same except that for latest we also rebuild with README and CHANGELOG files
- */
-export const getChangedActors = (
-    { filepathsChanged, actorConfigs, isLatest = false }: ShouldBuildAndTestOptions,
-): GetChangedActorsResult => {
-    let codeChanged = false;
-    // folder -> ActorConfig
-    const actorsChangedMap = new Map<string, ActorConfig>();
-
-    const actorConfigsWithoutStandalone = actorConfigs.filter(({ isStandalone }) => !isStandalone);
-
-    const lowercaseFiles = filepathsChanged.map((file) => file.toLowerCase());
-
-    for (const lowercaseFilePath of lowercaseFiles) {
-        if (isIgnoredTopLevelFile(lowercaseFilePath)) {
-            continue;
-        }
-        // First we check for specific actors that have configs in /actors or standalone actors in /standalone-actors
-        // This matches both actors/username_actorName and standalone-actors/username_actorName
-        const changedActorConfigMatch = lowercaseFilePath.match(/^(?:standalone-)?actors\/([^/]+)\/.+/);
-        if (changedActorConfigMatch) {
-            const sanitizedActorName = changedActorConfigMatch[1].replace('_', '/');
-            const actorConfigChanged = actorConfigs.find(({ actorName }) => actorName.toLowerCase() === sanitizedActorName);
-            if (actorConfigChanged === undefined) {
-                console.warn('changes was found in an actor folder which no longer exists in the current commit', {
-                    actorName: sanitizedActorName,
-                    actorFolderName: changedActorConfigMatch[1],
-                });
-                continue;
-            }
-
-            console.error(`actorConfigChanged ${actorConfigChanged.actorName}: sanitizedActorName ${sanitizedActorName} ${lowercaseFilePath} `);
-            // These can be nested at various folders inside the actor folder
-            if (isLatest || !isLatestBuildOnlyFile(lowercaseFilePath)) {
-                // We assume other files will are either actor.json or input_schema.json and those needs to be tested
-                // TODO: Check what changed in schema, we don't need to test description changes
-                actorsChangedMap.set(actorConfigChanged.folder, actorConfigChanged);
-            }
-            continue;
-        }
-
-        // We check top level files (formerly in /code and /shared folders) that are shared among all non-standalone Actors
-        // Standalone actors are always handled separately by name via changedActorConfigMatch
-        if (isLatest || !isLatestBuildOnlyFile(lowercaseFilePath)) {
-            codeChanged = !isLatest; // NOTE: code is changed only in PR
-            for (const actorConfig of actorConfigsWithoutStandalone) {
-                actorsChangedMap.set(actorConfig.folder, actorConfig);
-            }
-        }
-    }
-
-    const actorsChanged = Array.from(actorsChangedMap.values());
-
-    // All below here is just for logging
-    const ignoredFilesChanged = lowercaseFiles.filter((file) => isIgnoredTopLevelFile(file));
-    console.error(`[DIFF]: Top level files changed that we ignore (don't trigger test or build): ${ignoredFilesChanged.join(', ')}`);
-
-    const onlyLatestFilesChanged = lowercaseFiles.filter((file) => isLatestBuildOnlyFile(file));
-    console.error(`[DIFF]: Files changed that only trigger latest build: ${onlyLatestFilesChanged.join(', ')}`);
-
-    if (!isLatest && codeChanged) {
-        console.error(`[DIFF]: All non-standalone Actors need to be built and tested (changes in top-level code)`);
-    }
-
-    if (actorsChanged.length > 0) {
-        const miniactors = actorsChanged.filter((config) => !config.isStandalone).map((config) => config.actorName);
-        const standaloneActors = actorsChanged.filter((config) => config.isStandalone).map((config) => config.actorName);
-        console.error(`[DIFF]: MiniActors to be built and tested: ${miniactors.join(', ')}`);
-        console.error(`[DIFF]: Standalone Actors to be built and tested: ${standaloneActors.join(', ')}`);
-    } else {
-        console.error(`[DIFF]: No relevant files changed, skipping builds and tests`);
-    }
-
-    return {
-        actorsChanged,
-        codeChanged,
-    };
+export const getHeadCommitSha = (githubEvent: GitHubEvent) => {
+    return githubEvent.type === 'pull_request' ? githubEvent.pull_request.head.sha : githubEvent.head_commit.id;
 };

package/dist/bin/build.d.ts

@@ -12,7 +12,7 @@ type RunBuildsOptions = {
     branch: string;
     dryRun: boolean;
 };
-export declare const runBuilds: ({ repoUrl, branch, actorConfigs, isLatest, dryRun, }: RunBuildsOptions) => Promise<BuildPrActorOptions[] | BuildData[]>;
+export declare const runBuilds: ({ repoUrl, branch, actorConfigs, isLatest, dryRun }: RunBuildsOptions) => Promise<BuildPrActorOptions[] | BuildData[]>;
 export declare const deleteOldBuilds: (actorConfigs: ActorConfig[]) => Promise<void>;
 export {};
 //# sourceMappingURL=build.d.ts.map