@wordpress/e2e-tests 2.5.0 → 2.5.1

package/README.md

@@ -54,10 +54,10 @@ Debugging in a Chrome browser can be replaced with `vscode`'s debugger by adding
 			"type": "node",
 			"request": "launch",
 			"name": "Debug current e2e test",
-			"program": "${workspaceRoot}/node_modules/@wordpress/scripts/bin/wp-scripts.js",
+			"program": "${workspaceFolder}/node_modules/@wordpress/scripts/bin/wp-scripts.js",
 			"args": [
 				"test-e2e",
-				"--config=${workspaceRoot}/packages/e2e-tests/jest.config.js",
+				"--config=${workspaceFolder}/packages/e2e-tests/jest.config.js",
 				"--verbose=true",
 				"--runInBand",
 				"--watch",

package/config/flaky-tests-reporter.js

@@ -0,0 +1,94 @@
+/**
+ * A **flaky** test is defined as a test which passed after auto-retrying.
+ * - By default, all tests run once if they pass.
+ * - If a test fails, it will automatically re-run at most 2 times.
+ * - If it pass after retrying (below 2 times), then it's marked as **flaky**
+ *   but displayed as **passed** in the original test suite.
+ * - If it fail all 3 times, then it's a **failed** test.
+ */
+/**
+ * External dependencies
+ */
+const fs = require( 'fs' ).promises;
+const path = require( 'path' );
+const { formatResultsErrors } = require( 'jest-message-util' );
+
+class FlakyTestsReporter {
+	constructor( globalConfig, options ) {
+		this._globalConfig = globalConfig;
+		this._options = options;
+
+		this.failingTestCaseResults = new Map();
+	}
+
+	async onRunStart() {
+		try {
+			fs.mkdir( 'flaky-tests' );
+		} catch ( err ) {
+			// Ignore the error if the directory already exists.
+			if ( err.code !== 'EEXIST' ) {
+				throw err;
+			}
+		}
+	}
+
+	async onTestCaseResult( test, testCaseResult ) {
+		const testPath = path.relative( this._globalConfig.rootDir, test.path );
+		const testTitle = testCaseResult.title;
+
+		switch ( testCaseResult.status ) {
+			case 'failed': {
+				if ( ! this.failingTestCaseResults.has( testTitle ) ) {
+					this.failingTestCaseResults.set( testTitle, [] );
+				}
+				this.failingTestCaseResults
+					.get( testTitle )
+					.push( testCaseResult );
+				break;
+			}
+			case 'passed': {
+				if ( this.failingTestCaseResults.has( testTitle ) ) {
+					const failingResults = this.failingTestCaseResults.get(
+						testTitle
+					);
+
+					await fs.writeFile(
+						`flaky-tests/${ testTitle }.json`,
+						JSON.stringify( {
+							title: testTitle,
+							path: testPath,
+							results: failingResults,
+						} ),
+						'utf-8'
+					);
+
+					// Don't silence flaky error messages for debugging reason.
+					// eslint-disable-next-line no-console
+					console.error(
+						`Test passed after ${ failingResults.length } failed ${
+							failingResults.length === 1 ? 'attempt' : 'attempts'
+						}:`
+					);
+					// eslint-disable-next-line no-console
+					console.error(
+						formatResultsErrors(
+							failingResults,
+							this._globalConfig,
+							{},
+							test.path
+						)
+					);
+				}
+				break;
+			}
+			default:
+				break;
+		}
+	}
+
+	onRunComplete() {
+		this.failingTestCaseResults.clear();
+	}
+}
+
+module.exports = FlakyTestsReporter;

package/config/setup-test-framework.js

@@ -68,6 +68,13 @@ const pageEvents = [];
 // The Jest timeout is increased because these tests are a bit slow
 jest.setTimeout( PUPPETEER_TIMEOUT || 100000 );
 
+// Retry failed tests at most 2 times in CI.
+// This enables `flaky-tests-reporter` and `report-flaky-tests` GitHub action
+// to mark test as flaky and automatically create a tracking issue about it.
+if ( process.env.CI ) {
+	jest.retryTimes( 2 );
+}
+
 async function setupBrowser() {
 	await clearLocalStorage();
 	await setBrowserViewport( 'large' );

package/jest.config.js

@@ -1,5 +1,10 @@
+/**
+ * WordPress dependencies
+ */
+const baseConfig = require( '@wordpress/scripts/config/jest-e2e.config' );
+
 module.exports = {
-	...require( '@wordpress/scripts/config/jest-e2e.config' ),
+	...baseConfig,
 	setupFiles: [ '<rootDir>/config/gutenberg-phase.js' ],
 	setupFilesAfterEnv: [
 		'<rootDir>/config/setup-test-framework.js',
@@ -12,4 +17,9 @@ module.exports = {
 		'/node_modules/',
 		'e2e-tests/specs/performance/',
 	],
+	reporters: [
+		...baseConfig.reporters,
+		// Report flaky tests results into artifacts for used in `report-flaky-tests` action.
+		process.env.CI && '<rootDir>/config/flaky-tests-reporter.js',
+	].filter( Boolean ),
 };

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@wordpress/e2e-tests",
-	"version": "2.5.0",
+	"version": "2.5.1",
 	"description": "End-To-End (E2E) tests for WordPress.",
 	"author": "The WordPress Contributors",
 	"license": "GPL-2.0-or-later",
@@ -23,13 +23,14 @@
 		"node": ">=12"
 	},
 	"dependencies": {
-		"@wordpress/e2e-test-utils": "^5.4.3",
+		"@wordpress/e2e-test-utils": "^5.4.4",
 		"@wordpress/jest-console": "^4.1.0",
 		"@wordpress/jest-puppeteer-axe": "^3.1.0",
-		"@wordpress/scripts": "^18.0.1",
-		"@wordpress/url": "^3.2.2",
+		"@wordpress/scripts": "^18.1.0",
+		"@wordpress/url": "^3.2.3",
 		"chalk": "^4.0.0",
 		"expect-puppeteer": "^4.4.0",
+		"jest-message-util": "^27.0.6",
 		"lodash": "^4.17.21",
 		"puppeteer-testing-library": "^0.5.0",
 		"uuid": "^8.3.0"
@@ -42,5 +43,5 @@
 	"publishConfig": {
 		"access": "public"
 	},
-	"gitHead": "98c42a7187f788fe3e023f04df7f5dcbdae4e4e7"
+	"gitHead": "8f7f052bc04e3f4eb50f479ced14be1489b9fa79"
 }

package/plugins/iframed-block/block.json

@@ -0,0 +1,16 @@
+{
+	"apiVersion": 2,
+	"name": "test/iframed-block",
+	"title": "Iframed Block",
+	"category": "text",
+	"icon": "smiley",
+	"description": "",
+	"supports": {
+		"html": false
+	},
+	"textdomain": "iframed-block",
+	"editorScript": "iframed-block-editor",
+	"editorStyle": "file:./editor.css",
+	"style": "file:./style.css",
+	"script": "iframed-block-script"
+}

package/plugins/iframed-block/editor.css

@@ -0,0 +1,6 @@
+/**
+ * The following styles get applied inside the editor only.
+ */
+.wp-block-test-iframed-block {
+	border: 1px dotted #f00;
+}

package/plugins/iframed-block/editor.js

@@ -0,0 +1,18 @@
+( ( { wp: { element, blocks, blockEditor, compose }, jQuery: $ } ) => {
+	const { createElement: el } = element;
+	const { registerBlockType } = blocks;
+	const { useBlockProps } = blockEditor;
+	const { useRefEffect } = compose;
+
+	registerBlockType( 'test/iframed-block', {
+		edit: function Edit() {
+			const ref = useRefEffect( ( node ) => {
+				$( node ).test();
+			}, [] );
+			return el( 'p', useBlockProps( { ref } ), 'Iframed Block (edit)' );
+		},
+		save: function Save() {
+			return el( 'p', useBlockProps.save(), 'Iframed Block (saved)' );
+		},
+	} );
+} )( window );

package/plugins/iframed-block/jquery.test.js

@@ -0,0 +1,7 @@
+( ( $ ) => {
+	$.fn.test = function() {
+		return this.each( function() {
+			$( this ).text( 'Iframed Block (set with jQuery)' );
+		} );
+	};
+} )( window.jQuery );

package/plugins/iframed-block/script.js

@@ -0,0 +1,7 @@
+( ( win ) => {
+	const { jQuery: $ } = win;
+	win.addEventListener( 'DOMContentLoaded', () => {
+		$( '.wp-block-test-iframed-block' ).test();
+	} );
+} )( window );
+

package/plugins/iframed-block/style.css

@@ -0,0 +1,9 @@
+/**
+ * The following styles get applied both on the front of your site
+ * and in the editor.
+ */
+.wp-block-test-iframed-block {
+	background-color: #21759b;
+	color: #fff;
+	padding: 2px;
+}

package/plugins/iframed-block.php

@@ -0,0 +1,46 @@
+<?php
+/**
+ * Plugin Name: Gutenberg Test Iframed Block
+ * Plugin URI: https://github.com/WordPress/gutenberg
+ * Author: Gutenberg Team
+ *
+ * @package gutenberg-test-iframed-block
+ */
+
+add_action(
+	'setup_theme',
+	function() {
+		add_theme_support( 'block-templates' );
+	}
+);
+
+add_action(
+	'init',
+	function() {
+		wp_register_script(
+			'iframed-block-jquery-test',
+			plugin_dir_url( __FILE__ ) . 'iframed-block/jquery.test.js',
+			array( 'jquery' ),
+			filemtime( plugin_dir_path( __FILE__ ) . 'iframed-block/jquery.test.js' )
+		);
+		wp_register_script(
+			'iframed-block-editor',
+			plugin_dir_url( __FILE__ ) . 'iframed-block/editor.js',
+			array(
+				'wp-blocks',
+				'wp-block-editor',
+				'wp-element',
+				'wp-compose',
+				'iframed-block-jquery-test',
+			),
+			filemtime( plugin_dir_path( __FILE__ ) . 'iframed-block/editor.js' )
+		);
+		wp_register_script(
+			'iframed-block-script',
+			plugin_dir_url( __FILE__ ) . 'iframed-block/script.js',
+			array( 'iframed-block-jquery-test' ),
+			filemtime( plugin_dir_path( __FILE__ ) . 'iframed-block/script.js' )
+		);
+		register_block_type_from_metadata( __DIR__ . '/iframed-block' );
+	}
+);

package/specs/editor/plugins/__snapshots__/iframed-block.test.js.snap

@@ -0,0 +1,7 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`changing image size should load script and dependencies in iframe 1`] = `
+"<!-- wp:test/iframed-block -->
+<p class=\\"wp-block-test-iframed-block\\">Iframed Block (saved)</p>
+<!-- /wp:test/iframed-block -->"
+`;

package/specs/editor/plugins/align-hook.test.js

@@ -14,6 +14,7 @@ import {
 } from '@wordpress/e2e-test-utils';
 
 const alignLabels = {
+	none: 'None',
 	left: 'Align left',
 	center: 'Align center',
 	right: 'Align right',
@@ -21,114 +22,129 @@ const alignLabels = {
 	full: 'Full width',
 };
 
-describe( 'Align Hook Works As Expected', () => {
-	beforeAll( async () => {
-		await activatePlugin( 'gutenberg-test-align-hook' );
+/**
+ * Helper function to get the `labels` of align control. It actually evaluates the
+ * basic label of the button without the `info` text node if exists.
+ *
+ * @param {Object}  options                               Options for the util function
+ * @param {boolean} [options.getActiveButtonLabels=false] Flag for returning the active buttons labels only.
+ * @return {string[]} The matched labels.
+ */
+const getAlignmentToolbarLabels = async ( {
+	getActiveButtonLabels = false,
+} = {} ) => {
+	const selector = `.components-dropdown-menu__menu button${
+		getActiveButtonLabels ? '.is-active' : ''
+	} .components-menu-item__item`;
+	return page.evaluate( ( _selector ) => {
+		return (
+			Array.from( document.querySelectorAll( _selector ) )
+				/**
+				 * We neede this for now because conditionally there could be two nodes
+				 * with the same class(). This should be removed when the following
+				 * issue is resolved.
+				 *
+				 * @see https://github.com/WordPress/gutenberg/issues/34838
+				 */
+				.filter( ( contentNode ) => ! contentNode.childElementCount )
+				.map( ( contentNode ) => {
+					return contentNode.innerText;
+				} )
+		);
+	}, selector );
+};
+
+const expectActiveButtonLabelToBe = async ( expected ) => {
+	await clickBlockToolbarButton( 'Align' );
+	const activeButtonLabels = await getAlignmentToolbarLabels( {
+		getActiveButtonLabels: true,
 	} );
+	expect( activeButtonLabels ).toHaveLength( 1 );
+	expect( activeButtonLabels[ 0 ] ).toEqual( expected );
+};
 
-	beforeEach( async () => {
-		await createNewPost();
+const createShowsTheExpectedButtonsTest = ( blockName, buttonLabels ) => {
+	it( 'Shows the expected buttons on the alignment toolbar', async () => {
+		await insertBlock( blockName );
+		await clickBlockToolbarButton( 'Align' );
+		expect( await getAlignmentToolbarLabels() ).toEqual(
+			expect.arrayContaining( buttonLabels )
+		);
 	} );
+};
 
-	afterAll( async () => {
-		await deactivatePlugin( 'gutenberg-test-align-hook' );
+const createAppliesNoneAlignmentByDefaultTest = ( blockName ) => {
+	it( 'applies none alignment by default', async () => {
+		await insertBlock( blockName );
+		await expectActiveButtonLabelToBe( alignLabels.none );
 	} );
+};
+
+const verifyMarkupIsValid = async ( htmlMarkup ) => {
+	await setPostContent( htmlMarkup );
+	const blocks = await getAllBlocks();
+	expect( blocks ).toHaveLength( 1 );
+	expect( blocks[ 0 ].isValid ).toBeTruthy();
+};
+
+const createCorrectlyAppliesAndRemovesAlignmentTest = (
+	blockName,
+	alignment
+) => {
+	it( 'Correctly applies the selected alignment and correctly removes the alignment', async () => {
+		const BUTTON_XPATH = `//button[contains(@class,'components-dropdown-menu__menu-item')]//span[contains(text(), '${ alignLabels[ alignment ] }')]`;
 
-	const getAlignmentToolbarLabels = async () => {
+		// set the specified alignment.
+		await insertBlock( blockName );
 		await clickBlockToolbarButton( 'Align' );
-		const buttonLabels = await page.evaluate( () => {
-			return Array.from(
-				document.querySelectorAll(
-					'.components-dropdown-menu__menu button'
-				)
-			).map( ( button ) => {
-				return button.innerText;
-			} );
-		} );
-		return buttonLabels;
-	};
+		await ( await page.$x( BUTTON_XPATH ) )[ 0 ].click();
 
-	const createShowsTheExpectedButtonsTest = ( blockName, buttonLabels ) => {
-		it( 'Shows the expected buttons on the alignment toolbar', async () => {
-			await insertBlock( blockName );
-			expect( await getAlignmentToolbarLabels() ).toEqual( buttonLabels );
-		} );
-	};
+		// verify the button of the specified alignment is pressed.
+		await expectActiveButtonLabelToBe( alignLabels[ alignment ] );
 
-	const createDoesNotApplyAlignmentByDefaultTest = ( blockName ) => {
-		it( 'Does not apply any alignment by default', async () => {
-			await insertBlock( blockName );
-			await clickBlockToolbarButton( 'Align' );
-			const pressedButtons = await page.$$(
-				'.components-dropdown-menu__menu button.is-active'
-			);
-			expect( pressedButtons ).toHaveLength( 0 );
-		} );
-	};
-
-	const verifyMarkupIsValid = async ( htmlMarkup ) => {
-		await setPostContent( htmlMarkup );
-		const blocks = await getAllBlocks();
-		expect( blocks ).toHaveLength( 1 );
-		expect( blocks[ 0 ].isValid ).toBeTruthy();
-	};
-
-	const createCorrectlyAppliesAndRemovesAlignmentTest = (
-		blockName,
-		alignment
-	) => {
-		it( 'Correctly applies the selected alignment and correctly removes the alignment', async () => {
-			const BUTTON_XPATH = `//button[contains(@class,'components-dropdown-menu__menu-item') and contains(text(), '${ alignLabels[ alignment ] }')]`;
-			const BUTTON_PRESSED_SELECTOR =
-				'.components-dropdown-menu__menu button.is-active';
-			// set the specified alignment.
-			await insertBlock( blockName );
-			await clickBlockToolbarButton( 'Align' );
-			await ( await page.$x( BUTTON_XPATH ) )[ 0 ].click();
+		let htmlMarkup = await getEditedPostContent();
 
-			// verify the button of the specified alignment is pressed.
-			await clickBlockToolbarButton( 'Align' );
-			let pressedButtons = await page.$$( BUTTON_PRESSED_SELECTOR );
-			expect( pressedButtons ).toHaveLength( 1 );
+		// verify the markup of the selected alignment was generated.
+		expect( htmlMarkup ).toMatchSnapshot();
 
-			let htmlMarkup = await getEditedPostContent();
+		// verify the markup can be correctly parsed
+		await verifyMarkupIsValid( htmlMarkup );
 
-			// verify the markup of the selected alignment was generated.
-			expect( htmlMarkup ).toMatchSnapshot();
+		await selectBlockByClientId( ( await getAllBlocks() )[ 0 ].clientId );
 
-			// verify the markup can be correctly parsed
-			await verifyMarkupIsValid( htmlMarkup );
+		// remove the alignment.
+		await clickBlockToolbarButton( 'Align' );
+		await ( await page.$x( BUTTON_XPATH ) )[ 0 ].click();
 
-			await selectBlockByClientId(
-				( await getAllBlocks() )[ 0 ].clientId
-			);
+		// verify 'none' alignment button is in pressed state.
+		await expectActiveButtonLabelToBe( alignLabels.none );
 
-			// remove the alignment.
-			await clickBlockToolbarButton( 'Align' );
-			await ( await page.$x( BUTTON_XPATH ) )[ 0 ].click();
+		// verify alignment markup was removed from the block.
+		htmlMarkup = await getEditedPostContent();
+		expect( htmlMarkup ).toMatchSnapshot();
 
-			// verify no alignment button is in pressed state.
-			await clickBlockToolbarButton( 'Align' );
-			pressedButtons = await page.$$( BUTTON_PRESSED_SELECTOR );
-			expect( pressedButtons ).toHaveLength( 0 );
+		// verify the markup when no alignment is set is valid
+		await verifyMarkupIsValid( htmlMarkup );
 
-			// verify alignment markup was removed from the block.
-			htmlMarkup = await getEditedPostContent();
-			expect( htmlMarkup ).toMatchSnapshot();
+		await selectBlockByClientId( ( await getAllBlocks() )[ 0 ].clientId );
 
-			// verify the markup when no alignment is set is valid
-			await verifyMarkupIsValid( htmlMarkup );
+		// verify alignment `none` button is in pressed state after parsing the block.
+		await expectActiveButtonLabelToBe( alignLabels.none );
+	} );
+};
 
-			await selectBlockByClientId(
-				( await getAllBlocks() )[ 0 ].clientId
-			);
+describe( 'Align Hook Works As Expected', () => {
+	beforeAll( async () => {
+		await activatePlugin( 'gutenberg-test-align-hook' );
+	} );
 
-			// verify no alignment button is in pressed state after parsing the block.
-			await clickBlockToolbarButton( 'Align' );
-			pressedButtons = await page.$$( BUTTON_PRESSED_SELECTOR );
-			expect( pressedButtons ).toHaveLength( 0 );
-		} );
-	};
+	beforeEach( async () => {
+		await createNewPost();
+	} );
+
+	afterAll( async () => {
+		await deactivatePlugin( 'gutenberg-test-align-hook' );
+	} );
 
 	describe( 'Block with no alignment set', () => {
 		const BLOCK_NAME = 'Test No Alignment Set';
@@ -151,15 +167,12 @@ describe( 'Align Hook Works As Expected', () => {
 	describe( 'Block with align true', () => {
 		const BLOCK_NAME = 'Test Align True';
 
-		createShowsTheExpectedButtonsTest( BLOCK_NAME, [
-			alignLabels.left,
-			alignLabels.center,
-			alignLabels.right,
-			alignLabels.wide,
-			alignLabels.full,
-		] );
+		createShowsTheExpectedButtonsTest(
+			BLOCK_NAME,
+			Object.values( alignLabels )
+		);
 
-		createDoesNotApplyAlignmentByDefaultTest( BLOCK_NAME );
+		createAppliesNoneAlignmentByDefaultTest( BLOCK_NAME );
 
 		createCorrectlyAppliesAndRemovesAlignmentTest( BLOCK_NAME, 'right' );
 	} );
@@ -168,11 +181,12 @@ describe( 'Align Hook Works As Expected', () => {
 		const BLOCK_NAME = 'Test Align Array';
 
 		createShowsTheExpectedButtonsTest( BLOCK_NAME, [
+			alignLabels.none,
 			alignLabels.left,
 			alignLabels.center,
 		] );
 
-		createDoesNotApplyAlignmentByDefaultTest( BLOCK_NAME );
+		createAppliesNoneAlignmentByDefaultTest( BLOCK_NAME );
 
 		createCorrectlyAppliesAndRemovesAlignmentTest( BLOCK_NAME, 'center' );
 	} );
@@ -180,14 +194,11 @@ describe( 'Align Hook Works As Expected', () => {
 	describe( 'Block with default align', () => {
 		const BLOCK_NAME = 'Test Default Align';
 		const SELECTED_ALIGNMENT_CONTROL_SELECTOR =
-			'//div[contains(@class, "components-dropdown-menu__menu")]//button[contains(@class, "is-active")][text()="Align right"]';
-		createShowsTheExpectedButtonsTest( BLOCK_NAME, [
-			alignLabels.left,
-			alignLabels.center,
-			alignLabels.right,
-			alignLabels.wide,
-			alignLabels.full,
-		] );
+			'//div[contains(@class, "components-dropdown-menu__menu")]//button[contains(@class, "is-active")]/span[text()="Align right"]';
+		createShowsTheExpectedButtonsTest(
+			BLOCK_NAME,
+			Object.values( alignLabels )
+		);
 
 		it( 'Applies the selected alignment by default', async () => {
 			await insertBlock( BLOCK_NAME );

package/specs/editor/plugins/iframed-block.test.js

@@ -0,0 +1,58 @@
+/**
+ * WordPress dependencies
+ */
+import {
+	activatePlugin,
+	createNewPost,
+	deactivatePlugin,
+	insertBlock,
+	getEditedPostContent,
+	openDocumentSettingsSidebar,
+	clickButton,
+	canvas,
+} from '@wordpress/e2e-test-utils';
+
+describe( 'changing image size', () => {
+	beforeEach( async () => {
+		await activatePlugin( 'gutenberg-test-iframed-block' );
+		await createNewPost( { postType: 'page' } );
+	} );
+
+	afterEach( async () => {
+		await deactivatePlugin( 'gutenberg-test-iframed-block' );
+	} );
+
+	it( 'should load script and dependencies in iframe', async () => {
+		await insertBlock( 'Iframed Block' );
+
+		expect( await getEditedPostContent() ).toMatchSnapshot();
+
+		const element = await page.waitForSelector(
+			'.wp-block-test-iframed-block'
+		);
+		const text = await element.evaluate( ( el ) => el.textContent );
+
+		expect( text ).toBe( 'Iframed Block (set with jQuery)' );
+
+		await openDocumentSettingsSidebar();
+		await clickButton( 'Page' );
+		await clickButton( 'Template' );
+		await clickButton( 'New' );
+		await page.keyboard.press( 'Tab' );
+		await page.keyboard.press( 'Tab' );
+		await page.keyboard.type( 'Iframed Test' );
+		await clickButton( 'Create' );
+		await page.waitForSelector( 'iframe[name="editor-canvas"]' );
+
+		const iframeElement = await canvas().waitForSelector(
+			'.wp-block-test-iframed-block'
+		);
+		const iframedText = await iframeElement.evaluate(
+			( el ) => el.textContent
+		);
+
+		// Expect the script to load in the iframe, which replaces the block
+		// text.
+		expect( iframedText ).toBe( 'Iframed Block (set with jQuery)' );
+	} );
+} );