@wordpress/e2e-tests 6.0.0 → 6.2.0

package/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+## 6.2.0 (2023-01-11)
+
+## 6.1.0 (2023-01-02)
+
 ## 6.0.0 (2022-12-14)
 
 ### Breaking Changes

package/LICENSE.md

@@ -1,6 +1,6 @@
 ## Gutenberg
 
-    Copyright 2016-2022 by the contributors
+    Copyright 2016-2023 by the contributors
 
 **License for Contributions (on and after April 15, 2021)**
 

package/assets/small-post-with-containers.html

@@ -0,0 +1,77 @@
+<!-- wp:columns {"align":"full"} -->
+<div class="wp-block-columns alignfull"><!-- wp:column -->
+<div class="wp-block-column"><!-- wp:heading {"level":3} -->
+<h3 class="wp-block-heading">Heading</h3>
+<!-- /wp:heading -->
+
+<!-- wp:paragraph -->
+<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+<!-- /wp:paragraph -->
+
+<!-- wp:list -->
+<ul><!-- wp:list-item -->
+<li>one</li>
+<!-- /wp:list-item -->
+
+<!-- wp:list-item -->
+<li>two</li>
+<!-- /wp:list-item -->
+
+<!-- wp:list-item -->
+<li>three</li>
+<!-- /wp:list-item --></ul>
+<!-- /wp:list --></div>
+<!-- /wp:column -->
+
+<!-- wp:column -->
+<div class="wp-block-column"><!-- wp:heading {"level":3} -->
+<h3 class="wp-block-heading">Heading</h3>
+<!-- /wp:heading -->
+
+<!-- wp:paragraph -->
+<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+<!-- /wp:paragraph -->
+
+<!-- wp:list -->
+<ul><!-- wp:list-item -->
+<li>one</li>
+<!-- /wp:list-item -->
+
+<!-- wp:list-item -->
+<li>two</li>
+<!-- /wp:list-item -->
+
+<!-- wp:list-item -->
+<li>three</li>
+<!-- /wp:list-item --></ul>
+<!-- /wp:list --></div>
+<!-- /wp:column -->
+
+<!-- wp:column -->
+<div class="wp-block-column"><!-- wp:heading {"level":3} -->
+<h3 class="wp-block-heading">Heading</h3>
+<!-- /wp:heading -->
+
+<!-- wp:paragraph -->
+<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+<!-- /wp:paragraph -->
+
+<!-- wp:list -->
+<ul><!-- wp:list-item -->
+<li>one</li>
+<!-- /wp:list-item -->
+
+<!-- wp:list-item -->
+<li>two</li>
+<!-- /wp:list-item -->
+
+<!-- wp:list-item -->
+<li>three</li>
+<!-- /wp:list-item --></ul>
+<!-- /wp:list --></div>
+<!-- /wp:column --></div>
+<!-- /wp:columns -->
+
+<!-- wp:paragraph -->
+<p></p>
+<!-- /wp:paragraph -->

package/config/performance-reporter.js

@@ -36,6 +36,7 @@ class PerformanceReporter {
 			firstContentfulPaint,
 			firstBlock,
 			type,
+			typeContainer,
 			focus,
 			listViewOpen,
 			inserterOpen,
@@ -68,7 +69,7 @@ Average time to first block: ${ success(
 		if ( type && type.length ) {
 			// eslint-disable-next-line no-console
 			console.log( `
-${ title( 'Typing Performance:' ) }
+${ title( 'Typing:' ) }
 Average time to type character: ${ success( round( average( type ) ) + 'ms' ) }
 Slowest time to type character: ${ success(
 				round( Math.max( ...type ) ) + 'ms'
@@ -78,10 +79,25 @@ Fastest time to type character: ${ success(
 			) }` );
 		}
 
+		if ( typeContainer && typeContainer.length ) {
+			// eslint-disable-next-line no-console
+			console.log( `
+${ title( 'Typing within a container:' ) }
+Average time to type within a container: ${ success(
+				round( average( typeContainer ) ) + 'ms'
+			) }
+Slowest time to type within a container: ${ success(
+				round( Math.max( ...typeContainer ) ) + 'ms'
+			) }
+Fastest time to type within a container: ${ success(
+				round( Math.min( ...typeContainer ) ) + 'ms'
+			) }` );
+		}
+
 		if ( focus && focus.length ) {
 			// eslint-disable-next-line no-console
 			console.log( `
-${ title( 'Block Selection Performance:' ) }
+${ title( 'Block Selection:' ) }
 Average time to select a block: ${ success( round( average( focus ) ) + 'ms' ) }
 Slowest time to select a block: ${ success(
 				round( Math.max( ...focus ) ) + 'ms'
@@ -94,7 +110,7 @@ Fastest time to select a block: ${ success(
 		if ( listViewOpen && listViewOpen.length ) {
 			// eslint-disable-next-line no-console
 			console.log( `
-${ title( 'Opening List View Performance:' ) }
+${ title( 'Opening List View:' ) }
 Average time to open list view: ${ success(
 				round( average( listViewOpen ) ) + 'ms'
 			) }
@@ -109,7 +125,7 @@ Fastest time to open list view: ${ success(
 		if ( inserterOpen && inserterOpen.length ) {
 			// eslint-disable-next-line no-console
 			console.log( `
-${ title( 'Opening Global Inserter Performance:' ) }
+${ title( 'Opening Global Inserter:' ) }
 Average time to open global inserter: ${ success(
 				round( average( inserterOpen ) ) + 'ms'
 			) }
@@ -124,7 +140,7 @@ Fastest time to open global inserter: ${ success(
 		if ( inserterSearch && inserterSearch.length ) {
 			// eslint-disable-next-line no-console
 			console.log( `
-${ title( 'Inserter Search Performance:' ) }
+${ title( 'Inserter Search:' ) }
 Average time to type the inserter search input: ${ success(
 				round( average( inserterSearch ) ) + 'ms'
 			) }
@@ -139,7 +155,7 @@ Fastest time to type the inserter search input: ${ success(
 		if ( inserterHover && inserterHover.length ) {
 			// eslint-disable-next-line no-console
 			console.log( `
-${ title( 'Inserter Block Item Hover Performance:' ) }
+${ title( 'Inserter Block Item Hover:' ) }
 Average time to move mouse between two block item in the inserter: ${ success(
 				round( average( inserterHover ) ) + 'ms'
 			) }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@wordpress/e2e-tests",
-	"version": "6.0.0",
+	"version": "6.2.0",
 	"description": "End-To-End (E2E) tests for WordPress.",
 	"author": "The WordPress Contributors",
 	"license": "GPL-2.0-or-later",
@@ -23,11 +23,11 @@
 		"node": ">=14"
 	},
 	"dependencies": {
-		"@wordpress/e2e-test-utils": "^9.0.0",
-		"@wordpress/jest-console": "^6.6.0",
-		"@wordpress/jest-puppeteer-axe": "^5.6.0",
-		"@wordpress/scripts": "^25.0.0",
-		"@wordpress/url": "^3.24.0",
+		"@wordpress/e2e-test-utils": "^9.2.0",
+		"@wordpress/jest-console": "^6.8.0",
+		"@wordpress/jest-puppeteer-axe": "^5.8.0",
+		"@wordpress/scripts": "^25.2.0",
+		"@wordpress/url": "^3.26.0",
 		"chalk": "^4.0.0",
 		"expect-puppeteer": "^4.4.0",
 		"filenamify": "^4.2.0",
@@ -46,5 +46,5 @@
 	"publishConfig": {
 		"access": "public"
 	},
-	"gitHead": "1eb65aabe6738097f4c062e78f69ae8f05879848"
+	"gitHead": "204c880ff65295768e9695dfee6c7a9fee1fdd05"
 }

package/specs/editor/plugins/block-directory-add.test.js

@@ -166,7 +166,7 @@ const matchUrl = ( reqUrl, urls ) => {
 };
 
 describe( 'adding blocks from block directory', () => {
-	beforeEach( async () => {
+	beforeAll( async () => {
 		await createNewPost();
 	} );
 

package/specs/editor/various/a11y.test.js

@@ -16,7 +16,9 @@ describe( 'a11y', () => {
 
 	it( 'tabs header bar', async () => {
 		await pressKeyWithModifier( 'ctrl', '`' );
-
+		await pressKeyWithModifier( 'ctrl', '`' );
+		await pressKeyWithModifier( 'ctrl', '`' );
+		await pressKeyWithModifier( 'ctrl', '`' );
 		await page.keyboard.press( 'Tab' );
 
 		const isFocusedToggle = await page.$eval(

package/specs/editor/various/block-hierarchy-navigation.test.js

@@ -9,6 +9,7 @@ import {
 	pressKeyWithModifier,
 	openDocumentSettingsSidebar,
 	getListViewBlocks,
+	switchBlockInspectorTab,
 } from '@wordpress/e2e-test-utils';
 
 async function openListViewSidebar() {
@@ -20,6 +21,20 @@ async function tabToColumnsControl() {
 	let isColumnsControl = false;
 	do {
 		await page.keyboard.press( 'Tab' );
+
+		const isBlockInspectorTab = await page.evaluate( () => {
+			const activeElement = document.activeElement;
+			return (
+				activeElement.getAttribute( 'role' ) === 'tab' &&
+				activeElement.attributes.getNamedItem( 'aria-label' ).value ===
+					'Styles'
+			);
+		} );
+
+		if ( isBlockInspectorTab ) {
+			await page.keyboard.press( 'ArrowRight' );
+		}
+
 		isColumnsControl = await page.evaluate( () => {
 			const activeElement = document.activeElement;
 			return (
@@ -60,6 +75,7 @@ describe( 'Navigating the block hierarchy', () => {
 
 		// Tweak the columns count.
 		await openDocumentSettingsSidebar();
+		await switchBlockInspectorTab( 'Settings' );
 		await page.focus(
 			'.block-editor-block-inspector [aria-label="Columns"][type="number"]'
 		);
@@ -111,16 +127,14 @@ describe( 'Navigating the block hierarchy', () => {
 
 		// Move focus to the sidebar area.
 		await pressKeyWithModifier( 'ctrl', '`' );
-		await pressKeyWithModifier( 'ctrl', '`' );
-		await pressKeyWithModifier( 'ctrl', '`' );
 		await tabToColumnsControl();
 
 		// Tweak the columns count by increasing it by one.
 		await page.keyboard.press( 'ArrowRight' );
 
 		// Navigate to the third column in the columns block.
-		await pressKeyWithModifier( 'ctrl', '`' );
-		await pressKeyWithModifier( 'ctrl', '`' );
+		await pressKeyWithModifier( 'ctrlShift', '`' );
+		await pressKeyWithModifier( 'ctrlShift', '`' );
 		await pressKeyTimes( 'Tab', 4 );
 		await pressKeyTimes( 'ArrowDown', 4 );
 		await page.waitForSelector(

package/specs/editor/various/block-switcher.test.js

@@ -76,7 +76,7 @@ describe( 'Block Switcher', () => {
 				'core/group',
 				'core/heading',
 				'core/columns',
-			].map( ( block ) => wp.blocks.unregisterBlockType( block ) );
+			].forEach( ( block ) => wp.blocks.unregisterBlockType( block ) );
 		} );
 
 		await page.keyboard.press( 'ArrowUp' );

package/specs/editor/various/editor-modes.test.js

@@ -22,9 +22,7 @@ describe( 'Editing modes (visual/HTML)', () => {
 
 	it( 'should switch between visual and HTML modes', async () => {
 		// This block should be in "visual" mode by default.
-		let visualBlock = await page.$$(
-			'.block-editor-block-list__layout .block-editor-block-list__block.rich-text'
-		);
+		let visualBlock = await page.$$( '[data-block].rich-text' );
 		expect( visualBlock ).toHaveLength( 1 );
 
 		// Change editing mode from "Visual" to "HTML".
@@ -33,7 +31,7 @@ describe( 'Editing modes (visual/HTML)', () => {
 
 		// Wait for the block to be converted to HTML editing mode.
 		const htmlBlock = await page.$$(
-			'.block-editor-block-list__layout .block-editor-block-list__block .block-editor-block-list__block-html-textarea'
+			'[data-block] .block-editor-block-list__block-html-textarea'
 		);
 		expect( htmlBlock ).toHaveLength( 1 );
 
@@ -42,9 +40,7 @@ describe( 'Editing modes (visual/HTML)', () => {
 		await clickMenuItem( 'Edit visually' );
 
 		// This block should be in "visual" mode by default.
-		visualBlock = await page.$$(
-			'.block-editor-block-list__layout .block-editor-block-list__block.rich-text'
-		);
+		visualBlock = await page.$$( '[data-block].rich-text' );
 		expect( visualBlock ).toHaveLength( 1 );
 	} );
 

package/specs/editor/various/inserting-blocks.test.js

@@ -318,7 +318,7 @@ describe( 'Inserting blocks', () => {
 		);
 		await browseAll.click();
 		const availableBlocks = await page.$$(
-			'.block-editor-block-types-list__list-item'
+			'.edit-post-editor__inserter-panel .block-editor-block-types-list__list-item'
 		);
 		expect( availableBlocks ).toHaveLength( 1 );
 	} );

package/specs/editor/various/keyboard-navigable-blocks.test.js

@@ -23,6 +23,9 @@ const navigateToContentEditorTop = async () => {
 	// Use 'Ctrl+`' to return to the top of the editor.
 	await pressKeyWithModifier( 'ctrl', '`' );
 	await pressKeyWithModifier( 'ctrl', '`' );
+	await pressKeyWithModifier( 'ctrl', '`' );
+	await pressKeyWithModifier( 'ctrl', '`' );
+	await pressKeyWithModifier( 'ctrl', '`' );
 };
 
 const tabThroughParagraphBlock = async ( paragraphText ) => {
@@ -36,6 +39,10 @@ const tabThroughParagraphBlock = async ( paragraphText ) => {
 
 	await page.keyboard.press( 'Tab' );
 	await expect( await getActiveLabel() ).toBe( 'Open document settings' );
+
+	// Need to shift+tab here to end back in the block. If not, we'll be in the next region and it will only require 4 region jumps instead of 5.
+	await pressKeyWithModifier( 'shift', 'Tab' );
+	await expect( await getActiveLabel() ).toBe( 'Paragraph block' );
 };
 
 const tabThroughBlockToolbar = async () => {

package/specs/editor/various/sidebar.test.js

@@ -96,8 +96,6 @@ describe( 'Sidebar', () => {
 
 		// Region navigate to Sidebar.
 		await pressKeyWithModifier( 'ctrl', '`' );
-		await pressKeyWithModifier( 'ctrl', '`' );
-		await pressKeyWithModifier( 'ctrl', '`' );
 
 		// Tab lands at first (presumed selected) option "Post".
 		await page.keyboard.press( 'Tab' );

package/specs/performance/post-editor.test.js

@@ -15,6 +15,7 @@ import {
 	closeGlobalBlockInserter,
 	openListView,
 	closeListView,
+	canvas,
 } from '@wordpress/e2e-test-utils';
 
 /**
@@ -32,6 +33,34 @@ import {
 
 jest.setTimeout( 1000000 );
 
+async function loadHtmlIntoTheBlockEditor( html ) {
+	await page.evaluate( ( _html ) => {
+		const { parse } = window.wp.blocks;
+		const { dispatch } = window.wp.data;
+		const blocks = parse( _html );
+
+		blocks.forEach( ( block ) => {
+			if ( block.name === 'core/image' ) {
+				delete block.attributes.id;
+				delete block.attributes.url;
+			}
+		} );
+
+		dispatch( 'core/block-editor' ).resetBlocks( blocks );
+	}, html );
+}
+
+async function load1000Paragraphs() {
+	await page.evaluate( () => {
+		const { createBlock } = window.wp.blocks;
+		const { dispatch } = window.wp.data;
+		const blocks = Array.from( { length: 1000 } ).map( () =>
+			createBlock( 'core/paragraph' )
+		);
+		dispatch( 'core/block-editor' ).resetBlocks( blocks );
+	} );
+}
+
 describe( 'Post Editor Performance', () => {
 	const results = {
 		serverResponse: [],
@@ -41,6 +70,7 @@ describe( 'Post Editor Performance', () => {
 		firstContentfulPaint: [],
 		firstBlock: [],
 		type: [],
+		typeContainer: [],
 		focus: [],
 		listViewOpen: [],
 		inserterOpen: [],
@@ -50,29 +80,6 @@ describe( 'Post Editor Performance', () => {
 	const traceFile = __dirname + '/trace.json';
 	let traceResults;
 
-	beforeAll( async () => {
-		const html = readFile(
-			join( __dirname, '../../assets/large-post.html' )
-		);
-
-		await createNewPost();
-		await page.evaluate( ( _html ) => {
-			const { parse } = window.wp.blocks;
-			const { dispatch } = window.wp.data;
-			const blocks = parse( _html );
-
-			blocks.forEach( ( block ) => {
-				if ( block.name === 'core/image' ) {
-					delete block.attributes.id;
-					delete block.attributes.url;
-				}
-			} );
-
-			dispatch( 'core/block-editor' ).resetBlocks( blocks );
-		}, html );
-		await saveDraft();
-	} );
-
 	afterAll( async () => {
 		const resultsFilename = basename( __filename, '.js' ) + '.results.json';
 		writeFileSync(
@@ -83,6 +90,7 @@ describe( 'Post Editor Performance', () => {
 	} );
 
 	beforeEach( async () => {
+		await createNewPost();
 		// Disable auto-save to avoid impacting the metrics.
 		await page.evaluate( () => {
 			window.wp.data.dispatch( 'core/editor' ).updateEditorSettings( {
@@ -93,11 +101,17 @@ describe( 'Post Editor Performance', () => {
 	} );
 
 	it( 'Loading', async () => {
-		// Measuring loading time.
+		await loadHtmlIntoTheBlockEditor(
+			readFile( join( __dirname, '../../assets/large-post.html' ) )
+		);
+		await saveDraft();
 		let i = 5;
 		while ( i-- ) {
 			await page.reload();
-			await page.waitForSelector( '.wp-block' );
+			await page.waitForSelector( '.edit-post-layout', {
+				timeout: 120000,
+			} );
+			await canvas().waitForSelector( '.wp-block', { timeout: 120000 } );
 			const {
 				serverResponse,
 				firstPaint,
@@ -117,7 +131,9 @@ describe( 'Post Editor Performance', () => {
 	} );
 
 	it( 'Typing', async () => {
-		// Measuring typing performance.
+		await loadHtmlIntoTheBlockEditor(
+			readFile( join( __dirname, '../../assets/large-post.html' ) )
+		);
 		await insertBlock( 'Paragraph' );
 		let i = 20;
 		await page.tracing.start( {
@@ -151,18 +167,59 @@ describe( 'Post Editor Performance', () => {
 		}
 	} );
 
-	it( 'Selecting blocks', async () => {
-		// Measuring block selection performance.
-		await createNewPost();
-		await page.evaluate( () => {
-			const { createBlock } = window.wp.blocks;
-			const { dispatch } = window.wp.data;
-			const blocks = Array.from( { length: 1000 } ).map( () =>
-				createBlock( 'core/paragraph' )
-			);
-			dispatch( 'core/block-editor' ).resetBlocks( blocks );
+	it( 'Typing within containers', async () => {
+		await loadHtmlIntoTheBlockEditor(
+			readFile(
+				join(
+					__dirname,
+					'../../assets/small-post-with-containers.html'
+				)
+			)
+		);
+		// Select the block where we type in
+		await canvas().waitForSelector( 'p[aria-label="Paragraph block"]' );
+		await canvas().click( 'p[aria-label="Paragraph block"]' );
+		// Ignore firsted typed character because it's different
+		// It probably deserves a dedicated metric.
+		// (isTyping triggers so it's slower)
+		await page.keyboard.type( 'x' );
+
+		let i = 10;
+		await page.tracing.start( {
+			path: traceFile,
+			screenshots: false,
+			categories: [ 'devtools.timeline' ],
 		} );
-		const paragraphs = await page.$$( '.wp-block' );
+
+		while ( i-- ) {
+			// Wait for the browser to be idle before starting the monitoring.
+			// eslint-disable-next-line no-restricted-syntax
+			await page.waitForTimeout( 500 );
+			await page.keyboard.type( 'x' );
+		}
+		// eslint-disable-next-line no-restricted-syntax
+		await page.waitForTimeout( 500 );
+		await page.tracing.stop();
+		traceResults = JSON.parse( readFile( traceFile ) );
+		const [ keyDownEvents, keyPressEvents, keyUpEvents ] =
+			getTypingEventDurations( traceResults );
+		if (
+			keyDownEvents.length === keyPressEvents.length &&
+			keyPressEvents.length === keyUpEvents.length
+		) {
+			// The first character typed triggers a longer time (isTyping change)
+			// It can impact the stability of the metric, so we exclude it.
+			for ( let j = 1; j < keyDownEvents.length; j++ ) {
+				results.typeContainer.push(
+					keyDownEvents[ j ] + keyPressEvents[ j ] + keyUpEvents[ j ]
+				);
+			}
+		}
+	} );
+
+	it( 'Selecting blocks', async () => {
+		await load1000Paragraphs();
+		const paragraphs = await canvas().$$( '.wp-block' );
 		await page.tracing.start( {
 			path: traceFile,
 			screenshots: false,
@@ -182,8 +239,7 @@ describe( 'Post Editor Performance', () => {
 	} );
 
 	it( 'Opening persistent list view', async () => {
-		// Measure time to open inserter.
-		await page.waitForSelector( '.edit-post-layout' );
+		await load1000Paragraphs();
 		for ( let j = 0; j < 10; j++ ) {
 			await page.tracing.start( {
 				path: traceFile,
@@ -202,8 +258,7 @@ describe( 'Post Editor Performance', () => {
 	} );
 
 	it( 'Opening the inserter', async () => {
-		// Measure time to open inserter.
-		await page.waitForSelector( '.edit-post-layout' );
+		await load1000Paragraphs();
 		for ( let j = 0; j < 10; j++ ) {
 			await page.tracing.start( {
 				path: traceFile,
@@ -225,8 +280,7 @@ describe( 'Post Editor Performance', () => {
 		function sum( arr ) {
 			return arr.reduce( ( a, b ) => a + b, 0 );
 		}
-
-		// Measure time to search the inserter and get results.
+		await load1000Paragraphs();
 		await openGlobalBlockInserter();
 		for ( let j = 0; j < 10; j++ ) {
 			// Wait for the browser to be idle before starting the monitoring.
@@ -258,7 +312,7 @@ describe( 'Post Editor Performance', () => {
 	} );
 
 	it( 'Hovering Inserter Items', async () => {
-		// Measure inserter hover performance.
+		await load1000Paragraphs();
 		const paragraphBlockItem =
 			'.block-editor-inserter__menu .editor-block-list-item-paragraph';
 		const headingBlockItem =

package/specs/performance/site-editor.test.js

@@ -51,6 +51,7 @@ describe( 'Site Editor Performance', () => {
 			firstContentfulPaint: [],
 			firstBlock: [],
 			type: [],
+			typeContainer: [],
 			focus: [],
 			inserterOpen: [],
 			inserterHover: [],