@ckeditor/ckeditor5-dev-docs 31.1.13 → 32.0.0

package/lib/index.js

@@ -5,7 +5,8 @@
 
 'use strict';
 
+const build = require( './build' );
+
 module.exports = {
-	build: require( './docs/index' ),
-	verify: require( './web-crawler/index' )
+	build
 };

package/package.json

@@ -1,18 +1,14 @@
 {
   "name": "@ckeditor/ckeditor5-dev-docs",
-  "version": "31.1.13",
+  "version": "32.0.0",
   "description": "Tasks used to build and verify the documentation for CKEditor 5.",
   "keywords": [],
   "main": "lib/index.js",
   "dependencies": {
-    "@ckeditor/ckeditor5-dev-utils": "^31.1.13",
-    "@ckeditor/jsdoc-plugins": "^31.1.13",
+    "@ckeditor/ckeditor5-dev-utils": "^32.0.0",
+    "@ckeditor/jsdoc-plugins": "^32.0.0",
     "fast-glob": "^3.2.4",
     "fs-extra": "^9.0.0",
-    "puppeteer": "^13.1.3",
-    "chalk": "^4.1.0",
-    "strip-ansi": "^6.0.0",
-    "ora": "^5.2.0",
     "jsdoc": "ckeditor/jsdoc#fixed-trailing-comment-doclets",
     "tmp": "^0.2.1"
   },

package/lib/web-crawler/constants.js

@@ -1,71 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * @license Copyright (c) 2003-2022, CKSource Holding sp. z o.o. All rights reserved.
- * For licensing, see LICENSE.md.
- */
-
-/* eslint-env node */
-
-const DEFAULT_CONCURRENCY = require( 'os' ).cpus().length / 2;
-
-const DEFAULT_TIMEOUT = 15 * 1000;
-
-const DEFAULT_RESPONSIVENESS_CHECK_TIMEOUT = 1000;
-
-const DEFAULT_REMAINING_ATTEMPTS = 3;
-
-const ERROR_TYPES = {
-	PAGE_CRASH: {
-		event: 'error',
-		description: 'Page crash'
-	},
-	UNCAUGHT_EXCEPTION: {
-		event: 'pageerror',
-		description: 'Uncaught exception'
-	},
-	REQUEST_FAILURE: {
-		event: 'requestfailed',
-		description: 'Request failure'
-	},
-	RESPONSE_FAILURE: {
-		event: 'response',
-		description: 'Response failure'
-	},
-	CONSOLE_ERROR: {
-		event: 'console',
-		description: 'Console error'
-	},
-	NAVIGATION_ERROR: {
-		// Navigation error does not have the `event` property, because this error is not emitted by page.on() method as
-		// event, but it is thrown as exception from page.goto() method.
-		description: 'Navigation error'
-	}
-};
-
-const PATTERN_TYPE_TO_ERROR_TYPE_MAP = {
-	'page-crash': ERROR_TYPES.PAGE_CRASH,
-	'uncaught-exception': ERROR_TYPES.UNCAUGHT_EXCEPTION,
-	'request-failure': ERROR_TYPES.REQUEST_FAILURE,
-	'response-failure': ERROR_TYPES.RESPONSE_FAILURE,
-	'console-error': ERROR_TYPES.CONSOLE_ERROR,
-	'navigation-error': ERROR_TYPES.NAVIGATION_ERROR
-};
-
-const IGNORE_ALL_ERRORS_WILDCARD = '*';
-
-const META_TAG_NAME = 'x-cke-crawler-ignore-patterns';
-
-const DATA_ATTRIBUTE_NAME = 'data-cke-crawler-skip';
-
-module.exports = {
-	DEFAULT_CONCURRENCY,
-	DEFAULT_TIMEOUT,
-	DEFAULT_RESPONSIVENESS_CHECK_TIMEOUT,
-	DEFAULT_REMAINING_ATTEMPTS,
-	ERROR_TYPES,
-	PATTERN_TYPE_TO_ERROR_TYPE_MAP,
-	IGNORE_ALL_ERRORS_WILDCARD,
-	META_TAG_NAME,
-	DATA_ATTRIBUTE_NAME
-};

package/lib/web-crawler/index.js

@@ -1,729 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * @license Copyright (c) 2003-2022, CKSource Holding sp. z o.o. All rights reserved.
- * For licensing, see LICENSE.md.
- */
-
-/* eslint-env node */
-
-const puppeteer = require( 'puppeteer' );
-const chalk = require( 'chalk' );
-const util = require( 'util' );
-const stripAnsiEscapeCodes = require( 'strip-ansi' );
-const { getBaseUrl, toArray } = require( './utils' );
-const { createSpinner, getProgressHandler } = require( './spinner' );
-
-const {
-	DEFAULT_TIMEOUT,
-	DEFAULT_RESPONSIVENESS_CHECK_TIMEOUT,
-	DEFAULT_REMAINING_ATTEMPTS,
-	ERROR_TYPES,
-	PATTERN_TYPE_TO_ERROR_TYPE_MAP,
-	IGNORE_ALL_ERRORS_WILDCARD,
-	META_TAG_NAME,
-	DATA_ATTRIBUTE_NAME
-} = require( './constants' );
-
-/**
- * Main crawler function. Its purpose is to:
- * - create Puppeteer's browser instance,
- * - open simultaneously (up to concurrency limit) links from the provided URL in a dedicated Puppeteer's page for each link,
- * - show error summary after all links have been visited.
- *
- * @param {Object} options Parsed CLI arguments.
- * @param {String} options.url The URL to start crawling. This argument is required.
- * @param {Number} [options.depth=Infinity] Defines how many nested page levels should be examined. Infinity by default.
- * @param {Array.<String>} [options.exclusions=[]] An array of patterns to exclude links. Empty array by default to not exclude anything.
- * @param {Number} [options.concurrency=1] Number of concurrent pages (browser tabs) to be used during crawling. One by default.
- * @param {Boolean} [options.quit=false] Terminates the scan as soon as an error is found. False (off) by default.
- * @param {Boolean} [options.disableBrowserSandbox=false] Whether the browser should be created with the `--no-sandbox` flag.
- * @param {Boolean} [options.noSpinner=false] Whether to display the spinner with progress or a raw message with current progress.
- * @param {Boolean} [options.ignoreHTTPSErrors=false] Whether the browser should ignore invalid (self-signed) certificates.
- * @returns {Promise} Promise is resolved, when the crawler has finished the whole crawling procedure.
- */
-module.exports = async function verify( options ) {
-	const {
-		url,
-		depth = Infinity,
-		exclusions = [],
-		concurrency = 1,
-		quit = false,
-		disableBrowserSandbox = false,
-		noSpinner = false,
-		ignoreHTTPSErrors = false
-	} = options;
-
-	console.log( chalk.bold( '\n🔎 Starting the Crawler\n' ) );
-
-	process.on( 'unhandledRejection', reason => {
-		const error = util.inspect( reason, {
-			breakLength: Infinity,
-			compact: true
-		} );
-
-		console.log( chalk.red.bold( `\n🔥 Caught the \`unhandledRejection\` error: ${ error }\n` ) );
-
-		process.exit( 1 );
-	} );
-
-	const spinner = createSpinner( { noSpinner } );
-	const errors = new Map();
-	const browser = await createBrowser( { disableBrowserSandbox, ignoreHTTPSErrors } );
-
-	spinner.start( 'Checking pages…' );
-
-	let status = 'Done';
-
-	await openLinks( browser, {
-		baseUrl: getBaseUrl( url ),
-		linksQueue: [ {
-			url,
-			parentUrl: '(none)',
-			remainingNestedLevels: depth,
-			remainingAttempts: DEFAULT_REMAINING_ATTEMPTS
-		} ],
-		foundLinks: [ url ],
-		exclusions,
-		concurrency,
-		quit,
-		onError: getErrorHandler( errors ),
-		onProgress: getProgressHandler( spinner, { verbose: noSpinner } )
-	} ).catch( () => {
-		status = 'Terminated on first error';
-	} );
-
-	spinner.succeed( `Checking pages… ${ chalk.bold( status ) }` );
-
-	await browser.close();
-
-	logErrors( errors );
-
-	// Always exit the script because `spinner` can freeze the process of the crawler if it is executed in the `noSpinner:true` mode.
-	process.exit( errors.size ? 1 : 0 );
-};
-
-/**
- * Creates a new browser instance and closes the default blank page.
- *
- * @param {Object} options
- * @param {Boolean} [options.disableBrowserSandbox] Whether the browser should be created with the `--no-sandbox` flag.
- * @param {Boolean} [options.ignoreHTTPSErrors] Whether the browser should ignore invalid (self-signed) certificates.
- *
- * @returns {Promise.<Object>} A promise, which resolves to the Puppeteer browser instance.
- */
-async function createBrowser( options ) {
-	const browserOptions = {
-		args: []
-	};
-
-	if ( options.disableBrowserSandbox ) {
-		browserOptions.args.push( '--no-sandbox' );
-		browserOptions.args.push( '--disable-setuid-sandbox' );
-	}
-
-	if ( options.ignoreHTTPSErrors ) {
-		browserOptions.ignoreHTTPSErrors = options.ignoreHTTPSErrors;
-	}
-
-	const browser = await puppeteer.launch( browserOptions );
-
-	const [ defaultBlankPage ] = await browser.pages();
-
-	if ( defaultBlankPage ) {
-		await defaultBlankPage.close();
-	}
-
-	return browser;
-}
-
-/**
- * Returns an error handler, which is called every time new error is found.
- *
- * @param {Map.<ErrorType, ErrorCollection>} errors All errors grouped by their type.
- * @returns {Function} Error handler.
- */
-function getErrorHandler( errors ) {
-	return error => {
-		if ( !errors.has( error.type ) ) {
-			errors.set( error.type, new Map() );
-		}
-
-		// Split the message into the first line and all the rest. The first line is the key by which the errors are grouped together.
-		// All errors are grouped together only by the first message line (without the error call stack and other details, that could
-		// possibly exist after the first line), because there is a good chance that the same error can be triggered in a different
-		// contexts (so in a different call stacks). In order not to duplicate almost the same errors, we need to determine their common
-		// part.
-		const messageLines = error.message.split( '\n' );
-		const firstMessageLine = messageLines.shift();
-		const nextMessageLines = messageLines.join( '\n' );
-
-		const errorCollection = errors.get( error.type );
-
-		if ( !errorCollection.has( firstMessageLine ) ) {
-			errorCollection.set( firstMessageLine, {
-				// Store only unique pages, because given error can occur multiple times on the same page.
-				pages: new Set(),
-				details: nextMessageLines
-			} );
-		}
-
-		errorCollection.get( firstMessageLine ).pages.add( error.pageUrl );
-	};
-}
-
-/**
- * Searches and opens all found links in the document body from requested URL, recursively.
- *
- * @param {Object} browser The headless browser instance from Puppeteer.
- * @param {Object} data All data needed for crawling the links.
- * @param {String} data.baseUrl The base URL from the initial page URL.
- * @param {Array.<Link>} data.linksQueue An array of link to crawl.
- * @param {Array.<String>} data.foundLinks An array of all links, which have been already discovered.
- * @param {Array.<String>} data.exclusions An array of patterns to exclude links. Empty array by default to not exclude anything.
- * @param {Number} data.concurrency Number of concurrent pages (browser tabs) to be used during crawling.
- * @param {Boolean} data.quit Terminates the scan as soon as an error is found.
- * @param {Function} data.onError Callback called ever time an error has been found.
- * @param {Function} data.onProgress Callback called every time just before opening a new link.
- * @returns {Promise} Promise is resolved, when all links have been visited.
- */
-async function openLinks( browser, { baseUrl, linksQueue, foundLinks, exclusions, concurrency, quit, onError, onProgress } ) {
-	const numberOfOpenPages = ( await browser.pages() ).length;
-
-	// Check if the limit of simultaneously opened pages in the browser has been reached.
-	if ( numberOfOpenPages >= concurrency ) {
-		return;
-	}
-
-	return Promise.all(
-		linksQueue
-			// Get links from the queue, up to the concurrency limit...
-			.splice( 0, concurrency - numberOfOpenPages )
-			// ...and open each of them in a dedicated page to collect nested links and errors (if any) they contain.
-			.map( async link => {
-				let newErrors = [];
-				let newLinks = [];
-
-				onProgress( {
-					total: foundLinks.length
-				} );
-
-				// If opening a given link causes an error, try opening it again until the limit of remaining attempts is reached.
-				do {
-					const { errors, links } = await openLink( browser, { baseUrl, link, foundLinks, exclusions } );
-
-					link.remainingAttempts--;
-
-					newErrors = [ ...errors ];
-					newLinks = [ ...links ];
-				} while ( newErrors.length && link.remainingAttempts );
-
-				newErrors.forEach( newError => onError( newError ) );
-
-				newLinks.forEach( newLink => {
-					foundLinks.push( newLink );
-
-					linksQueue.push( {
-						url: newLink,
-						parentUrl: link.url,
-						remainingNestedLevels: link.remainingNestedLevels - 1,
-						remainingAttempts: DEFAULT_REMAINING_ATTEMPTS
-					} );
-				} );
-
-				// Terminate the scan as soon as an error is found, if `--quit` or `-q` CLI argument has been set.
-				if ( newErrors.length > 0 && quit ) {
-					return Promise.reject();
-				}
-
-				// When currently examined link has been checked, try to open new links up to the concurrency limit.
-				return openLinks( browser, { baseUrl, linksQueue, foundLinks, exclusions, concurrency, quit, onError, onProgress } );
-			} )
-	);
-}
-
-/**
- * Creates a dedicated Puppeteer's page for URL to be tested and collects all links from it. Only links from the same base URL
- * as the tested URL are collected. Only the base URL part consisting of a protocol, a host, a port, and a path is stored, without
- * a hash and search parts. Duplicated links, which were already found and enqueued, are skipped to avoid loops. Explicitly
- * excluded links are also skipped. If the requested traversing depth has been reached, nested links from this URL are not collected
- * anymore.
- *
- * @param {Object} browser The headless browser instance from Puppeteer.
- * @param {Object} data All data needed for crawling the link.
- * @param {String} data.baseUrl The base URL from the initial page URL.
- * @param {Link} data.link A link to crawl.
- * @param {Array.<String>} data.foundLinks An array of all links, which have been already discovered.
- * @param {Array.<String>} data.exclusions An array of patterns to exclude links. Empty array by default to not exclude anything.
- * @returns {Promise.<ErrorsAndLinks>} A promise, which resolves to a collection of unique errors and links.
- */
-async function openLink( browser, { baseUrl, link, foundLinks, exclusions } ) {
-	const errors = [];
-
-	const onError = error => errors.push( error );
-
-	// Create dedicated page for current link.
-	const page = await createPage( browser, { link, onError } );
-
-	try {
-		// Consider navigation to be finished when the `load` event is fired and there are no network connections for at least 500 ms.
-		await page.goto( link.url, { waitUntil: [ 'load', 'networkidle0' ] } );
-	} catch ( error ) {
-		const errorMessage = error.message || '(empty message)';
-
-		// All navigation errors starting with the `net::` prefix are already covered by the "request" error handler, so it should
-		// not be also reported as the "navigation error".
-		const ignoredMessage = 'net::';
-
-		if ( !errorMessage.startsWith( ignoredMessage ) ) {
-			onError( {
-				pageUrl: link.url,
-				type: ERROR_TYPES.NAVIGATION_ERROR,
-				message: errorMessage
-			} );
-		}
-
-		const isResponding = await isPageResponding( page );
-
-		// Exit immediately and do not try to call any function in the context of the page, that is not responding or if it has not been
-		// opened. However, once the page has been opened (its URL is the same as the one requested), continue as usual and do not close
-		// the page yet, because the page may contain error exclusions, that should be taken into account. Such a case can happen when,
-		// for example, the `load` event was not fired because the external resource was not loaded yet.
-		if ( !isResponding || page.url() !== link.url ) {
-			page.removeAllListeners();
-
-			await page.close();
-
-			return {
-				errors,
-				links: []
-			};
-		}
-	}
-
-	// Create patterns from meta tags to ignore errors.
-	const errorIgnorePatterns = await getErrorIgnorePatternsFromPage( page );
-
-	// Iterates over recently found errors to mark them as ignored ones, if they match the patterns.
-	markErrorsAsIgnored( errors, errorIgnorePatterns );
-
-	// Skip crawling deeper, if the bottom has been reached, or get all unique links from the page body otherwise.
-	const links = link.remainingNestedLevels === 0 ?
-		[] :
-		await getLinksFromPage( page, { baseUrl, foundLinks, exclusions } );
-
-	page.removeAllListeners();
-
-	await page.close();
-
-	return {
-		errors: errors.filter( error => !error.ignored ),
-		links
-	};
-}
-
-/**
- * Finds all links in opened page and filters out external, already discovered and exlicitly excluded ones.
- *
- * @param {Object} page The page instance from Puppeteer.
- * @param {Object} data All data needed for crawling the link.
- * @param {String} data.baseUrl The base URL from the initial page URL.
- * @param {Array.<String>} data.foundLinks An array of all links, which have been already discovered.
- * @param {Array.<String>} data.exclusions An array patterns to exclude links. Empty array by default to not exclude anything.
- * @returns {Promise.<Array.<String>>} A promise, which resolves to an array of unique links.
- */
-async function getLinksFromPage( page, { baseUrl, foundLinks, exclusions } ) {
-	const evaluatePage = anchors => [ ...new Set( anchors
-		.filter( anchor => /http(s)?:/.test( anchor.protocol ) )
-		.map( anchor => `${ anchor.origin }${ anchor.pathname }` ) )
-	];
-
-	return ( await page.$$eval( `body a[href]:not([${ DATA_ATTRIBUTE_NAME }])`, evaluatePage ) )
-		.filter( link => {
-			// Skip external link.
-			if ( !link.startsWith( baseUrl ) ) {
-				return false;
-			}
-
-			// Skip already discovered link.
-			if ( foundLinks.includes( link ) ) {
-				return false;
-			}
-
-			// Skip explicitly excluded link.
-			if ( exclusions.some( exclusion => link.includes( exclusion ) ) ) {
-				return false;
-			}
-
-			return true;
-		} );
-}
-
-/**
- * Finds all meta tags, that contain a pattern to ignore errors, and then returns a map between error type and these patterns.
- *
- * @param {Object} page The page instance from Puppeteer.
- * @returns {Promise.<Map.<ErrorType, Set.<String>>>} A promise, which resolves to a map between an error type and a set of patterns.
- */
-async function getErrorIgnorePatternsFromPage( page ) {
-	const metaTag = await page.$( `head > meta[name=${ META_TAG_NAME }]` );
-
-	const patterns = new Map();
-
-	// If meta tag is not defined, return an empty map.
-	if ( !metaTag ) {
-		return patterns;
-	}
-
-	const contentString = await metaTag.evaluate( metaTag => metaTag.getAttribute( 'content' ) );
-
-	let content;
-
-	try {
-		// Try to parse value from meta tag...
-		content = JSON.parse( contentString );
-	} catch ( error ) {
-		// ...but if it is not a valid JSON, return an empty map.
-		return patterns;
-	}
-
-	Object.entries( content ).forEach( ( [ type, pattern ] ) => {
-		const patternCollection = new Set( toArray( pattern )
-			// Only string patterns are supported, as the error message produced by the crawler is always a string.
-			.filter( pattern => typeof pattern === 'string' )
-			// Only non-empty patterns are supported, because an empty pattern would cause all errors in a given type to be ignored.
-			.filter( pattern => pattern.length > 0 )
-		);
-
-		if ( !patternCollection.size ) {
-			return;
-		}
-
-		const errorType = PATTERN_TYPE_TO_ERROR_TYPE_MAP[ type ];
-
-		patterns.set( errorType, patternCollection );
-	} );
-
-	return patterns;
-}
-
-/**
- * Iterates over all found errors from given link and marks errors as ingored, if their message match the ignore pattern.
- *
- * @param {Array.<Error>} errors An array of errors to check.
- * @param {Map.<ErrorType, Set.<String>>} errorIgnorePatterns A map between an error type and a set of patterns.
- */
-function markErrorsAsIgnored( errors, errorIgnorePatterns ) {
-	errors.forEach( error => {
-		// Skip, if there is no pattern defined for currently examined error type.
-		if ( !errorIgnorePatterns.has( error.type ) ) {
-			return;
-		}
-
-		const patterns = [ ...errorIgnorePatterns.get( error.type ) ];
-
-		const isPatternMatched = pattern => {
-			if ( pattern === IGNORE_ALL_ERRORS_WILDCARD ) {
-				return true;
-			}
-
-			if ( stripAnsiEscapeCodes( error.message ).includes( pattern ) ) {
-				return true;
-			}
-
-			if ( error.failedResourceUrl && error.failedResourceUrl.includes( pattern ) ) {
-				return true;
-			}
-
-			return false;
-		};
-
-		// If at least one pattern matches the error message, mark currently examined error as ignored.
-		if ( patterns.some( isPatternMatched ) ) {
-			error.ignored = true;
-		}
-	} );
-}
-
-/**
- * Creates a new page in Puppeteer's browser instance.
- *
- * @param {Object} browser The headless browser instance from Puppeteer.
- * @param {Object} data All data needed for creating a new page.
- * @param {Link} data.link A link to crawl.
- * @param {Function} data.onError Callback called every time just before opening a new link.
- * @returns {Promise.<Object>} A promise, which resolves to the page instance from Puppeteer.
- */
-async function createPage( browser, { link, onError } ) {
-	const page = await browser.newPage();
-
-	await page.setDefaultTimeout( DEFAULT_TIMEOUT );
-
-	await page.setCacheEnabled( false );
-
-	dismissDialogs( page );
-
-	registerErrorHandlers( page, { link, onError } );
-
-	await registerRequestInterception( page );
-
-	return page;
-}
-
-/**
- * Dismisses any dialogs (alert, prompt, confirm, beforeunload) that could be displayed on page load.
- *
- * @param {Object} page The page instance from Puppeteer.
- */
-function dismissDialogs( page ) {
-	page.on( 'dialog', async dialog => {
-		await dialog.dismiss();
-	} );
-}
-
-/**
- * Registers all error handlers on given page instance.
- *
- * @param {Object} page The page instance from Puppeteer.
- * @param {Object} data All data needed for registering error handlers.
- * @param {Link} data.link A link to crawl associated with Puppeteer's page.
- * @param {Function} data.onError Called each time an error has been found.
- */
-function registerErrorHandlers( page, { link, onError } ) {
-	page.on( ERROR_TYPES.PAGE_CRASH.event, error => onError( {
-		pageUrl: page.url(),
-		type: ERROR_TYPES.PAGE_CRASH,
-		message: error.message || '(empty message)'
-	} ) );
-
-	page.on( ERROR_TYPES.UNCAUGHT_EXCEPTION.event, error => onError( {
-		pageUrl: page.url(),
-		type: ERROR_TYPES.UNCAUGHT_EXCEPTION,
-		message: error.message || '(empty message)'
-	} ) );
-
-	page.on( ERROR_TYPES.REQUEST_FAILURE.event, request => {
-		const errorText = request.failure().errorText;
-
-		// Do not log errors explicitly aborted by the crawler.
-		if ( errorText !== 'net::ERR_BLOCKED_BY_CLIENT.Inspector' ) {
-			const url = request.url();
-			const host = new URL( url ).host;
-			const isNavigation = isNavigationRequest( request );
-			const message = isNavigation ?
-				`Failed to open link ${ chalk.bold( url ) }` :
-				`Failed to load resource from ${ chalk.bold( host ) }`;
-
-			onError( {
-				pageUrl: isNavigation ? link.parentUrl : page.url(),
-				type: ERROR_TYPES.REQUEST_FAILURE,
-				message: `${ message } (failure message: ${ chalk.bold( errorText ) })`,
-				failedResourceUrl: url
-			} );
-		}
-	} );
-
-	page.on( ERROR_TYPES.RESPONSE_FAILURE.event, response => {
-		const responseStatus = response.status();
-
-		if ( responseStatus > 399 ) {
-			const url = response.url();
-			const host = new URL( url ).host;
-			const isNavigation = isNavigationRequest( response.request() );
-			const message = isNavigation ?
-				`Failed to open link ${ chalk.bold( url ) }` :
-				`Failed to load resource from ${ chalk.bold( host ) }`;
-
-			onError( {
-				pageUrl: isNavigation ? link.parentUrl : page.url(),
-				type: ERROR_TYPES.RESPONSE_FAILURE,
-				message: `${ message } (HTTP response status code: ${ chalk.bold( responseStatus ) })`,
-				failedResourceUrl: url
-			} );
-		}
-	} );
-
-	page.on( ERROR_TYPES.CONSOLE_ERROR.event, async message => {
-		// The resource loading failure is already covered by the "request" or "response" error handlers, so it should
-		// not be also reported as the "console error".
-		const ignoredMessage = 'Failed to load resource:';
-
-		if ( message.text().startsWith( ignoredMessage ) ) {
-			return;
-		}
-
-		if ( message.type() !== 'error' ) {
-			return;
-		}
-
-		const serializeArgumentInPageContext = argument => {
-			// Since errors are not serializable, return the message with the call stack as the output text.
-			if ( argument instanceof Error ) {
-				return argument.stack;
-			}
-
-			// Cast non-string iterable argument to an array.
-			if ( typeof argument !== 'string' && argument[ Symbol.iterator ] ) {
-				return [ ...argument ];
-			}
-
-			// Return argument right away. Since we use `executionContext().evaluate()`, it'll return JSON value of the
-			// argument if possible, or `undefined` if it fails to stringify it.
-			return argument;
-		};
-
-		const serializeArguments = argument => argument
-			.executionContext()
-			.evaluate( serializeArgumentInPageContext, argument );
-
-		const serializedArguments = await Promise.all( message.args().map( serializeArguments ) );
-
-		const serializedMessage = serializedArguments
-			.map( argument => {
-				// Do not wrap the string in additional quotes and just return it as is.
-				if ( typeof argument === 'string' ) {
-					return argument;
-				}
-
-				return util.inspect( argument, {
-					breakLength: Infinity,
-					compact: true
-				} );
-			} )
-			.join( ' ' );
-
-		onError( {
-			pageUrl: page.url(),
-			type: ERROR_TYPES.CONSOLE_ERROR,
-			message: serializedMessage || message.text() || '(empty message)'
-		} );
-	} );
-}
-
-/**
- * Checks, if HTTP request was a navigation one, i.e. request that is driving frame's navigation. Requests sent from child frames
- * (i.e. from <iframe>) are not treated as a navigation. Only a request from a top-level frame is navigation.
- *
- * @param {Object} request The Puppeteer's HTTP request instance.
- * @returns {Boolean}
- */
-function isNavigationRequest( request ) {
-	return request.isNavigationRequest() && request.frame().parentFrame() === null;
-}
-
-/**
- * Checks, if the page is not hung by trying to evaluate a function within the page context in defined time.
- *
- * @param {Object} page The page instance from Puppeteer.
- * @returns {Promise.<Boolean>}
- */
-async function isPageResponding( page ) {
-	return Promise.race( [
-		page.title(),
-		new Promise( ( resolve, reject ) => setTimeout( () => reject(), DEFAULT_RESPONSIVENESS_CHECK_TIMEOUT ) )
-	] ).then( () => true ).catch( () => false );
-}
-
-/**
- * Registers a request interception procedure to explicitly block all 'media' requests (resources loaded by a <video> or <audio> elements).
- *
- * @param {Object} page The page instance from Puppeteer.
- * @returns {Promise} Promise is resolved, when the request interception procedure is registered.
- */
-async function registerRequestInterception( page ) {
-	await page.setRequestInterception( true );
-
-	page.on( 'request', request => {
-		const resourceType = request.resourceType();
-
-		// Block all 'media' requests, as they are likely to fail anyway due to limitations in Puppeteer.
-		if ( resourceType === 'media' ) {
-			request.abort( 'blockedbyclient' );
-		} else {
-			request.continue();
-		}
-	} );
-}
-
-/**
- * Analyzes collected errors and logs them in the console.
- *
- * @param {Map.<ErrorType, ErrorCollection>} errors All found errors grouped by their type.
- */
-function logErrors( errors ) {
-	if ( !errors.size ) {
-		console.log( chalk.green.bold( '\n✨ No errors have been found.\n' ) );
-		return;
-	}
-
-	console.log( chalk.red.bold( '\n🔥 The following errors have been found:' ) );
-
-	errors.forEach( ( errorCollection, errorType ) => {
-		const numberOfErrors = errorCollection.size;
-		const separator = chalk.gray( ' ➜ ' );
-		const errorName = chalk.bgRed.white.bold( ` ${ errorType.description.toUpperCase() } ` );
-		const errorSummary = chalk.red( `${ chalk.bold( numberOfErrors ) } ${ numberOfErrors > 1 ? 'errors' : 'error' }` );
-
-		console.group( `\n${ errorName } ${ separator } ${ errorSummary }` );
-
-		errorCollection.forEach( ( error, message ) => {
-			console.group( `\n❌ ${ message }` );
-
-			if ( error.details ) {
-				console.log( error.details );
-			}
-
-			console.log( chalk.red( `\n…found on the following ${ error.pages.size > 1 ? 'pages' : 'page' }:` ) );
-
-			error.pages.forEach( pageUrl => console.log( chalk.gray( `➥  ${ pageUrl }` ) ) );
-
-			console.groupEnd();
-		} );
-
-		console.groupEnd();
-	} );
-
-	// Blank message only to separate the errors output log.
-	console.log();
-}
-
-/**
- * @typedef {Object.<String, String|Number>} Link
- * @property {String} url The URL associated with the link.
- * @property {String} parentUrl The page on which the link was found.
- * @property {Number} remainingNestedLevels The remaining number of nested levels to be checked. If this value is 0, the
- * requested traversing depth has been reached and nested links from the URL associated with this link are not collected anymore.
- * @property {Number} remainingAttempts The total number of reopenings allowed for the given link.
- */
-
-/**
- * @typedef {Object.<String, String>} ErrorType
- * @property {String} [event] The event name emitted by Puppeteer.
- * @property {String} description Human-readable description of the error.
- */
-
-/**
- * @typedef {Object.<String, String|Boolean|ErrorType>} Error
- * @property {String} pageUrl The URL, where error has occurred.
- * @property {ErrorType} type Error type.
- * @property {String} message Error message.
- * @property {String} [failedResourceUrl] Full resource URL, that has failed. Necessary for matching against exclusion patterns.
- * @property {Boolean} [ignored] Indicates that error should be ignored, because its message matches the exclusion pattern.
- */
-
-/**
- * @typedef {Object.<String, Set.<String>>} ErrorOccurrence
- * @property {Set.<String>} pages A set of unique pages, where error has been found.
- * @property {Set.<String>} [details] Additional error details (i.e. an error stack).
- */
-
-/**
- * @typedef {Map.<String, ErrorOccurrence>} ErrorCollection
- * @property {ErrorOccurrence} [*] Error message.
- */
-
-/**
- * @typedef {Object.<String, Array.<String>>} ErrorsAndLinks Collection of unique errors and links.
- * @property {Array.<String>} errors An array of errors.
- * @property {Array.<String>} links An array of links.
- */

package/lib/web-crawler/spinner.js

@@ -1,57 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * @license Copyright (c) 2003-2022, CKSource Holding sp. z o.o. All rights reserved.
- * For licensing, see LICENSE.md.
- */
-
-/* eslint-env node */
-
-const chalk = require( 'chalk' );
-const ora = require( 'ora' );
-
-/**
- * Creates nice-looking CLI spinner.
- * @param {Object} options
- * @param {Boolean} [options.noSpinner=false] Whether to display the spinner with progress or a message with current progress.
- */
-function createSpinner( { noSpinner } ) {
-	return ora( {
-		spinner: {
-			frames: [ '⣾', '⣷', '⣯', '⣟', '⡿', '⢿', '⣻', '⣽' ]
-		},
-		// Do not render the spinner if the `verbose` mode is enabled.
-		isSilent: noSpinner,
-		stream: noSpinner ? process.stdout : process.stderr
-	} );
-}
-
-/**
- * Returns a progress handler, which is called every time just before opening a new link.
- *
- * @param {Object} spinner Spinner instance
- * @param {Object} options
- * @param {Boolean} [options.verbose] Whether to display raw log instead of modifying the spinner instance.
- * @returns {Function} Progress handler.
- */
-function getProgressHandler( spinner, { verbose } ) {
-	let current = 0;
-
-	return ( { total } ) => {
-		current++;
-
-		const progress = Math.round( current / total * 100 );
-		const logMessage = `Checking pages… ${ chalk.bold( `${ progress }% (${ current } of ${ total })` ) }`;
-
-		if ( verbose ) {
-			console.log( logMessage );
-		} else {
-			spinner.text = logMessage;
-		}
-	};
-}
-
-module.exports = {
-	createSpinner,
-	getProgressHandler
-};

package/lib/web-crawler/utils.js

@@ -1,51 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * @license Copyright (c) 2003-2022, CKSource Holding sp. z o.o. All rights reserved.
- * For licensing, see LICENSE.md.
- */
-
-/* eslint-env node */
-
-/**
- * Extracts base URL from the provided page URL. Base URL consists of a protocol, a host, a port, and a path.
- * A hash and search parts are omitted, because they would have navigated to the same page if they were set.
- *
- * @param {String} url Page URL.
- * @returns {String} Base URL from page URL.
- */
-function getBaseUrl( url ) {
-	const { origin, pathname } = new URL( url );
-
-	return `${ origin }${ pathname }`;
-}
-
-/**
- * Checks, if provided string is a valid URL utilizing the HTTP or HTTPS protocols.
- *
- * @param {String} url The URL to validate.
- * @returns {Boolean}
- */
-function isUrlValid( url ) {
-	try {
-		return [ 'http:', 'https:' ].includes( new URL( url ).protocol );
-	} catch ( error ) {
-		return false;
-	}
-}
-
-/**
- * Transforms any value to an array. If the provided value is already an array, it is returned unchanged.
- *
- * @param {*} data The value to transform to an array.
- * @returns {Array.<*>} An array created from data.
- */
-function toArray( data ) {
-	return Array.isArray( data ) ? data : [ data ];
-}
-
-module.exports = {
-	getBaseUrl,
-	isUrlValid,
-	toArray
-};