@govtechsg/oobee 0.10.76 → 0.10.78-alpha1

package/dist/constants/sampleData.js

@@ -0,0 +1,176 @@
+// File used to hold sample data for unit testing
+export const sampleXmlSitemap = `
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd">
+  <!--  created with Free Online Sitemap Generator www.xml-sitemaps.com  -->
+  <url>
+    <loc>http://www.google.com/</loc>
+    <lastmod>2023-01-17T02:49:19+00:00</lastmod>
+    <priority>1.00</priority>
+  </url>
+  <url>
+    <loc>http://www.google.com/intl/en/policies/privacy/</loc>
+    <lastmod>2020-05-26T07:00:00+00:00</lastmod>
+    <priority>0.80</priority>
+  </url>
+  <!-- <url>
+<loc>http://www.google.com/intl/en/policies/terms/</loc>
+<lastmod>2020-05-26T07:00:00+00:00</lastmod>
+<priority>0.80</priority>
+</url> -->
+</urlset>`;
+export const sampleXmlSitemapLinks = [
+    'http://www.google.com/',
+    'http://www.google.com/intl/en/policies/privacy/',
+];
+// Source: https://www.feedforall.com/sample-feed.xml
+export const sampleRssFeed = `<?xml version="1.0" encoding="windows-1252"?>
+<rss version="2.0">
+  <channel>
+    <title>Sample Feed - Favorite RSS Related Software &amp; Resources</title>
+    <description>Take a look at some of FeedForAll&apos;s favorite software and resources for learning more about RSS.</description>
+    <link>http://www.feedforall.com</link>
+    <category domain="www.dmoz.com">Computers/Software/Internet/Site Management/Content Management</category>
+    <copyright>Copyright 2004 NotePage, Inc.</copyright>
+    <docs>http://blogs.law.harvard.edu/tech/rss</docs>
+    <language>en-us</language>
+    <lastBuildDate>Mon, 1 Nov 2004 13:17:17 -0500</lastBuildDate>
+    <managingEditor>marketing@feedforall.com</managingEditor>
+    <pubDate>Tue, 26 Oct 2004 14:06:44 -0500</pubDate>
+    <webMaster>webmaster@feedforall.com</webMaster>
+    <generator>FeedForAll Beta1 (0.0.1.8)</generator>
+    <image>
+      <url>http://www.feedforall.com/feedforall-temp.gif</url>
+      <title>FeedForAll Sample Feed</title>
+      <link>http://www.feedforall.com/industry-solutions.htm</link>
+      <description>FeedForAll Sample Feed</description>
+      <width>144</width>
+      <height>117</height>
+    </image>
+    <item>
+      <title>RSS Resources</title>
+      <description>Be sure to take a look at some of our favorite RSS Resources&lt;br&gt;
+&lt;a href=&quot;http://www.rss-specifications.com&quot;&gt;RSS Specifications&lt;/a&gt;&lt;br&gt;
+&lt;a href=&quot;http://www.blog-connection.com&quot;&gt;Blog Connection&lt;/a&gt;&lt;br&gt;
+&lt;br&gt;</description>
+      <link>http://www.feedforall.com</link>
+      <pubDate>Tue, 26 Oct 2004 14:01:01 -0500</pubDate>
+    </item>
+    <item>
+      <title>Recommended Desktop Feed Reader Software</title>
+      <description>&lt;b&gt;FeedDemon&lt;/b&gt; enables you to quickly read and gather information from hundreds of web sites - without having to visit them. Don&apos;t waste any more time checking your favorite web sites for updates. Instead, use FeedDemon and make them come to you. &lt;br&gt;
+More &lt;a href=&quot;http://store.esellerate.net/a.asp?c=1_SKU5139890208_AFL403073819&quot;&gt;FeedDemon Information&lt;/a&gt;</description>
+      <link>http://www.feedforall.com/feedforall-partners.htm</link>
+      <pubDate>Tue, 26 Oct 2004 14:03:25 -0500</pubDate>
+    </item>
+    <item>
+      <title>Recommended Web Based Feed Reader Software</title>
+      <description>&lt;b&gt;FeedScout&lt;/b&gt; enables you to view RSS/ATOM/RDF feeds from different sites directly in Internet Explorer. You can even set your Home Page to show favorite feeds. Feed Scout is a plug-in for Internet Explorer, so you won&apos;t have to learn anything except for how to press 2 new buttons on Internet Explorer toolbar. &lt;br&gt;
+More &lt;a href=&quot;http://www.bytescout.com/feedscout.html&quot;&gt;Information on FeedScout&lt;/a&gt;&lt;br&gt;
+&lt;br&gt;
+&lt;br&gt;
+&lt;b&gt;SurfPack&lt;/b&gt; can feature search tools, horoscopes, current weather conditions, LiveJournal diaries, humor, web modules and other dynamically updated content. &lt;br&gt;
+More &lt;a href=&quot;http://www.surfpack.com/&quot;&gt;Information on SurfPack&lt;/a&gt;&lt;br&gt;</description>
+      <link>http://www.feedforall.com/feedforall-partners.htm</link>
+      <pubDate>Tue, 26 Oct 2004 14:06:44 -0500</pubDate>
+    </item>
+  </channel>
+</rss>`;
+export const sampleRssFeedLinks = [
+    'http://www.feedforall.com',
+    'http://www.feedforall.com/industry-solutions.htm',
+    'http://www.feedforall.com/feedforall-partners.htm',
+];
+// Source: https://www.ietf.org/rfc/rfc4287.txt
+export const sampleAtomFeed = `<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+  <title type="text">dive into mark</title>
+  <subtitle type="html">
+    A &lt;em&gt;lot&lt;/em&gt; of effort
+    went into making this effortless
+  </subtitle>
+  <updated>2005-07-31T12:29:29Z</updated>
+  <id>tag:example.org,2003:3</id>
+  <link rel="alternate" type="text/html"
+   hreflang="en" href="http://example.org/"/>
+  <link rel="self" type="application/atom+xml"
+   href="http://example.org/feed.atom"/>
+  <rights>Copyright (c) 2003, Mark Pilgrim</rights>
+  <generator uri="http://www.example.com/" version="1.0">
+    Example Toolkit
+  </generator>
+  <entry>
+    <title>Atom draft-07 snapshot</title>
+    <link rel="alternate" type="text/html"
+     href="http://example.org/2005/04/02/atom"/>
+    <link rel="enclosure" type="audio/mpeg" length="1337"
+     href="http://example.org/audio/ph34r_my_podcast.mp3"/>
+    <id>tag:example.org,2003:3.2397</id>
+    <updated>2005-07-31T12:29:29Z</updated>
+    <published>2003-12-13T08:29:29-04:00</published>
+    <author>
+      <name>Mark Pilgrim</name>
+      <uri>http://example.org/</uri>
+      <email>f8dy@example.com</email>
+    </author>
+    <contributor>
+      <name>Sam Ruby</name>
+    </contributor>
+    <contributor>
+      <name>Joe Gregorio</name>
+    </contributor>
+    <content type="xhtml" xml:lang="en"
+     xml:base="http://diveintomark.org/">
+      <div xmlns="http://www.w3.org/1999/xhtml">
+        <p><i>[Update: The Atom draft is finished.]</i></p>
+      </div>
+    </content>
+  </entry>
+</feed>`;
+export const sampleAtomFeedLinks = [
+    'http://example.org/',
+    'http://example.org/feed.atom',
+    'http://example.org/2005/04/02/atom',
+    'http://example.org/audio/ph34r_my_podcast.mp3',
+];
+// Following format stated in https://developers.google.com/search/docs/crawling-indexing/sitemaps/build-sitemap#text
+export const sampleTxtSitemap = `https://www.example.com/file1.html
+https://www.example.com/file2.html
+https://www.example.com/file3.html`;
+export const sampleTxtSitemapLinks = [
+    'https://www.example.com/file1.html',
+    'https://www.example.com/file2.html',
+    'https://www.example.com/file3.html',
+];
+export const sampleNonStandardXmlSitemap = `<?xml version="1.0" encoding="UTF-8"?>
+<contents>
+  <url>https://www.example.com/file1.html</url>
+  <link href='https://www.example.com/file2.html' />
+  <link>https://www.example.com/file3.html</link>
+</contents>`;
+export const sampleNonStandardXmlSitemapLinks = [
+    'https://www.example.com/file1.html',
+    'https://www.example.com/file2.html',
+    'https://www.example.com/file3.html',
+];
+export const sampleRssSitemap2 = `<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+
+  <title>Example Feed</title>
+  <link href="http://example.org/"/>
+  <updated>2003-12-13T18:30:02Z</updated>
+  <author>
+    <name>John Doe</name>
+  </author>
+  <id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
+
+  <entry>
+    <title>Atom-Powered Robots Run Amok</title>
+    <link href="http://example.org/2003/12/13/atom03"/>
+    <id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
+    <updated>2003-12-13T18:30:02Z</updated>
+    <summary>Some text.</summary>
+  </entry>
+
+</feed>
+`;

package/dist/crawlers/commonCrawlerFunc.js

@@ -0,0 +1,428 @@
+import { Dataset, RequestQueue, log, playwrightUtils } from 'crawlee';
+import axe from 'axe-core';
+import { axeScript, disallowedListOfPatterns, guiInfoStatusTypes, RuleFlags, saflyIconSelector, } from '../constants/constants.js';
+import { consoleLogger, guiInfoLog } from '../logs.js';
+import { takeScreenshotForHTMLElements } from '../screenshotFunc/htmlScreenshotFunc.js';
+import { isFilePath } from '../constants/common.js';
+import { extractAndGradeText } from './custom/extractAndGradeText.js';
+import { evaluateAltText } from './custom/evaluateAltText.js';
+import { escapeCssSelector } from './custom/escapeCssSelector.js';
+import { framesCheck } from './custom/framesCheck.js';
+import { findElementByCssSelector } from './custom/findElementByCssSelector.js';
+import { getAxeConfiguration } from './custom/getAxeConfiguration.js';
+import { flagUnlabelledClickableElements } from './custom/flagUnlabelledClickableElements.js';
+import xPathToCss from './custom/xPathToCss.js';
+import { getStoragePath } from '../utils.js';
+import path from 'path';
+const truncateHtml = (html, maxBytes = 1024, suffix = '…') => {
+    const encoder = new TextEncoder();
+    if (encoder.encode(html).length <= maxBytes)
+        return html;
+    let left = 0;
+    let right = html.length;
+    let result = '';
+    while (left <= right) {
+        const mid = Math.floor((left + right) / 2);
+        const truncated = html.slice(0, mid) + suffix;
+        const bytes = encoder.encode(truncated).length;
+        if (bytes <= maxBytes) {
+            result = truncated;
+            left = mid + 1;
+        }
+        else {
+            right = mid - 1;
+        }
+    }
+    return result;
+};
+export const filterAxeResults = (results, pageTitle, customFlowDetails) => {
+    const { violations, passes, incomplete, url } = results;
+    let totalItems = 0;
+    const mustFix = { totalItems: 0, rules: {} };
+    const goodToFix = { totalItems: 0, rules: {} };
+    const passed = { totalItems: 0, rules: {} };
+    const needsReview = { totalItems: 0, rules: {} };
+    const process = (item, displayNeedsReview) => {
+        const { id: rule, help: description, helpUrl, tags, nodes } = item;
+        if (rule === 'frame-tested')
+            return;
+        const conformance = tags.filter(tag => tag.startsWith('wcag') || tag === 'best-practice');
+        // handle rare cases where conformance level is not the first element
+        const wcagRegex = /^wcag\d+a+$/;
+        if (conformance[0] !== 'best-practice' && !wcagRegex.test(conformance[0])) {
+            conformance.sort((a, b) => {
+                if (wcagRegex.test(a) && !wcagRegex.test(b)) {
+                    return -1;
+                }
+                if (!wcagRegex.test(a) && wcagRegex.test(b)) {
+                    return 1;
+                }
+                return 0;
+            });
+        }
+        const addTo = (category, node) => {
+            const { html, failureSummary, screenshotPath, target, impact: axeImpact } = node;
+            if (!(rule in category.rules)) {
+                category.rules[rule] = {
+                    description,
+                    axeImpact,
+                    helpUrl,
+                    conformance,
+                    totalItems: 0,
+                    items: [],
+                };
+            }
+            const message = displayNeedsReview
+                ? failureSummary.slice(failureSummary.indexOf('\n') + 1).trim()
+                : failureSummary;
+            let finalHtml = html;
+            if (html.includes('</script>')) {
+                finalHtml = html.replaceAll('</script>', '&lt;/script>');
+            }
+            finalHtml = truncateHtml(finalHtml);
+            const xpath = target.length === 1 && typeof target[0] === 'string' ? target[0] : null;
+            // add in screenshot path
+            category.rules[rule].items.push({
+                html: finalHtml,
+                message,
+                screenshotPath,
+                xpath: xpath || undefined,
+                displayNeedsReview: displayNeedsReview || undefined,
+            });
+            category.rules[rule].totalItems += 1;
+            category.totalItems += 1;
+            totalItems += 1;
+        };
+        nodes.forEach(node => {
+            const hasWcagA = conformance.some(tag => /^wcag\d*a$/.test(tag));
+            const hasWcagAA = conformance.some(tag => /^wcag\d*aa$/.test(tag));
+            // const hasWcagAAA = conformance.some(tag => /^wcag\d*aaa$/.test(tag));
+            if (displayNeedsReview) {
+                addTo(needsReview, node);
+            }
+            else if (hasWcagA || hasWcagAA) {
+                addTo(mustFix, node);
+            }
+            else {
+                addTo(goodToFix, node);
+            }
+        });
+    };
+    violations.forEach(item => process(item, false));
+    incomplete.forEach(item => process(item, true));
+    passes.forEach((item) => {
+        const { id: rule, help: description, impact: axeImpact, helpUrl, tags, nodes } = item;
+        if (rule === 'frame-tested')
+            return;
+        const conformance = tags.filter(tag => tag.startsWith('wcag') || tag === 'best-practice');
+        nodes.forEach(node => {
+            const { html } = node;
+            if (!(rule in passed.rules)) {
+                passed.rules[rule] = {
+                    description,
+                    axeImpact,
+                    helpUrl,
+                    conformance,
+                    totalItems: 0,
+                    items: [],
+                };
+            }
+            const finalHtml = truncateHtml(html);
+            passed.rules[rule].items.push({ html: finalHtml, screenshotPath: '', message: '', xpath: '' });
+            passed.totalItems += 1;
+            passed.rules[rule].totalItems += 1;
+            totalItems += 1;
+        });
+    });
+    return {
+        url,
+        pageTitle: customFlowDetails ? `${customFlowDetails.pageIndex}: ${pageTitle}` : pageTitle,
+        pageIndex: customFlowDetails ? customFlowDetails.pageIndex : undefined,
+        metadata: customFlowDetails?.metadata
+            ? `${customFlowDetails.pageIndex}: ${customFlowDetails.metadata}`
+            : undefined,
+        pageImagePath: customFlowDetails ? customFlowDetails.pageImagePath : undefined,
+        totalItems,
+        mustFix,
+        goodToFix,
+        needsReview,
+        passed,
+    };
+};
+export const runAxeScript = async ({ includeScreenshots, page, randomToken, customFlowDetails = null, selectors = [], ruleset = [], }) => {
+    const browserContext = page.context();
+    const requestUrl = page.url();
+    try {
+        // Checking for DOM mutations before proceeding to scan
+        await page.evaluate(() => {
+            return new Promise(resolve => {
+                let timeout;
+                let mutationCount = 0;
+                const MAX_MUTATIONS = 500;
+                const MAX_SAME_MUTATION_LIMIT = 10;
+                const mutationHash = {};
+                const observer = new MutationObserver(mutationsList => {
+                    clearTimeout(timeout);
+                    mutationCount += 1;
+                    if (mutationCount > MAX_MUTATIONS) {
+                        observer.disconnect();
+                        resolve('Too many mutations detected');
+                    }
+                    // To handle scenario where DOM elements are constantly changing and unable to exit
+                    mutationsList.forEach(mutation => {
+                        let mutationKey;
+                        if (mutation.target instanceof Element) {
+                            Array.from(mutation.target.attributes).forEach(attr => {
+                                mutationKey = `${mutation.target.nodeName}-${attr.name}`;
+                                if (mutationKey) {
+                                    if (!mutationHash[mutationKey]) {
+                                        mutationHash[mutationKey] = 1;
+                                    }
+                                    else {
+                                        mutationHash[mutationKey] += 1;
+                                    }
+                                    if (mutationHash[mutationKey] >= MAX_SAME_MUTATION_LIMIT) {
+                                        observer.disconnect();
+                                        resolve(`Repeated mutation detected for ${mutationKey}`);
+                                    }
+                                }
+                            });
+                        }
+                    });
+                    timeout = setTimeout(() => {
+                        observer.disconnect();
+                        resolve('DOM stabilized after mutations.');
+                    }, 1000);
+                });
+                timeout = setTimeout(() => {
+                    observer.disconnect();
+                    resolve('No mutations detected, exit from idle state');
+                }, 1000);
+                observer.observe(document, { childList: true, subtree: true, attributes: true });
+            });
+        });
+    }
+    catch (e) {
+        // do nothing, just continue
+    }
+    // Omit logging of browser console errors to reduce unnecessary verbosity
+    /*
+    page.on('console', msg => {
+      const type = msg.type();
+      if (type === 'error') {
+        consoleLogger.error(msg.text());
+      } else {
+        consoleLogger.info(msg.text());
+      }
+    });
+    */
+    const disableOobee = ruleset.includes(RuleFlags.DISABLE_OOBEE);
+    const enableWcagAaa = ruleset.includes(RuleFlags.ENABLE_WCAG_AAA);
+    const gradingReadabilityFlag = await extractAndGradeText(page); // Ensure flag is obtained before proceeding
+    await playwrightUtils.injectFile(page, axeScript);
+    const results = await page.evaluate(async ({ selectors, saflyIconSelector, disableOobee, enableWcagAaa, gradingReadabilityFlag, evaluateAltTextFunctionString, escapeCssSelectorFunctionString, framesCheckFunctionString, findElementByCssSelectorFunctionString, getAxeConfigurationFunctionString, flagUnlabelledClickableElementsFunctionString, xPathToCssFunctionString, }) => {
+        try {
+            // Load functions into the browser context
+            eval(evaluateAltTextFunctionString);
+            eval(escapeCssSelectorFunctionString);
+            eval(framesCheckFunctionString);
+            eval(findElementByCssSelectorFunctionString);
+            eval(flagUnlabelledClickableElementsFunctionString);
+            eval(xPathToCssFunctionString);
+            eval(getAxeConfigurationFunctionString);
+            // remove so that axe does not scan
+            document.querySelector(saflyIconSelector)?.remove();
+            const oobeeAccessibleLabelFlaggedXpaths = disableOobee
+                ? []
+                : (await flagUnlabelledClickableElements()).map(item => item.xpath);
+            const oobeeAccessibleLabelFlaggedCssSelectors = oobeeAccessibleLabelFlaggedXpaths
+                .map(xpath => {
+                try {
+                    const cssSelector = xPathToCss(xpath);
+                    return cssSelector;
+                }
+                catch (e) {
+                    console.error('Error converting XPath to CSS: ', xpath, e);
+                    return '';
+                }
+            })
+                .filter(item => item !== '');
+            const axeConfig = getAxeConfiguration({
+                enableWcagAaa,
+                gradingReadabilityFlag,
+                disableOobee,
+            });
+            axe.configure(axeConfig);
+            // removed needsReview condition
+            const defaultResultTypes = ['violations', 'passes', 'incomplete'];
+            return axe
+                .run(selectors, {
+                resultTypes: defaultResultTypes,
+            })
+                .then(results => {
+                if (disableOobee) {
+                    return results;
+                }
+                // handle css id selectors that start with a digit
+                const escapedCssSelectors = oobeeAccessibleLabelFlaggedCssSelectors.map(escapeCssSelector);
+                // Add oobee violations to Axe's report
+                const oobeeAccessibleLabelViolations = {
+                    id: 'oobee-accessible-label',
+                    impact: 'serious',
+                    tags: ['wcag2a', 'wcag211', 'wcag412'],
+                    description: 'Ensures clickable elements have an accessible label.',
+                    help: 'Clickable elements (i.e. elements with mouse-click interaction) must have accessible labels.',
+                    helpUrl: 'https://www.deque.com/blog/accessible-aria-buttons',
+                    nodes: escapedCssSelectors
+                        .map((cssSelector) => ({
+                        html: findElementByCssSelector(cssSelector),
+                        target: [cssSelector],
+                        impact: 'serious',
+                        failureSummary: 'Fix any of the following:\n  The clickable element does not have an accessible label.',
+                        any: [
+                            {
+                                id: 'oobee-accessible-label',
+                                data: null,
+                                relatedNodes: [],
+                                impact: 'serious',
+                                message: 'The clickable element does not have an accessible label.',
+                            },
+                        ],
+                        all: [],
+                        none: [],
+                    }))
+                        .filter(item => item.html),
+                };
+                results.violations = [...results.violations, oobeeAccessibleLabelViolations];
+                return results;
+            })
+                .catch(e => {
+                console.error('Error at axe.run', e);
+                throw e;
+            });
+        }
+        catch (e) {
+            console.error(e);
+            throw e;
+        }
+    }, {
+        selectors,
+        saflyIconSelector,
+        disableOobee,
+        enableWcagAaa,
+        gradingReadabilityFlag,
+        evaluateAltTextFunctionString: evaluateAltText.toString(),
+        escapeCssSelectorFunctionString: escapeCssSelector.toString(),
+        framesCheckFunctionString: framesCheck.toString(),
+        findElementByCssSelectorFunctionString: findElementByCssSelector.toString(),
+        getAxeConfigurationFunctionString: getAxeConfiguration.toString(),
+        flagUnlabelledClickableElementsFunctionString: flagUnlabelledClickableElements.toString(),
+        xPathToCssFunctionString: xPathToCss.toString(),
+    });
+    if (includeScreenshots) {
+        results.violations = await takeScreenshotForHTMLElements(results.violations, page, randomToken);
+        results.incomplete = await takeScreenshotForHTMLElements(results.incomplete, page, randomToken);
+    }
+    let pageTitle = null;
+    try {
+        pageTitle = await page.evaluate(() => document.title);
+    }
+    catch (e) {
+        consoleLogger.info(`Error while getting page title: ${e}`);
+        if (page.isClosed()) {
+            consoleLogger.info(`Page was closed for ${requestUrl}, creating new page`);
+            page = await browserContext.newPage();
+            await page.goto(requestUrl, { waitUntil: 'domcontentloaded' });
+            pageTitle = await page.evaluate(() => document.title);
+        }
+    }
+    return filterAxeResults(results, pageTitle, customFlowDetails);
+};
+export const createCrawleeSubFolders = async (randomToken) => {
+    const crawleeDir = path.join(getStoragePath(randomToken), "crawlee");
+    const dataset = await Dataset.open(crawleeDir);
+    const requestQueue = await RequestQueue.open(crawleeDir);
+    return { dataset, requestQueue };
+};
+export const preNavigationHooks = (extraHTTPHeaders) => {
+    return [
+        async (crawlingContext, gotoOptions) => {
+            if (extraHTTPHeaders) {
+                crawlingContext.request.headers = extraHTTPHeaders;
+            }
+            gotoOptions = { waitUntil: 'networkidle', timeout: 30000 };
+        },
+    ];
+};
+export const postNavigationHooks = [
+    async (_crawlingContext) => {
+        guiInfoLog(guiInfoStatusTypes.COMPLETED, {});
+    },
+];
+export const failedRequestHandler = async ({ request }) => {
+    guiInfoLog(guiInfoStatusTypes.ERROR, { numScanned: 0, urlScanned: request.url });
+    log.error(`Failed Request - ${request.url}: ${request.errorMessages}`);
+};
+export const isUrlPdf = (url) => {
+    if (isFilePath(url)) {
+        return /\.pdf$/i.test(url);
+    }
+    const parsedUrl = new URL(url);
+    return /\.pdf($|\?|#)/i.test(parsedUrl.pathname) || /\.pdf($|\?|#)/i.test(parsedUrl.href);
+};
+export async function shouldSkipClickDueToDisallowedHref(page, element) {
+    return await page.evaluate(({ el, disallowedPrefixes }) => {
+        function isDisallowedHref(href) {
+            if (!href)
+                return false;
+            href = href.toLowerCase();
+            return disallowedPrefixes.some((prefix) => href.startsWith(prefix));
+        }
+        const castEl = el;
+        // Check descendant <a href="">
+        const descendants = castEl.querySelectorAll('a[href]');
+        for (const a of descendants) {
+            const href = a.getAttribute('href');
+            if (isDisallowedHref(href)) {
+                return true;
+            }
+        }
+        // Check self and ancestors for disallowed <a>
+        let current = castEl;
+        while (current) {
+            if (current.tagName === 'A' &&
+                isDisallowedHref(current.getAttribute('href'))) {
+                return true;
+            }
+            current = current.parentElement;
+        }
+        return false;
+    }, {
+        el: element,
+        disallowedPrefixes: disallowedListOfPatterns,
+    });
+}
+/**
+ * Check if response should be skipped based on content headers.
+ * @param response - Playwright Response object
+ * @param requestUrl - Optional: request URL for logging
+ * @returns true if the content should be skipped
+ */
+export const shouldSkipDueToUnsupportedContent = (response, requestUrl = '') => {
+    if (!response)
+        return false;
+    const headers = response.headers();
+    const contentDisposition = headers['content-disposition'] || '';
+    const contentType = headers['content-type'] || '';
+    if (contentDisposition.includes('attachment')) {
+        // consoleLogger.info(`Skipping attachment (content-disposition) at ${requestUrl}`);
+        return true;
+    }
+    if (contentType.startsWith('application/') ||
+        contentType.includes('octet-stream') ||
+        (!contentType.startsWith('text/') && !contentType.includes('html'))) {
+        // consoleLogger.info(`Skipping non-processible content-type "${contentType}" at ${requestUrl}`);
+        return true;
+    }
+    return false;
+};