crawlforge-mcp-server 6.22.2 → 6.23.0

package/src/skills/agent-skills/crawlforge-web-scraping/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-web-scraping
 description: "Scrapes web pages and returns clean Markdown, HTML, plain text, links, or metadata using CrawlForge's scrape, fetch_url, extract_content, extract_text, extract_links, extract_metadata, map_site, and crawl_deep tools. Use when the user wants to scrape a URL, fetch a page, get the markdown or text of a website, extract links or metadata, read an article, map a site's URLs, generate a sitemap, or crawl a whole website or documentation site. Prefer the unified scrape tool to get multiple formats in one call; use map_site to discover URLs and crawl_deep to walk an entire site."
 metadata:
-  version: 6.22.2
+  version: 6.23.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-web-scraping/references/tool-reference.md

@@ -122,7 +122,7 @@ Reads a result a tool returned with `truncated: true` and a `result_handle`
 | `length` | number | — | `slice`: characters (default 10,000); `lines`: lines (default 200, at most 5,000). |
 | `query` | string | — | `search`: case-insensitive literal substring. |
 | `max_matches` | number | `20` | `search`: 1–100. |
-| `path` | string | — | `json_path`: path into the stored result object (or the parsed body when the stored text is JSON, e.g. a `fetch_url` body). |
+| `path` | string | — | `json_path`: path from the root of the stored result; a `fetch_url` body is under `body.` (parsed when it holds JSON), and a key containing a dot goes in brackets, e.g. `files["llms.txt"]`. |
 | `max_inline_chars` | number | `40000` | 1,000–10,000,000. |
 
 Every response carries `handle`, `tool`, `operation`, `view` (`text` or `json`),

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -13,6 +13,7 @@ import { flattenText } from 'crawlforge-extractors';
 import ExtractContentTool from '../extract/extractContent.js';
 import { stealthDocumentVerdict } from '../../utils/stealthVerdict.js';
 import { elementText } from '../../utils/elementText.js';
+import { parseSelectorSpec } from '../basic/scrapeStructured.js';
 import { pageTitle } from '../../utils/pageTitle.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 import { resolveStealthEngine, redactProxyEntry, assertProxyEngineAllowed } from '../../core/StealthBrowserManager.js';
@@ -779,7 +780,7 @@ export class ScrapeWithActionsTool extends EventEmitter {
         }
 
         if (params.extractionOptions?.selectors) {
-          state.content.extracted = this.extractWithSelectors($, params.extractionOptions.selectors);
+          state.content.extracted = this.extractWithSelectors($, params.extractionOptions.selectors, stateData.formValues);
         }
 
         states.push(state);
@@ -827,16 +828,21 @@ export class ScrapeWithActionsTool extends EventEmitter {
         $('script, style, noscript, template').remove();
         extractResult.content = extractResult.content || {};
         if (params.extractionOptions?.selectors) {
-          extractResult.content.extracted = this.extractWithSelectors($, params.extractionOptions.selectors);
+          extractResult.content.extracted = this.extractWithSelectors($, params.extractionOptions.selectors, chainResult.finalFormValues);
         }
         // Readability picks a "main" block and drops form controls; on a small
         // post-action page (a demo form, a results panel) that leaves the
         // footer alone, or a page minus the very buttons the actions produced.
         // A short readable result means it found nothing substantial, so hand
-        // back the whole (equally short) body text instead.
+        // back the whole (equally short) body text instead. So does a page
+        // whose Readability article was not its main content (a landing or
+        // list page): the whole page, as browser_session read returns it,
+        // rather than the boilerplate-removal guess — that read kept all 10
+        // of quotes.toscrape.com's quotes where this text lost 3 (2026-10-20).
         const readable = extractResult.content.text || '';
         const bodyText = flattenText($);
-        if (readable.length < 300 && bodyText.length > readable.length) {
+        const notMainContent = extractResult.extractionMethod && extractResult.extractionMethod !== 'readability';
+        if ((readable.length < 300 || notMainContent) && bodyText.length > readable.length) {
           extractResult.content.text = bodyText;
           extractResult.content.textSource = 'body';
         }
@@ -940,10 +946,29 @@ export class ScrapeWithActionsTool extends EventEmitter {
     return links;
   }
 
-  extractWithSelectors($, selectors) {
+  /**
+   * Run `extractionOptions.selectors` over a captured page. A selector may end
+   * in `@attr` to read that attribute instead of the text, as in
+   * scrape_structured. `@value` on an input, textarea or select reads what the
+   * control holds now — typed or selected — from `formValues` (captured from
+   * the live page, in document order); every other `@attr` reads the
+   * attribute as the page's HTML has it.
+   */
+  extractWithSelectors($, selectors, formValues = []) {
     const extracted = {};
+    const controls = $('input, textarea, select').toArray();
 
-    for (const [key, selector] of Object.entries(selectors)) {
+    const read = (el, attribute) => {
+      if (!attribute) return elementText($, el);
+      if (attribute === 'value') {
+        const live = formValues[controls.indexOf(el)];
+        if (live?.tag === el.name) return live.value;
+      }
+      return $(el).attr(attribute) ?? null;
+    };
+
+    for (const [key, raw] of Object.entries(selectors)) {
+      const { selector, attribute } = parseSelectorSpec(raw);
       try {
         const elements = $(selector);
         
@@ -953,12 +978,13 @@ export class ScrapeWithActionsTool extends EventEmitter {
         if (elements.length === 0) {
           extracted[key] = null;
         } else if (elements.length === 1) {
-          extracted[key] = elementText($, elements.get(0));
+          extracted[key] = read(elements.get(0), attribute);
         } else {
-          extracted[key] = elements.map((_, el) => elementText($, el)).get();
+          // Not .map().get(), which drops the nulls an absent attribute gives.
+          extracted[key] = elements.toArray().map((el) => read(el, attribute));
         }
       } catch (error) {
-        extracted[key] = { error: `Invalid selector: ${selector}` };
+        extracted[key] = { error: `Invalid selector: ${raw}` };
       }
     }
 

package/src/tools/agent/sourceChecks.js

@@ -0,0 +1,103 @@
+/**
+ * sourceChecks.js — deterministic checks on which sources the `agent` trusts
+ * and which dates it reports (live sweep 2026-10-20, item 2.5).
+ *
+ * "Latest stable Rust version and its release date" was first searched as the
+ * bare word "Rust": the game's site (rust.facepunch.com, a root titled "Rust")
+ * passed as the entity's official site, was promoted to the live page, and the
+ * run came back degraded:true blaming it. The answer gave 1.99.1 "as of 16
+ * October 2026": endoflife.date's "Last updated on" date, not the 15 October
+ * release date in its table.
+ *
+ * Imports nothing, so AgentOrchestrator can read it without a cycle.
+ */
+
+/** The registrable label of a hostname: "python" for www.python.org, "bbc" for bbc.co.uk. */
+export function siteLabel(hostname) {
+  const labels = hostname.toLowerCase().split('.');
+  const ccSecondLevel = labels.length >= 3 && labels.at(-1).length === 2 && labels.at(-2).length <= 3;
+  return labels[Math.max(labels.length - (ccSecondLevel ? 3 : 2), 0)];
+}
+
+function labelOf(url) {
+  try { return siteLabel(new URL(url).hostname); } catch { return null; }
+}
+
+/**
+ * The origins a search for the whole question also returns (same site label,
+ * any subdomain). A bare entity name is ambiguous — "Rust" is a language, a
+ * game and a film — and the site that owns the bare name need not be the one
+ * the question is about. A site the question's own search does not surface is
+ * not promoted as its live page. When that search returned nothing there is
+ * nothing to check against, and every origin passes.
+ *
+ * @param {Iterable<string>} origins candidate origins from the entity query
+ * @param {string[]} questionUrls result URLs of the other queries
+ * @returns {Set<string>}
+ */
+export function corroboratedOrigins(origins, questionUrls) {
+  const all = new Set(origins);
+  const labels = new Set((questionUrls || []).map(labelOf).filter(Boolean));
+  if (labels.size === 0) return all;
+  return new Set([...all].filter(origin => labels.has(labelOf(origin))));
+}
+
+const MONTHS = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'];
+const MONTH = '(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\\.?';
+const DATE_RE = new RegExp(
+  `\\b(\\d{4})-(\\d{2})-(\\d{2})\\b` +
+  `|\\b${MONTH}\\s+(\\d{1,2})(?:st|nd|rd|th)?,?\\s+(\\d{4})\\b` +
+  `|\\b(\\d{1,2})(?:st|nd|rd|th)?\\s+${MONTH}\\s+(\\d{4})\\b`,
+  'gi'
+);
+
+/** Every date in a text, written any of three ways, keyed as YYYY-MM-DD. */
+function datesIn(text) {
+  const found = [];
+  for (const m of String(text || '').matchAll(DATE_RE)) {
+    let y, mo, d;
+    if (m[1]) [y, mo, d] = [m[1], Number(m[2]), Number(m[3])];
+    else if (m[4]) [y, mo, d] = [m[6], MONTHS.indexOf(m[4].toLowerCase()) + 1, Number(m[5])];
+    else [y, mo, d] = [m[9], MONTHS.indexOf(m[8].toLowerCase()) + 1, Number(m[7])];
+    found.push({ raw: m[0], key: `${y}-${String(mo).padStart(2, '0')}-${String(d).padStart(2, '0')}`, index: m.index });
+  }
+  return found;
+}
+
+/** A date written right after one of these is when a page changed, not when the thing happened. */
+const UPDATE_LABEL = /\b(?:updated|modified|edited|revised|retrieved|accessed|crawled)\b[^\d\n]{0,20}$/i;
+
+/** A task that asks when something was updated wants exactly those dates. */
+const ASKS_FOR_UPDATES = /\b(?:updated?|modified|last (?:changed|edited))\b/i;
+
+/**
+ * Dates the answer states that its sources give only as a page's
+ * "last updated" / "modified" date, never as anything else. Each comes with
+ * the first source that labels it so, so a warning names a page the run used.
+ *
+ * @param {string} answer
+ * @param {{url: string, text?: string}[]} evidence
+ * @param {string} prompt
+ * @returns {{date: string, url: string}[]}
+ */
+export function updateLabelledDates(answer, evidence, prompt) {
+  if (!answer || ASKS_FOR_UPDATES.test(prompt || '')) return [];
+  const byKey = new Map();
+  for (const e of evidence || []) {
+    const text = e.text || '';
+    for (const { key, index } of datesIn(text)) {
+      const lineStart = text.lastIndexOf('\n', index - 1) + 1;
+      const updated = UPDATE_LABEL.test(text.slice(Math.max(lineStart, index - 40), index));
+      const seen = byKey.get(key) || { other: false, url: null };
+      if (updated) seen.url = seen.url || e.url;
+      else seen.other = true;
+      byKey.set(key, seen);
+    }
+  }
+  const flagged = [];
+  for (const { raw, key } of datesIn(answer.replace(/https?:\/\/\S+/g, ' '))) {
+    const seen = byKey.get(key);
+    if (seen?.url && !seen.other && !flagged.some(f => f.key === key)) flagged.push({ key, date: raw, url: seen.url });
+  }
+  return flagged.map(({ date, url }) => ({ date, url }));
+}

package/src/tools/basic/extractMetadata.js

@@ -88,9 +88,10 @@ export async function extractMetadataHandler({ url, user_agent, respect_robots,
       $('h1').first().text().trim() ||
       '';
 
-    const description =
+    const description = (
       $('meta[name="description"]').attr('content') ||
-      $('meta[property="og:description"]').attr('content') || '';
+      $('meta[property="og:description"]').attr('content') || ''
+    ).trim();
     const keywords = $('meta[name="keywords"]').attr('content') || '';
     const canonical = $('link[rel="canonical"]').attr('href') || '';
 
@@ -112,9 +113,13 @@ export async function extractMetadataHandler({ url, user_agent, respect_robots,
     const author = $('meta[name="author"]').attr('content') || '';
     const robots = $('meta[name="robots"]').attr('content') || '';
     const viewport = $('meta[name="viewport"]').attr('content') || '';
-    const charset =
+    // Only the charset token: the http-equiv form used to come back whole,
+    // "text/html; charset=utf-8" (live sweep 2026-10-20).
+    const charset = (
       $('meta[charset]').attr('charset') ||
-      $('meta[http-equiv="Content-Type"]').attr('content') || '';
+      ($('meta[http-equiv="Content-Type"]').attr('content') || '').match(/charset\s*=\s*["']?([^\s;"']+)/i)?.[1] ||
+      ''
+    ).trim().toUpperCase();
 
     const jsonLd = parseJsonLd($);
     const microdata = parseMicrodata($);

package/src/tools/basic/extractText.js

@@ -14,7 +14,7 @@ import { flattenText, gridTables, absoluteUrls } from 'crawlforge-extractors';
 import { JSDOM } from 'jsdom';
 import { Readability } from '@mozilla/readability';
 import { fetchLadder, ladderErrorResult, isJsonType } from '../../utils/fetchLadder.js';
-import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
+import { htmlToMarkdown, normalizeCodeBlocks } from '../../utils/htmlToMarkdown.js';
 import { tooLargeForReadability } from '../scrape/_mainContent.js';
 
 // Block-level elements whose boundaries should become paragraph breaks
@@ -67,11 +67,13 @@ export function extractBlockText($) {
  * @returns {string}
  */
 export function readabilityToMarkdown(html, pageUrl) {
-  let articleHtml = html;
+  // Code-block languages live in classes Readability strips (2.2).
+  const codeHtml = normalizeCodeBlocks(html);
+  let articleHtml = codeHtml;
   // A page too large for Readability is converted whole (R25).
-  if (!tooLargeForReadability(html)) {
+  if (!tooLargeForReadability(codeHtml)) {
     try {
-      const dom = new JSDOM(html, { url: pageUrl });
+      const dom = new JSDOM(codeHtml, { url: pageUrl });
       const reader = new Readability(dom.window.document);
       const article = reader.parse();
       if (article) articleHtml = article.content;

package/src/tools/basic/fetchUrl.js

@@ -36,6 +36,9 @@ export async function fetchUrlHandler({ url, headers, timeout, user_agent, respe
           contentType: response.headers.get('content-type') || 'unknown',
           size: body.length,
           url: response.url,
+          // Whether a redirect was followed, and the URL the body came from.
+          redirected: response.redirected === true,
+          final_url: response.url || url,
           // Time the target took to respond and deliver the body, in ms.
           // Excludes the per-host throttle, so it can back a latency check.
           responseTime: response._responseTime

package/src/tools/basic/scrapeStructured.js

@@ -5,6 +5,9 @@
  *     fix elements_found to report real per-field DOM match counts.
  * 3.2: Optional row_selector returns row-aligned records instead of the
  *     default parallel arrays, which are matched independently per field.
+ * 6.4: A field may be an object {selector, multiple}; multiple:true returns
+ *     every match as an array (inside a row it otherwise keeps only the first).
+ *     @href and @src values are resolved to absolute URLs against the page URL.
  */
 
 import { load } from 'cheerio';
@@ -25,7 +28,7 @@ const ATTR_SUFFIX_RE = /@[A-Za-z_:][\w:.-]*$/;
  * @param {string} raw
  * @returns {{ selector: string, attribute: string|null }}
  */
-function parseSelectorSpec(raw) {
+export function parseSelectorSpec(raw) {
   const match = ATTR_SUFFIX_RE.exec(raw);
   // Only treat it as an attribute suffix if it isn't inside brackets/quotes,
   // i.e. the selector portion before it has balanced [ ] and quotes.
@@ -41,6 +44,42 @@ function parseSelectorSpec(raw) {
   return { selector: raw, attribute: null };
 }
 
+/**
+ * A field is either a selector string ("css" or "css@attr") or an object
+ * { selector: "css@attr", multiple?: boolean }.
+ * @param {string|{ selector: string, multiple?: boolean }} field
+ * @returns {{ raw: string, selector: string, attribute: string|null, multiple: boolean }}
+ */
+export function parseFieldSpec(field) {
+  const raw = typeof field === 'string' ? field : field?.selector;
+  if (typeof raw !== 'string' || raw === '') {
+    throw new Error('a field must be a selector string or an object with a selector string');
+  }
+  return { raw, ...parseSelectorSpec(raw), multiple: typeof field === 'object' && field.multiple === true };
+}
+
+// Attributes holding a URL, resolved against the page URL so a relative
+// "/item?id=1" comes back usable.
+const URL_ATTRIBUTES = new Set(['href', 'src']);
+
+/**
+ * The value of one matched element: its text, or the named attribute (null
+ * when absent), with href/src made absolute against `baseUrl`.
+ */
+function extractValue($, el, attribute, baseUrl) {
+  if (!attribute) return elementText($, el);
+  const value = $(el).attr(attribute);
+  if (value == null) return null;
+  if (URL_ATTRIBUTES.has(attribute.toLowerCase()) && baseUrl) {
+    try {
+      return new URL(value.trim(), baseUrl).href;
+    } catch {
+      return value;
+    }
+  }
+  return value;
+}
+
 /**
  * Row-aligned extraction: every field is matched *inside* each row element, so
  * record N of the output is row N of the page. A field a row lacks is null
@@ -48,37 +87,46 @@ function parseSelectorSpec(raw) {
  * parallel-array output does — there each field is matched independently across
  * the whole document and the arrays are not row-aligned.
  *
+ * A field with multiple:true is an array of every match inside the row ([] when
+ * the row has none); otherwise it is the first match.
+ *
  * @param {import('cheerio').CheerioAPI} $
- * @param {{ row_selector: string, selectors: Record<string, string>, max_results?: number }} params
+ * @param {{ row_selector: string, selectors: Record<string, string|object>, max_results?: number, baseUrl?: string }} params
  * @returns {{ records: object[], rowsFound: number, matchCounts: Record<string, number> }}
  */
-function extractRows($, { row_selector, selectors, max_results }) {
+function extractRows($, { row_selector, selectors, max_results, baseUrl }) {
   const allRows = $(row_selector);
   // In row mode max_results caps rows, not per-field matches.
   const rows = (max_results != null && max_results > 0)
     ? allRows.slice(0, max_results)
     : allRows;
 
-  const specs = Object.entries(selectors).map(([field, raw]) => [field, raw, parseSelectorSpec(raw)]);
+  const specs = Object.entries(selectors).map(([field, spec]) => [field, spec]);
   const matchCounts = Object.fromEntries(specs.map(([field]) => [field, 0]));
 
   const records = rows.toArray().map((rowEl) => {
     const $row = $(rowEl);
     const record = {};
-    for (const [field, raw, { selector, attribute }] of specs) {
+    for (const [field, spec] of specs) {
       try {
+        const { selector, attribute, multiple } = parseFieldSpec(spec);
         // A field selector may name the row element itself (row "tr.athing",
         // field "tr@id"); .find() searches descendants only.
-        const el = $row.is(selector) ? $row : $row.find(selector).first();
-        if (el.length === 0) {
+        const matches = $row.is(selector) ? $row : $row.find(selector);
+        if (multiple) {
+          record[field] = matches.toArray().map((el) => extractValue($, el, attribute, baseUrl));
+          if (matches.length > 0) matchCounts[field] += 1;
+          continue;
+        }
+        if (matches.length === 0) {
           record[field] = null;
           continue;
         }
-        record[field] = attribute ? (el.attr(attribute) ?? null) : elementText($, el.get(0));
+        record[field] = extractValue($, matches.get(0), attribute, baseUrl);
         matchCounts[field] += 1;
       } catch (selectorError) {
         record[field] = {
-          error: `Invalid selector: ${raw}`,
+          error: `Invalid selector: ${typeof spec === 'string' ? spec : spec?.selector}`,
           message: selectorError.message
         };
       }
@@ -90,7 +138,7 @@ function extractRows($, { row_selector, selectors, max_results }) {
 }
 
 /**
- * @param {{ url: string, selectors: Record<string, string>, row_selector?: string,
+ * @param {{ url: string, selectors: Record<string, string|{ selector: string, multiple?: boolean }>, row_selector?: string,
  *   max_results?: number, user_agent?: string, respect_robots?: boolean }} params
  */
 export async function scrapeStructuredHandler({ url, selectors, row_selector, max_results, user_agent, respect_robots }) {
@@ -108,7 +156,7 @@ export async function scrapeStructuredHandler({ url, selectors, row_selector, ma
     const $ = load(html);
 
     if (row_selector) {
-      const { records, rowsFound, matchCounts } = extractRows($, { row_selector, selectors, max_results });
+      const { records, rowsFound, matchCounts } = extractRows($, { row_selector, selectors, max_results, baseUrl: response.url || url });
       return {
         content: [{
           type: 'text',
@@ -127,29 +175,26 @@ export async function scrapeStructuredHandler({ url, selectors, row_selector, ma
     const results = {};
     const matchCounts = {};
 
+    const baseUrl = response.url || url;
     for (const [fieldName, rawSelector] of Object.entries(selectors)) {
       try {
-        const { selector, attribute } = parseSelectorSpec(rawSelector);
+        const { selector, attribute, multiple } = parseFieldSpec(rawSelector);
         let elements = $(selector);
         const domCount = elements.length;
         matchCounts[fieldName] = domCount;
 
         if (domCount === 0) {
-          results[fieldName] = null;
+          results[fieldName] = multiple ? [] : null;
         } else {
           // Apply max_results cap if specified
           if (max_results != null && max_results > 0 && domCount > max_results) {
             elements = elements.slice(0, max_results);
           }
 
-          const extract = (el) => {
-            if (attribute) {
-              return $(el).attr(attribute) ?? null;
-            }
-            return elementText($, el);
-          };
+          const extract = (el) => extractValue($, el, attribute, baseUrl);
 
-          if (elements.length === 1) {
+          // multiple:true always answers with an array, even for one match.
+          if (elements.length === 1 && !multiple) {
             results[fieldName] = extract(elements.get(0));
           } else {
             // cheerio's .map().get() drops null/undefined results, which would
@@ -160,7 +205,7 @@ export async function scrapeStructuredHandler({ url, selectors, row_selector, ma
         }
       } catch (selectorError) {
         results[fieldName] = {
-          error: `Invalid selector: ${rawSelector}`,
+          error: `Invalid selector: ${typeof rawSelector === 'string' ? rawSelector : rawSelector?.selector}`,
           message: selectorError.message
         };
         matchCounts[fieldName] = 0;

package/src/tools/crawl/mapSite.js

@@ -48,6 +48,21 @@ export function searchScore(url, search) {
   return terms.filter((t) => haystack.includes(t)).length;
 }
 
+/**
+ * normalizeUrl, plus a directory index folded into its directory: `/` and
+ * `/index.html` were listed as two URLs (live sweep 2026-10-20). Kept here,
+ * not in the shared normalizer, so other tools' URL keys do not change.
+ * @param {string} url
+ * @returns {string}
+ */
+function normalizeMapUrl(url) {
+  const normalized = normalizeUrl(url);
+  const u = new URL(normalized);
+  if (!/\/index\.html?$/i.test(u.pathname)) return normalized;
+  u.pathname = u.pathname.replace(/index\.html?$/i, '');
+  return normalizeUrl(u.href);
+}
+
 // A gate refusal is the answer to the request, not a page we failed to read.
 const REFUSAL_CODES = new Set(['ROBOTS_DISALLOWED', 'HOST_BLOCKED', 'USE_REDDIT_SEARCH', 'SSRF_BLOCKED']);
 
@@ -159,7 +174,7 @@ export class MapSiteTool {
       // Try to fetch sitemap first
       if (validated.include_sitemap) {
         const sitemapUrls = await this.fetchSitemapUrls(baseUrl, domainFilter, poolLimit, scopePath);
-        sitemapUrls.forEach(url => urls.add(normalizeUrl(url)));
+        sitemapUrls.forEach(url => urls.add(normalizeMapUrl(url)));
       }
 
       // Fetch and parse the main page for additional URLs. A start page that
@@ -170,11 +185,11 @@ export class MapSiteTool {
         // The start page is a page of the site too: a one-page site with no
         // sitemap used to map to 0 URLs (R25 1.2).
         if (!domainFilter || domainFilter.isAllowed(validated.url).allowed) {
-          urls.add(normalizeUrl(validated.url));
+          urls.add(normalizeMapUrl(validated.url));
         }
         pageUrls.forEach(url => {
           if (urls.size < poolLimit) {
-            urls.add(normalizeUrl(url));
+            urls.add(normalizeMapUrl(url));
           }
         });
       } catch (error) {