@jaimevalasek/aioson 1.8.0 → 1.9.1

package/src/web.js

@@ -1,284 +1,284 @@
-'use strict';
-
-const DEFAULT_TIMEOUT_MS = 10000;
-const DEFAULT_MAX_HTML_CHARS = 500000;
-const DEFAULT_USER_AGENT = 'AIOSON-Web/1.0 (+https://aioson.dev)';
-
-function ensureValidUrl(input) {
-  try {
-    return new URL(String(input || '').trim());
-  } catch {
-    throw new Error(`Invalid URL: ${input || ''}`);
-  }
-}
-
-function normalizeUrl(input) {
-  const url = input instanceof URL ? new URL(input.toString()) : ensureValidUrl(input);
-  url.hash = '';
-  if ((url.protocol === 'http:' && url.port === '80') || (url.protocol === 'https:' && url.port === '443')) {
-    url.port = '';
-  }
-  const normalized = url.toString();
-  return normalized.endsWith('/') ? normalized.slice(0, -1) : normalized;
-}
-
-function decodeHtmlEntities(text) {
-  return String(text || '')
-    .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code)))
-    .replace(/&#x([0-9a-f]+);/gi, (_, code) => String.fromCodePoint(parseInt(code, 16)))
-    .replace(/ /gi, ' ')
-    .replace(/&/gi, '&')
-    .replace(/"/gi, '"')
-    .replace(/'/gi, "'")
-    .replace(/&lt;/gi, '<')
-    .replace(/&gt;/gi, '>');
-}
-
-function stripTags(html) {
-  return decodeHtmlEntities(String(html || '').replace(/<[^>]+>/g, ' '));
-}
-
-function cleanWhitespace(text) {
-  return String(text || '')
-    .replace(/\r/g, '')
-    .replace(/\t/g, ' ')
-    .replace(/[ \u00A0]+\n/g, '\n')
-    .replace(/\n[ \u00A0]+/g, '\n')
-    .replace(/[ \u00A0]{2,}/g, ' ')
-    .replace(/\n{3,}/g, '\n\n')
-    .trim();
-}
-
-function extractTagContent(html, tagName) {
-  const match = String(html || '').match(new RegExp(`<${tagName}[^>]*>([\\s\\S]*?)<\\/${tagName}>`, 'i'));
-  return match ? match[1] : '';
-}
-
-function extractTitle(html) {
-  return cleanWhitespace(stripTags(extractTagContent(html, 'title')));
-}
-
-function extractMetaContent(html, name) {
-  const match = String(html || '').match(new RegExp(`<meta[^>]+(?:name|property)=['\"]${name}['\"][^>]+content=['\"]([^'\"]+)['\"][^>]*>`, 'i'));
-  return match ? decodeHtmlEntities(match[1]).trim() : '';
-}
-
-function extractCanonical(html) {
-  const match = String(html || '').match(/<link[^>]+rel=['\"]canonical['\"][^>]+href=['\"]([^'\"]+)['\"][^>]*>/i);
-  return match ? match[1].trim() : '';
-}
-
-function extractMainHtml(html) {
-  const source = String(html || '');
-  const patterns = [
-    /<main\b[^>]*>([\s\S]*?)<\/main>/i,
-    /<article\b[^>]*>([\s\S]*?)<\/article>/i,
-    /<div\b[^>]*role=['\"]main['\"][^>]*>([\s\S]*?)<\/div>/i,
-    /<body\b[^>]*>([\s\S]*?)<\/body>/i
-  ];
-
-  for (const pattern of patterns) {
-    const match = source.match(pattern);
-    if (match && match[1]) return match[1];
-  }
-  return source;
-}
-
-function htmlToMarkdown(html) {
-  let output = String(html || '');
-  output = output.replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, ' ');
-  output = output.replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, ' ');
-  output = output.replace(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi, ' ');
-  output = output.replace(/<svg\b[^>]*>[\s\S]*?<\/svg>/gi, ' ');
-  output = output.replace(/<h1\b[^>]*>([\s\S]*?)<\/h1>/gi, '\n# $1\n\n');
-  output = output.replace(/<h2\b[^>]*>([\s\S]*?)<\/h2>/gi, '\n## $1\n\n');
-  output = output.replace(/<h3\b[^>]*>([\s\S]*?)<\/h3>/gi, '\n### $1\n\n');
-  output = output.replace(/<h4\b[^>]*>([\s\S]*?)<\/h4>/gi, '\n#### $1\n\n');
-  output = output.replace(/<h5\b[^>]*>([\s\S]*?)<\/h5>/gi, '\n##### $1\n\n');
-  output = output.replace(/<h6\b[^>]*>([\s\S]*?)<\/h6>/gi, '\n###### $1\n\n');
-  output = output.replace(/<li\b[^>]*>([\s\S]*?)<\/li>/gi, '\n- $1');
-  output = output.replace(/<(p|div|section|article|main|header|footer|aside|blockquote)\b[^>]*>/gi, '\n');
-  output = output.replace(/<\/(p|div|section|article|main|header|footer|aside|blockquote)>/gi, '\n');
-  output = output.replace(/<br\s*\/?>/gi, '\n');
-  output = output.replace(/<a\b[^>]*href=['\"]([^'\"]+)['\"][^>]*>([\s\S]*?)<\/a>/gi, '$2 ($1)');
-  output = stripTags(output);
-  return cleanWhitespace(output);
-}
-
-function htmlToText(html) {
-  return cleanWhitespace(stripTags(String(html || '')
-    .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, ' ')
-    .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, ' ')
-    .replace(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi, ' ')
-    .replace(/<br\s*\/?>/gi, '\n')
-    .replace(/<\/(p|div|section|article|main|header|footer|aside|li|ul|ol|h1|h2|h3|h4|h5|h6)>/gi, '\n')));
-}
-
-function resolveHref(href, pageUrl) {
-  const value = String(href || '').trim();
-  if (!value || value.startsWith('#') || value.startsWith('mailto:') || value.startsWith('tel:') || value.startsWith('javascript:')) {
-    return '';
-  }
-
-  try {
-    return normalizeUrl(new URL(value, pageUrl));
-  } catch {
-    return '';
-  }
-}
-
-function extractLinks(html, pageUrl, options = {}) {
-  const sameOriginOnly = options.sameOriginOnly !== false;
-  const base = ensureValidUrl(pageUrl);
-  const links = new Set();
-  const pattern = /<a\b[^>]*href=['\"]([^'\"]+)['\"][^>]*>/gi;
-  let match;
-
-  while ((match = pattern.exec(String(html || ''))) !== null) {
-    const normalized = resolveHref(match[1], base);
-    if (!normalized) continue;
-    if (sameOriginOnly) {
-      try {
-        if (new URL(normalized).origin !== base.origin) continue;
-      } catch {
-        continue;
-      }
-    }
-    links.add(normalized);
-  }
-
-  return Array.from(links).sort();
-}
-
-async function fetchPage(url, options = {}) {
-  const timeoutMs = Number(options.timeoutMs) > 0 ? Number(options.timeoutMs) : DEFAULT_TIMEOUT_MS;
-  const maxHtmlChars = Number(options.maxHtmlChars) > 0 ? Number(options.maxHtmlChars) : DEFAULT_MAX_HTML_CHARS;
-  const headers = {
-    'user-agent': options.userAgent || DEFAULT_USER_AGENT,
-    'accept': 'text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.8',
-    ...(options.headers || {})
-  };
-
-  const response = await fetch(String(url), {
-    method: 'GET',
-    headers,
-    redirect: 'follow',
-    signal: AbortSignal.timeout(timeoutMs)
-  });
-
-  const finalUrl = normalizeUrl(response.url || url);
-  const contentType = response.headers.get('content-type') || '';
-  const rawText = await response.text();
-  const html = rawText.length > maxHtmlChars ? rawText.slice(0, maxHtmlChars) : rawText;
-
-  return {
-    ok: response.ok,
-    url: finalUrl,
-    statusCode: response.status,
-    contentType,
-    html,
-    truncated: rawText.length > html.length
-  };
-}
-
-async function scrapePage(url, options = {}) {
-  const page = await fetchPage(url, options);
-  const mainHtml = extractMainHtml(page.html);
-  const title = extractTitle(page.html);
-  const description = extractMetaContent(page.html, 'description') || extractMetaContent(page.html, 'og:description');
-  const canonical = extractCanonical(page.html);
-  const links = extractLinks(page.html, page.url, { sameOriginOnly: options.sameOriginOnly !== false });
-  const text = htmlToText(mainHtml);
-  const markdown = htmlToMarkdown(mainHtml);
-
-  return {
-    ok: page.ok,
-    url: page.url,
-    statusCode: page.statusCode,
-    contentType: page.contentType,
-    title,
-    description,
-    canonical,
-    html: mainHtml,
-    text,
-    markdown,
-    links,
-    truncated: page.truncated
-  };
-}
-
-async function mapWebsite(url, options = {}) {
-  const startUrl = normalizeUrl(url);
-  const start = ensureValidUrl(startUrl);
-  const maxDepth = Number(options.maxDepth) >= 0 ? Number(options.maxDepth) : 2;
-  const maxPages = Number(options.maxPages) > 0 ? Number(options.maxPages) : 25;
-  const sameOriginOnly = options.sameOriginOnly !== false;
-
-  const queue = [{ url: startUrl, depth: 0 }];
-  const visited = new Set();
-  const pages = [];
-
-  while (queue.length > 0 && visited.size < maxPages) {
-    const current = queue.shift();
-    if (!current || visited.has(current.url)) continue;
-    visited.add(current.url);
-
-    try {
-      const page = await scrapePage(current.url, options);
-      const sameOriginLinks = extractLinks(page.html || '', page.url, { sameOriginOnly });
-      pages.push({
-        url: page.url,
-        depth: current.depth,
-        statusCode: page.statusCode,
-        title: page.title,
-        description: page.description,
-        linkCount: sameOriginLinks.length
-      });
-
-      if (current.depth >= maxDepth) continue;
-      for (const link of sameOriginLinks) {
-        if (visited.has(link)) continue;
-        if (sameOriginOnly) {
-          try {
-            if (new URL(link).origin != start.origin) continue;
-          } catch {
-            continue;
-          }
-        }
-        queue.push({ url: link, depth: current.depth + 1 });
-      }
-    } catch (error) {
-      pages.push({
-        url: current.url,
-        depth: current.depth,
-        statusCode: 0,
-        title: '',
-        description: '',
-        linkCount: 0,
-        error: error.message
-      });
-    }
-  }
-
-  return {
-    startUrl,
-    maxDepth,
-    maxPages,
-    pageCount: pages.length,
-    urls: pages.map((item) => item.url),
-    pages
-  };
-}
-
-module.exports = {
-  DEFAULT_TIMEOUT_MS,
-  DEFAULT_MAX_HTML_CHARS,
-  DEFAULT_USER_AGENT,
-  normalizeUrl,
-  extractLinks,
-  htmlToMarkdown,
-  htmlToText,
-  fetchPage,
-  scrapePage,
-  mapWebsite
-};
+'use strict';
+
+const DEFAULT_TIMEOUT_MS = 10000;
+const DEFAULT_MAX_HTML_CHARS = 500000;
+const DEFAULT_USER_AGENT = 'AIOSON-Web/1.0 (+https://aioson.dev)';
+
+function ensureValidUrl(input) {
+  try {
+    return new URL(String(input || '').trim());
+  } catch {
+    throw new Error(`Invalid URL: ${input || ''}`);
+  }
+}
+
+function normalizeUrl(input) {
+  const url = input instanceof URL ? new URL(input.toString()) : ensureValidUrl(input);
+  url.hash = '';
+  if ((url.protocol === 'http:' && url.port === '80') || (url.protocol === 'https:' && url.port === '443')) {
+    url.port = '';
+  }
+  const normalized = url.toString();
+  return normalized.endsWith('/') ? normalized.slice(0, -1) : normalized;
+}
+
+function decodeHtmlEntities(text) {
+  return String(text || '')
+    .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code)))
+    .replace(/&#x([0-9a-f]+);/gi, (_, code) => String.fromCodePoint(parseInt(code, 16)))
+    .replace(/&nbsp;/gi, ' ')
+    .replace(/&amp;/gi, '&')
+    .replace(/&quot;/gi, '"')
+    .replace(/&#39;/gi, "'")
+    .replace(/&lt;/gi, '<')
+    .replace(/&gt;/gi, '>');
+}
+
+function stripTags(html) {
+  return decodeHtmlEntities(String(html || '').replace(/<[^>]+>/g, ' '));
+}
+
+function cleanWhitespace(text) {
+  return String(text || '')
+    .replace(/\r/g, '')
+    .replace(/\t/g, ' ')
+    .replace(/[ \u00A0]+\n/g, '\n')
+    .replace(/\n[ \u00A0]+/g, '\n')
+    .replace(/[ \u00A0]{2,}/g, ' ')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+}
+
+function extractTagContent(html, tagName) {
+  const match = String(html || '').match(new RegExp(`<${tagName}[^>]*>([\\s\\S]*?)<\\/${tagName}>`, 'i'));
+  return match ? match[1] : '';
+}
+
+function extractTitle(html) {
+  return cleanWhitespace(stripTags(extractTagContent(html, 'title')));
+}
+
+function extractMetaContent(html, name) {
+  const match = String(html || '').match(new RegExp(`<meta[^>]+(?:name|property)=['\"]${name}['\"][^>]+content=['\"]([^'\"]+)['\"][^>]*>`, 'i'));
+  return match ? decodeHtmlEntities(match[1]).trim() : '';
+}
+
+function extractCanonical(html) {
+  const match = String(html || '').match(/<link[^>]+rel=['\"]canonical['\"][^>]+href=['\"]([^'\"]+)['\"][^>]*>/i);
+  return match ? match[1].trim() : '';
+}
+
+function extractMainHtml(html) {
+  const source = String(html || '');
+  const patterns = [
+    /<main\b[^>]*>([\s\S]*?)<\/main>/i,
+    /<article\b[^>]*>([\s\S]*?)<\/article>/i,
+    /<div\b[^>]*role=['\"]main['\"][^>]*>([\s\S]*?)<\/div>/i,
+    /<body\b[^>]*>([\s\S]*?)<\/body>/i
+  ];
+
+  for (const pattern of patterns) {
+    const match = source.match(pattern);
+    if (match && match[1]) return match[1];
+  }
+  return source;
+}
+
+function htmlToMarkdown(html) {
+  let output = String(html || '');
+  output = output.replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, ' ');
+  output = output.replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, ' ');
+  output = output.replace(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi, ' ');
+  output = output.replace(/<svg\b[^>]*>[\s\S]*?<\/svg>/gi, ' ');
+  output = output.replace(/<h1\b[^>]*>([\s\S]*?)<\/h1>/gi, '\n# $1\n\n');
+  output = output.replace(/<h2\b[^>]*>([\s\S]*?)<\/h2>/gi, '\n## $1\n\n');
+  output = output.replace(/<h3\b[^>]*>([\s\S]*?)<\/h3>/gi, '\n### $1\n\n');
+  output = output.replace(/<h4\b[^>]*>([\s\S]*?)<\/h4>/gi, '\n#### $1\n\n');
+  output = output.replace(/<h5\b[^>]*>([\s\S]*?)<\/h5>/gi, '\n##### $1\n\n');
+  output = output.replace(/<h6\b[^>]*>([\s\S]*?)<\/h6>/gi, '\n###### $1\n\n');
+  output = output.replace(/<li\b[^>]*>([\s\S]*?)<\/li>/gi, '\n- $1');
+  output = output.replace(/<(p|div|section|article|main|header|footer|aside|blockquote)\b[^>]*>/gi, '\n');
+  output = output.replace(/<\/(p|div|section|article|main|header|footer|aside|blockquote)>/gi, '\n');
+  output = output.replace(/<br\s*\/?>/gi, '\n');
+  output = output.replace(/<a\b[^>]*href=['\"]([^'\"]+)['\"][^>]*>([\s\S]*?)<\/a>/gi, '$2 ($1)');
+  output = stripTags(output);
+  return cleanWhitespace(output);
+}
+
+function htmlToText(html) {
+  return cleanWhitespace(stripTags(String(html || '')
+    .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, ' ')
+    .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, ' ')
+    .replace(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi, ' ')
+    .replace(/<br\s*\/?>/gi, '\n')
+    .replace(/<\/(p|div|section|article|main|header|footer|aside|li|ul|ol|h1|h2|h3|h4|h5|h6)>/gi, '\n')));
+}
+
+function resolveHref(href, pageUrl) {
+  const value = String(href || '').trim();
+  if (!value || value.startsWith('#') || value.startsWith('mailto:') || value.startsWith('tel:') || value.startsWith('javascript:')) {
+    return '';
+  }
+
+  try {
+    return normalizeUrl(new URL(value, pageUrl));
+  } catch {
+    return '';
+  }
+}
+
+function extractLinks(html, pageUrl, options = {}) {
+  const sameOriginOnly = options.sameOriginOnly !== false;
+  const base = ensureValidUrl(pageUrl);
+  const links = new Set();
+  const pattern = /<a\b[^>]*href=['\"]([^'\"]+)['\"][^>]*>/gi;
+  let match;
+
+  while ((match = pattern.exec(String(html || ''))) !== null) {
+    const normalized = resolveHref(match[1], base);
+    if (!normalized) continue;
+    if (sameOriginOnly) {
+      try {
+        if (new URL(normalized).origin !== base.origin) continue;
+      } catch {
+        continue;
+      }
+    }
+    links.add(normalized);
+  }
+
+  return Array.from(links).sort();
+}
+
+async function fetchPage(url, options = {}) {
+  const timeoutMs = Number(options.timeoutMs) > 0 ? Number(options.timeoutMs) : DEFAULT_TIMEOUT_MS;
+  const maxHtmlChars = Number(options.maxHtmlChars) > 0 ? Number(options.maxHtmlChars) : DEFAULT_MAX_HTML_CHARS;
+  const headers = {
+    'user-agent': options.userAgent || DEFAULT_USER_AGENT,
+    'accept': 'text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.8',
+    ...(options.headers || {})
+  };
+
+  const response = await fetch(String(url), {
+    method: 'GET',
+    headers,
+    redirect: 'follow',
+    signal: AbortSignal.timeout(timeoutMs)
+  });
+
+  const finalUrl = normalizeUrl(response.url || url);
+  const contentType = response.headers.get('content-type') || '';
+  const rawText = await response.text();
+  const html = rawText.length > maxHtmlChars ? rawText.slice(0, maxHtmlChars) : rawText;
+
+  return {
+    ok: response.ok,
+    url: finalUrl,
+    statusCode: response.status,
+    contentType,
+    html,
+    truncated: rawText.length > html.length
+  };
+}
+
+async function scrapePage(url, options = {}) {
+  const page = await fetchPage(url, options);
+  const mainHtml = extractMainHtml(page.html);
+  const title = extractTitle(page.html);
+  const description = extractMetaContent(page.html, 'description') || extractMetaContent(page.html, 'og:description');
+  const canonical = extractCanonical(page.html);
+  const links = extractLinks(page.html, page.url, { sameOriginOnly: options.sameOriginOnly !== false });
+  const text = htmlToText(mainHtml);
+  const markdown = htmlToMarkdown(mainHtml);
+
+  return {
+    ok: page.ok,
+    url: page.url,
+    statusCode: page.statusCode,
+    contentType: page.contentType,
+    title,
+    description,
+    canonical,
+    html: mainHtml,
+    text,
+    markdown,
+    links,
+    truncated: page.truncated
+  };
+}
+
+async function mapWebsite(url, options = {}) {
+  const startUrl = normalizeUrl(url);
+  const start = ensureValidUrl(startUrl);
+  const maxDepth = Number(options.maxDepth) >= 0 ? Number(options.maxDepth) : 2;
+  const maxPages = Number(options.maxPages) > 0 ? Number(options.maxPages) : 25;
+  const sameOriginOnly = options.sameOriginOnly !== false;
+
+  const queue = [{ url: startUrl, depth: 0 }];
+  const visited = new Set();
+  const pages = [];
+
+  while (queue.length > 0 && visited.size < maxPages) {
+    const current = queue.shift();
+    if (!current || visited.has(current.url)) continue;
+    visited.add(current.url);
+
+    try {
+      const page = await scrapePage(current.url, options);
+      const sameOriginLinks = extractLinks(page.html || '', page.url, { sameOriginOnly });
+      pages.push({
+        url: page.url,
+        depth: current.depth,
+        statusCode: page.statusCode,
+        title: page.title,
+        description: page.description,
+        linkCount: sameOriginLinks.length
+      });
+
+      if (current.depth >= maxDepth) continue;
+      for (const link of sameOriginLinks) {
+        if (visited.has(link)) continue;
+        if (sameOriginOnly) {
+          try {
+            if (new URL(link).origin != start.origin) continue;
+          } catch {
+            continue;
+          }
+        }
+        queue.push({ url: link, depth: current.depth + 1 });
+      }
+    } catch (error) {
+      pages.push({
+        url: current.url,
+        depth: current.depth,
+        statusCode: 0,
+        title: '',
+        description: '',
+        linkCount: 0,
+        error: error.message
+      });
+    }
+  }
+
+  return {
+    startUrl,
+    maxDepth,
+    maxPages,
+    pageCount: pages.length,
+    urls: pages.map((item) => item.url),
+    pages
+  };
+}
+
+module.exports = {
+  DEFAULT_TIMEOUT_MS,
+  DEFAULT_MAX_HTML_CHARS,
+  DEFAULT_USER_AGENT,
+  normalizeUrl,
+  extractLinks,
+  htmlToMarkdown,
+  htmlToText,
+  fetchPage,
+  scrapePage,
+  mapWebsite
+};