extract-webpage 1.2.419 → 1.2.421

package/src/html-to-cite/extract-date/date-validators.ts

@@ -1,191 +0,0 @@
-/**
- * @fileoverview Validation and filtering logic for extracted date candidates.
- * Ensures dates fall within plausible ranges and meet format requirements.
- */
-
-// Function cache
-var CACHE_SIZE = 8192;
-// earliest possible date to take into account (inclusive)
-var MIN_DATE = new Date(2002, 0, 0, 1); // Note: month is 0-indexed in JavaScript
-
-var DEBUG = 0;
-
-const LOGGER = DEBUG ? console : { debug: () => { }, log: () => { }, error: () => { }, warn: () => { } };
-
-function is_valid_date(date_input, outputformat, earliest, latest) {
-    // safety check
-    if (date_input === null) {
-        return false;
-    }
-    if (!earliest || !earliest.getFullYear)
-        earliest = new Date(2002, 0, 0, 1);
-    if (!latest)
-        latest = new Date();
-
-    let dateobject;
-    if (date_input instanceof Date) {
-        dateobject = date_input;
-    } else {
-        try {
-            if (outputformat === 'YYYY-MM-DD') {
-                dateobject = new Date(date_input.slice(0, 4), date_input.slice(5, 7) - 1, date_input.slice(8, 10));
-            } else {
-                dateobject = new Date(date_input);
-            }
-        } catch (error) {
-            return false;
-        }
-    }
-
-    if (
-        earliest.getFullYear() <= dateobject.getFullYear() &&
-        dateobject.getFullYear() <= latest.getFullYear() &&
-        earliest.getTime() <= dateobject.getTime() &&
-        dateobject.getTime() <= latest.getTime()
-    ) {
-        return true;
-    }
-    LOGGER.debug(`date not valid: ${date_input}`);
-    return false;
-}
-
-function is_valid_format(outputformat) {
-    // test with date object
-    var dateobject = new Date(2017, 8, 1, 0, 0);
-    try {
-        dateobject.toLocaleDateString('en-US', { year: 'numeric', month: '2-digit', day: '2-digit' });
-    } catch (err) {
-        LOGGER.error(`wrong output format or type: ${outputformat} ${err}`);
-        return false;
-    }
-    // test in abstracto (could be the only test)
-    if (typeof outputformat !== 'string' || !outputformat.includes('%')) {
-        LOGGER.error(`malformed output format: ${outputformat}`);
-        return false;
-    }
-    return true;
-}
-
-function plausible_year_filter(htmlstring, pattern, yearpat, earliest, latest, incomplete = false) {
-    var occurrences = new Map();
-
-
-    if (!htmlstring)
-        return occurrences;
-
-    var matches = htmlstring.match(new RegExp(pattern, 'g')) || [];
-
-    matches.forEach(item => {
-        var count = occurrences.get(item) || 0;
-        occurrences.set(item, count + 1);
-    });
-
-    for (var [item, count] of occurrences) {
-        var year_match = item.match(yearpat);
-        if (!year_match) {
-            LOGGER.debug(`not a year pattern: ${item}`);
-            occurrences.delete(item);
-            continue;
-        }
-
-        var lastdigits = year_match[1];
-        let potential_year;
-        if (!incomplete) {
-            potential_year = parseInt(lastdigits);
-        } else {
-            var century = lastdigits[0] === '9' ? '19' : '20';
-            potential_year = parseInt(century + lastdigits);
-        }
-
-        if (!(earliest.getFullYear() <= potential_year && potential_year <= latest.getFullYear())) {
-            LOGGER.debug(`no potential year: ${item}`);
-            occurrences.delete(item);
-        }
-    }
-
-    return occurrences;
-}
-
-function compare_values(reference, attempt, options) {
-    try {
-        var timestamp = new Date(attempt).getTime() / 1000;
-        if (options.original) {
-            reference = reference ? Math.min(reference, timestamp) : timestamp;
-        } else {
-            reference = Math.max(reference, timestamp);
-        }
-    } catch (err) {
-        LOGGER.debug(`Date parsing exception: ${err} for string ${attempt}`);
-    }
-    return reference;
-}
-
-function filter_ymd_candidate(bestmatch, pattern, original_date, copyear, outputformat, min_date, max_date) {
-    if (bestmatch) {
-        var pagedate = `${bestmatch[1]}-${bestmatch[2]}-${bestmatch[3]}`;
-        if (is_valid_date(pagedate, 'YYYY-MM-DD', min_date, max_date) &&
-            (copyear === 0 || parseInt(bestmatch[1]) >= copyear)) {
-            LOGGER.debug(`date found for pattern "${pattern}": ${pagedate}`);
-            return convert_date(pagedate, 'YYYY-MM-DD', outputformat);
-        }
-    }
-    return null;
-}
-
-function convert_date(datestring, inputformat, outputformat) {
-    if (inputformat === outputformat) {
-        return datestring;
-    }
-    if (datestring instanceof Date) {
-        return datestring.toLocaleDateString('en-US', { year: 'numeric', month: '2-digit', day: '2-digit' });
-    }
-    var dateobject = new Date(datestring);
-    return dateobject.toLocaleDateString('en-US', { year: 'numeric', month: '2-digit', day: '2-digit' });
-}
-
-function check_extracted_reference(reference, options) {
-    if (reference > 0) {
-        var dateobject = new Date(reference * 1000);
-        var converted = dateobject.toLocaleDateString('en-US', { year: 'numeric', month: '2-digit', day: '2-digit' });
-        if (is_valid_date(converted, options.format, options.min, options.max)) {
-            return converted;
-        }
-    }
-    return null;
-}
-
-function check_date_input(date_object, default_date) {
-    if (date_object instanceof Date) {
-        return date_object;
-    }
-    if (typeof date_object === 'string') {
-        try {
-            return new Date(date_object);
-        } catch (error) {
-            LOGGER.warn(`invalid datetime string: ${date_object}`);
-        }
-    }
-    return default_date;
-}
-
-function get_min_date(min_date) {
-    return check_date_input(min_date, MIN_DATE);
-}
-
-function get_max_date(max_date) {
-    return check_date_input(max_date, new Date());
-}
-
-
-export {
-    is_valid_date,
-    is_valid_format,
-    plausible_year_filter,
-    compare_values,
-    filter_ymd_candidate,
-    convert_date,
-    check_extracted_reference,
-    check_date_input,
-    get_min_date,
-    get_max_date
-};

package/src/html-to-cite/extract-date/extract-date-quick.ts

@@ -1,184 +0,0 @@
-/**
- * @fileoverview High-performance date extraction utility for HTML documents.
- * Prioritizes meta tags, specific selectors, and URL patterns for fast results.
- */
-// Constants
-const DATE_PUBLISHED_META_TAGS = [
-  'article:published_time',
-  'displaydate',
-  'dc.date',
-  'dc.date.issued',
-  'rbpubdate',
-  'publish_date',
-  'pub_date',
-  'pagedate',
-  'pubdate',
-  'revision_date',
-  'doc_date',
-  'date_created',
-  'content_create_date',
-  'lastmodified',
-  'created',
-  'date',
-];
-
-const DATE_PUBLISHED_SELECTORS = [
-  '.hentry .dtstamp.published',
-  '.hentry .published',
-  '.hentry .dtstamp.updated',
-  '.hentry .updated',
-  '.single .published',
-  '.meta .published',
-  '.meta .postDate',
-  '.entry-date',
-  '.byline .date',
-  '.postmetadata .date',
-  '.article_datetime',
-  '.date-header',
-  '.story-date',
-  '.dateStamp',
-  '#story .datetime',
-  '.dateline',
-  '.pubdate',
-];
-
-const abbrevMonthsStr = '(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)';
-const DATE_PUBLISHED_URL_RES = [
-  new RegExp('/(20\\d{2}/\\d{2}/\\d{2})/', 'i'),
-  new RegExp('(20\\d{2}-[01]\\d-[0-3]\\d)', 'i'),
-  new RegExp(`/(20\\d{2}/${abbrevMonthsStr}/[0-3]\\d)/`, 'i'),
-];
-
-const MS_DATE_STRING = /^\d{13}$/;
-const SEC_DATE_STRING = /^\d{10}$/;
-const CLEAN_DATE_STRING_RE = /^(.+?)(?: [a-z]{3})?(?::\d{2}| \d{4})? (?:[a-z]{3} )?[a-z]{3}$/i;
-const TIME_AGO_STRING = /(\d+)\s+(.*?)\s+ago/;
-const TIME_NOW_STRING = /^\s*(now|\d{1,2}\s*min(ute)?s?)\s*$/i;
-
-/**
- * Extract date from document using various methods
- *
- * @param {Document} document - DOM object with article content
- * @param {string} url - URL of the page
- * @returns {string|null} Extracted date or null if not found
- */
-export function extractDateQuick(document, url) {
-  let datePublished;
-
-  // Helper function to extract content from meta tags
-  function extractFromMeta(tags) {
-    for (const tag of tags) {
-      const metaTag = document.querySelector(`meta[name="${tag}"], meta[property="${tag}"]`);
-      if (metaTag) {
-        return metaTag.getAttribute('content');
-      }
-    }
-    return null;
-  }
-
-  // Helper function to extract content from selectors
-  function extractFromSelectors(selectors) {
-    for (const selector of selectors) {
-      const element = document.querySelector(selector);
-      if (element) {
-        return element.textContent.trim();
-      }
-    }
-    return null;
-  }
-
-  // 1. Check meta tags
-  datePublished = extractFromMeta(DATE_PUBLISHED_META_TAGS);
-  if (datePublished) return cleanDatePublished(datePublished);
-
-  // 2. Check selectors
-  datePublished = extractFromSelectors(DATE_PUBLISHED_SELECTORS);
-  if (datePublished) return cleanDatePublished(datePublished);
-
-  // 3. Check URL
-  for (const regex of DATE_PUBLISHED_URL_RES) {
-    const match = url.match(regex);
-    if (match) {
-      return cleanDatePublished(match[1]);
-    }
-  }
-
-  // 4. Check for specific text patterns
-  const textPatterns = ['Published', 'Updated', 'Posted'];
-  for (const pattern of textPatterns) {
-    for (const tag of ['time', 'span', 'div', 'p']) {
-      const elements = document.getElementsByTagName(tag);
-      for (const el of elements) {
-        if (el.textContent.includes(pattern)) {
-          datePublished = el.textContent.split(pattern)[1].trim();
-          if (datePublished) return cleanDatePublished(datePublished);
-        }
-      }
-    }
-  }
-
-  return null;
-}
-
-function cleanDatePublished(dateString) {
-  // If string is in milliseconds or seconds, convert to int and return
-  if (MS_DATE_STRING.test(dateString)) {
-    return new Date(parseInt(dateString, 10)).toISOString();
-  }
-  if (SEC_DATE_STRING.test(dateString)) {
-    return new Date(parseInt(dateString, 10) * 1000).toISOString();
-  }
-
-  dateString = cleanDateString(dateString);
-
-  return dateString
-}
-
-function cleanDateString(dateString) {
-  return dateString?.replace(CLEAN_DATE_STRING_RE, '$1').trim();
-}
-
-function createDate(dateString) {
-  if (TIME_AGO_STRING.test(dateString)) {
-    const [, amount, unit] = TIME_AGO_STRING.exec(dateString);
-    const now = new Date();
-    now.setMilliseconds(0);
-    switch (unit?.toLowerCase()) {
-      case 'second':
-      case 'seconds':
-        now.setSeconds(now.getSeconds() - parseInt(amount));
-        break;
-      case 'minute':
-      case 'minutes':
-        now.setMinutes(now.getMinutes() - parseInt(amount));
-        break;
-      case 'hour':
-      case 'hours':
-        now.setHours(now.getHours() - parseInt(amount));
-        break;
-      case 'day':
-      case 'days':
-        now.setDate(now.getDate() - parseInt(amount));
-        break;
-      case 'week':
-      case 'weeks':
-        now.setDate(now.getDate() - parseInt(amount) * 7);
-        break;
-      case 'month':
-      case 'months':
-        now.setMonth(now.getMonth() - parseInt(amount));
-        break;
-      case 'year':
-      case 'years':
-        now.setFullYear(now.getFullYear() - parseInt(amount));
-        break;
-    }
-    return now;
-  }
-
-  if (TIME_NOW_STRING.test(dateString)) {
-    return new Date();
-  }
-
-  return new Date(dateString);
-}