@jocmp/mercury-parser 2.2.3-dev → 2.2.4

package/README.md

@@ -1,5 +1,7 @@
 # Mercury Parser - Extracting content from chaos
 
+![Tests](https://github.com/jocmp/mercury-parser/actions/workflows/ci.yaml/badge.svg)
+
 Mercury Parser extracts the bits that humans care about from any URL you give it. That includes article content, titles, authors, published dates, excerpts, lead images, and more.
 
 Mercury Parser allows you to easily create custom parsers using simple JavaScript and CSS selectors. This allows you to proactively manage parsing and migration edge cases. There are [many examples available](https://github.com/jocmp/mercury-parser/tree/master/src/extractors/custom) along with [documentation](https://github.com/jocmp/mercury-parser/blob/master/src/extractors/custom/README.md).
@@ -54,7 +56,7 @@ If Parser is unable to find a field, that field will return `null`.
 
 ##### Content Formats
 
-By default, Postlight Parser returns the `content` field as HTML. However, you can override this behavior by passing in options to the `parse` function, specifying whether or not to scrape all pages of an article, and what type of output to return (valid values are `'html'`, `'markdown'`, and `'text'`). For example:
+By default, Mercury Parser returns the `content` field as HTML. However, you can override this behavior by passing in options to the `parse` function, specifying whether or not to scrape all pages of an article, and what type of output to return (valid values are `'html'`, `'markdown'`, and `'text'`). For example:
 
 ```javascript
 Parser.parse(url, { contentType: 'markdown' }).then(result =>
@@ -84,7 +86,7 @@ Parser.parse(url, {
 
 ##### Pre-fetched HTML
 
-You can use Postlight Parser to parse custom or pre-fetched HTML by passing an HTML string to the `parse` function as follows:
+You can use Mercury Parser to parse custom or pre-fetched HTML by passing an HTML string to the `parse` function as follows:
 
 ```javascript
 Parser.parse(url, {
@@ -97,12 +99,12 @@ Note that the URL argument is still supplied, in order to identify the web site
 
 #### The command-line parser
 
-Postlight Parser also ships with a CLI, meaning you can use it from your command line like so:
+Mercury Parser also ships with a CLI, meaning you can use it from your command line like so:
 
-![Postlight Parser CLI Basic Usage](./assets/parser-basic-usage.gif)
+![Mercury Parser CLI Basic Usage](./assets/parser-basic-usage.gif)
 
 ```bash
-# Install Postlight Parser globally
+# Install Mercury Parser globally
 yarn global add @jocmp/mercury-parser
 #   or
 npm -g install @jocmp/mercury-parser
@@ -140,10 +142,6 @@ Licensed under either of the below, at your preference:
 
 ## Contributing
 
-For details on how to contribute to Postlight Parser, including how to write a custom content extractor for any site, see [CONTRIBUTING.md](./CONTRIBUTING.md)
+For details on how to contribute to Mercury Parser, including how to write a custom content extractor for any site, see [CONTRIBUTING.md](./CONTRIBUTING.md)
 
 Unless it is explicitly stated otherwise, any contribution intentionally submitted for inclusion in the work, as defined in the Apache-2.0 license, shall be dual licensed as above without any additional terms or conditions.
-
----
-
-🔬 A Labs project from your friends at [Postlight](https://postlight.com). Happy coding!

package/dist/mercury.js

@@ -6,7 +6,7 @@ var _regeneratorRuntime = _interopDefault(require('@babel/runtime-corejs2/regene
 var _objectSpread = _interopDefault(require('@babel/runtime-corejs2/helpers/objectSpread'));
 var _objectWithoutProperties = _interopDefault(require('@babel/runtime-corejs2/helpers/objectWithoutProperties'));
 var _asyncToGenerator = _interopDefault(require('@babel/runtime-corejs2/helpers/asyncToGenerator'));
-var URL = _interopDefault(require('url'));
+var URL$1 = _interopDefault(require('url'));
 var cheerio = _interopDefault(require('cheerio'));
 var TurndownService = _interopDefault(require('turndown'));
 var iconv = _interopDefault(require('iconv-lite'));
@@ -24,7 +24,6 @@ var _getIterator = _interopDefault(require('@babel/runtime-corejs2/core-js/get-i
 var _Object$assign = _interopDefault(require('@babel/runtime-corejs2/core-js/object/assign'));
 var _Object$keys = _interopDefault(require('@babel/runtime-corejs2/core-js/object/keys'));
 var stringDirection = _interopDefault(require('string-direction'));
-var validUrl = _interopDefault(require('valid-url'));
 var moment = _interopDefault(require('moment-timezone'));
 var parseFormat = _interopDefault(require('moment-parseformat'));
 var wuzzy = _interopDefault(require('wuzzy'));
@@ -120,7 +119,7 @@ function isGoodSegment(segment, index, firstSegmentHasLetters) {
 
 
 function articleBaseUrl(url, parsed) {
-  var parsedUrl = parsed || URL.parse(url);
+  var parsedUrl = parsed || URL$1.parse(url);
   var protocol = parsedUrl.protocol,
       host = parsedUrl.host,
       path = parsedUrl.path;
@@ -273,7 +272,7 @@ function _fetchResource() {
   _regeneratorRuntime.mark(function _callee(url, parsedUrl) {
     var headers,
         options,
-        _ref2,
+        _yield$get,
         response,
         body,
         _args = arguments;
@@ -283,7 +282,7 @@ function _fetchResource() {
         switch (_context.prev = _context.next) {
           case 0:
             headers = _args.length > 2 && _args[2] !== undefined ? _args[2] : {};
-            parsedUrl = parsedUrl || URL.parse(encodeURI(url));
+            parsedUrl = parsedUrl || URL$1.parse(encodeURI(url));
             options = _objectSpread({
               url: parsedUrl.href,
               headers: _objectSpread({}, REQUEST_HEADERS, headers),
@@ -305,9 +304,9 @@ function _fetchResource() {
             return get(options);
 
           case 5:
-            _ref2 = _context.sent;
-            response = _ref2.response;
-            body = _ref2.body;
+            _yield$get = _context.sent;
+            response = _yield$get.response;
+            body = _yield$get.body;
             _context.prev = 8;
             validateResponse(response);
             return _context.abrupt("return", {
@@ -637,7 +636,7 @@ function markToKeep(article, $, url) {
   }
 
   if (url) {
-    var _URL$parse = URL.parse(url),
+    var _URL$parse = URL$1.parse(url),
         protocol = _URL$parse.protocol,
         hostname = _URL$parse.hostname;
 
@@ -1250,7 +1249,7 @@ function absolutize($, rootUrl, attr) {
     var attrs = getAttrs(node);
     var url = attrs[attr];
     if (!url) return;
-    var absoluteUrl = URL.resolve(baseUrl || rootUrl, url);
+    var absoluteUrl = URL$1.resolve(baseUrl || rootUrl, url);
     setAttr(node, attr, absoluteUrl);
   });
 }
@@ -1270,7 +1269,7 @@ function absolutizeSet($, rootUrl, $content) {
         // a candidate URL cannot start or end with a comma
         // descriptors are separated from the URLs by unescaped whitespace
         var parts = candidate.trim().replace(/,$/, '').split(/\s+/);
-        parts[0] = URL.resolve(rootUrl, parts[0]);
+        parts[0] = URL$1.resolve(rootUrl, parts[0]);
         return parts.join(' ');
       });
 
@@ -2024,13 +2023,16 @@ var NewYorkerExtractor = {
 var WiredExtractor = {
   domain: 'www.wired.com',
   title: {
-    selectors: ['h1[data-testId="ContentHeaderHed"]']
+    selectors: ['h1[data-testId="ContentHeaderHed"]' // enter title selectors
+    ]
   },
   author: {
-    selectors: [['meta[name="article:author"]', 'value'], 'a[rel="author"]']
+    selectors: [['meta[name="article:author"]', 'value'], 'a[rel="author"]' // enter author selectors
+    ]
   },
   content: {
-    selectors: ['article.article.main-content', 'article.content'],
+    selectors: ['article.article.main-content', 'article.content' // enter content selectors
+    ],
     // Is there anything in the content you selected that needs transformed
     // before it's consumable content? E.g., unusual lazy loaded images
     transforms: [],
@@ -2058,13 +2060,16 @@ var WiredExtractor = {
 var MSNExtractor = {
   domain: 'www.msn.com',
   title: {
-    selectors: ['h1']
+    selectors: ['h1' // enter title selectors
+    ]
   },
   author: {
-    selectors: ['span.authorname-txt']
+    selectors: ['span.authorname-txt' // enter author selectors
+    ]
   },
   content: {
-    selectors: ['div.richtext'],
+    selectors: ['div.richtext' // enter content selectors
+    ],
     // Is there anything in the content you selected that needs transformed
     // before it's consumable content? E.g., unusual lazy loaded images
     transforms: [],
@@ -2092,10 +2097,12 @@ var MSNExtractor = {
 var YahooExtractor = {
   domain: 'www.yahoo.com',
   title: {
-    selectors: ['header.canvas-header']
+    selectors: ['header.canvas-header' // enter title selectors
+    ]
   },
   author: {
-    selectors: ['span.provider-name']
+    selectors: ['span.provider-name' // enter author selectors
+    ]
   },
   content: {
     selectors: [// enter content selectors
@@ -2129,10 +2136,12 @@ var BuzzfeedExtractor = {
   domain: 'www.buzzfeed.com',
   supportedDomains: ['www.buzzfeednews.com'],
   title: {
-    selectors: ['h1.embed-headline-title']
+    selectors: ['h1.embed-headline-title' // enter title selectors
+    ]
   },
   author: {
-    selectors: ['a[data-action="user/username"]', 'byline__author', ['meta[name="author"]', 'value']]
+    selectors: ['a[data-action="user/username"]', 'byline__author', ['meta[name="author"]', 'value'] // enter author selectors
+    ]
   },
   content: {
     selectors: [['div[class^="featureimage_featureImageWrapper"]', '.js-subbuzz-wrapper'], ['.js-subbuzz-wrapper']],
@@ -2174,13 +2183,16 @@ var BuzzfeedExtractor = {
 var WikiaExtractor = {
   domain: 'fandom.wikia.com',
   title: {
-    selectors: ['h1.entry-title']
+    selectors: ['h1.entry-title' // enter title selectors
+    ]
   },
   author: {
-    selectors: ['.author vcard', '.fn']
+    selectors: ['.author vcard', '.fn' // enter author selectors
+    ]
   },
   content: {
-    selectors: ['.grid-content', '.entry-content'],
+    selectors: ['.grid-content', '.entry-content' // enter content selectors
+    ],
     // Is there anything in the content you selected that needs transformed
     // before it's consumable content? E.g., unusual lazy loaded images
     transforms: [],
@@ -2208,10 +2220,12 @@ var WikiaExtractor = {
 var LittleThingsExtractor = {
   domain: 'www.littlethings.com',
   title: {
-    selectors: ['h1[class*="PostHeader"]', 'h1.post-title']
+    selectors: ['h1[class*="PostHeader"]', 'h1.post-title' // enter title selectors
+    ]
   },
   author: {
-    selectors: ['div[class^="PostHeader__ScAuthorNameSection"]', ['meta[name="author"]', 'value']]
+    selectors: ['div[class^="PostHeader__ScAuthorNameSection"]', ['meta[name="author"]', 'value'] // enter author selectors
+    ]
   },
   content: {
     selectors: [// enter content selectors
@@ -2665,7 +2679,8 @@ var WwwThevergeComExtractor = {
     // Is there anything that is in the result that shouldn't be?
     // The clean selectors will remove anything that matches from
     // the result
-    clean: ['.aside', 'img.c-dynamic-image']
+    clean: ['.aside', 'img.c-dynamic-image' // images come from noscript transform
+    ]
   }
 };
 
@@ -3409,7 +3424,8 @@ var WwwThepoliticalinsiderComExtractor = {
     ]
   },
   lead_image_url: {
-    selectors: [['meta[name="og:image"]', 'value']]
+    selectors: [['meta[name="og:image"]', 'value'] // enter selectors
+    ]
   },
   content: {
     selectors: ['div#article-body'],
@@ -4787,7 +4803,8 @@ var WwwRedditComExtractor = {
   content: {
     selectors: [['div[data-test-id="post-content"] p'], // text post
     ['div[data-test-id="post-content"] a[target="_blank"]:not([data-click-id="timestamp"])', // external link
-    'div[data-test-id="post-content"] div[data-click-id="media"]'], // external link with media preview (YouTube, imgur album, etc...)
+    'div[data-test-id="post-content"] div[data-click-id="media"]' // embedded media
+    ], // external link with media preview (YouTube, imgur album, etc...)
     ['div[data-test-id="post-content"] div[data-click-id="media"]'], // Embedded media (Reddit video)
     ['div[data-test-id="post-content"] a'], // external link
     'div[data-test-id="post-content"]'],
@@ -5513,7 +5530,7 @@ var WiredJpExtractor = {
       'img[data-original]': function imgDataOriginal($node) {
         var dataOriginal = $node.attr('data-original');
         var src = $node.attr('src');
-        var url = URL.resolve(src, dataOriginal);
+        var url = URL$1.resolve(src, dataOriginal);
         $node.attr('src', url);
       }
     },
@@ -6138,6 +6155,54 @@ var WwwCbcCaExtractor = {
   }
 };
 
+var WwwVersantsComExtractor = {
+  domain: 'www.versants.com',
+  title: {
+    selectors: [['meta[name="og:title"]', 'value']]
+  },
+  author: {
+    selectors: [['meta[name="author"]', 'value']]
+  },
+  date_published: {
+    selectors: [['meta[name="article:published_time"]', 'value']]
+  },
+  lead_image_url: {
+    selectors: [['meta[name="og:image"]', 'value']]
+  },
+  content: {
+    selectors: ['.entry-content'],
+    clean: ['.adv-link', '.versa-target']
+  }
+};
+
+var Www1pezeshkComExtractor = {
+  domain: 'www.1pezeshk.com',
+  title: {
+    selectors: [['meta[name="og:title"]', 'value'], 'h1.post-title']
+  },
+  author: {
+    selectors: [['meta[name="author"]', 'value']]
+  },
+  date_published: {
+    selectors: [['meta[name="article:published_time"]', 'value']]
+  },
+  lead_image_url: {
+    selectors: [['.featured-area img', 'src']]
+  },
+  content: {
+    selectors: ['article > .entry-content'],
+    transforms: {
+      img: function img($node) {
+        $node.src = decodeURIComponent($node.src);
+      }
+    },
+    // Is there anything that is in the result that shouldn't be?
+    // The clean selectors will remove anything that matches from
+    // the result
+    clean: []
+  }
+};
+
 
 
 var CustomExtractors = /*#__PURE__*/Object.freeze({
@@ -6283,7 +6348,9 @@ var CustomExtractors = /*#__PURE__*/Object.freeze({
   SpektrumExtractor: SpektrumExtractor,
   PostlightComExtractor: PostlightComExtractor,
   WwwInvestmentexecutiveComExtractor: WwwInvestmentexecutiveComExtractor,
-  WwwCbcCaExtractor: WwwCbcCaExtractor
+  WwwCbcCaExtractor: WwwCbcCaExtractor,
+  WwwVersantsComExtractor: WwwVersantsComExtractor,
+  Www1pezeshkComExtractor: Www1pezeshkComExtractor
 });
 
 var Extractors = _Object$keys(CustomExtractors).reduce(function (acc, key) {
@@ -6327,13 +6394,11 @@ function cleanAuthor(author) {
 }
 
 function clean$1(leadImageUrl) {
-  leadImageUrl = leadImageUrl.trim();
-
-  if (validUrl.isWebUri(leadImageUrl)) {
-    return leadImageUrl;
+  try {
+    return new URL(leadImageUrl.trim()).toString();
+  } catch (_unused) {
+    return null;
   }
-
-  return null;
 }
 
 // Return None if the dek wasn't good enough.
@@ -6520,7 +6585,7 @@ function cleanDomainFromTitle(splitTitle, url) {
   //
   // Strip out the big TLDs - it just makes the matching a bit more
   // accurate. Not the end of the world if it doesn't strip right.
-  var _URL$parse = URL.parse(url),
+  var _URL$parse = URL$1.parse(url),
       host = _URL$parse.host;
 
   var nakedDomain = host.replace(DOMAIN_ENDINGS_RE, '');
@@ -7273,7 +7338,7 @@ function shouldScore(href, articleUrl, baseUrl, parsedUrl, linkText, previousUrl
 
   var hostname = parsedUrl.hostname;
 
-  var _URL$parse = URL.parse(href),
+  var _URL$parse = URL$1.parse(href),
       linkHost = _URL$parse.hostname; // Domain mismatch.
 
 
@@ -7355,7 +7420,7 @@ function scoreLinks(_ref) {
       $ = _ref.$,
       _ref$previousUrls = _ref.previousUrls,
       previousUrls = _ref$previousUrls === void 0 ? [] : _ref$previousUrls;
-  parsedUrl = parsedUrl || URL.parse(articleUrl);
+  parsedUrl = parsedUrl || URL$1.parse(articleUrl);
   var baseRegex = makeBaseRegex(baseUrl);
   var isWp = isWordpress($); // Loop through all links, looking for hints that they may be next-page
   // links. Things like having "page" in their textContent, className or
@@ -7418,7 +7483,7 @@ var GenericNextPageUrlExtractor = {
         parsedUrl = _ref.parsedUrl,
         _ref$previousUrls = _ref.previousUrls,
         previousUrls = _ref$previousUrls === void 0 ? [] : _ref$previousUrls;
-    parsedUrl = parsedUrl || URL.parse(url);
+    parsedUrl = parsedUrl || URL$1.parse(url);
     var articleUrl = removeAnchor(url);
     var baseUrl = articleBaseUrl(url, parsedUrl);
     var links = $('a[href]').toArray();
@@ -7454,7 +7519,7 @@ var GenericNextPageUrlExtractor = {
 var CANONICAL_META_SELECTORS = ['og:url'];
 
 function parseDomain(url) {
-  var parsedUrl = URL.parse(url);
+  var parsedUrl = URL$1.parse(url);
   var hostname = parsedUrl.hostname;
   return hostname;
 }
@@ -7624,7 +7689,7 @@ function detectByHtml($) {
 }
 
 function getExtractor(url, parsedUrl, $) {
-  parsedUrl = parsedUrl || URL.parse(url);
+  parsedUrl = parsedUrl || URL$1.parse(url);
   var _parsedUrl = parsedUrl,
       hostname = _parsedUrl.hostname;
   var baseDomain = hostname.split('.').slice(-2).join('.');
@@ -7846,6 +7911,12 @@ var RootExtractor = {
       };
     }
 
+    var extendedResults = {};
+
+    if (extractor.extend) {
+      extendedResults = selectExtendedTypes(extractor.extend, opts);
+    }
+
     var title = extractResult(_objectSpread({}, opts, {
       type: 'title'
     }));
@@ -7894,12 +7965,6 @@ var RootExtractor = {
         url = _ref3.url,
         domain = _ref3.domain;
 
-    var extendedResults = {};
-
-    if (extractor.extend) {
-      extendedResults = selectExtendedTypes(extractor.extend, opts);
-    }
-
     return _objectSpread({
       title: title,
       content: content,
@@ -8033,7 +8098,7 @@ var Parser = {
                 html = html || cheerio.html();
               }
 
-              parsedUrl = URL.parse(url);
+              parsedUrl = URL$1.parse(url);
 
               if (validateUrl(parsedUrl)) {
                 _context.next = 6;