@jocmp/mercury-parser 2.2.3-dev

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Postlight
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,149 @@
+# Mercury Parser - Extracting content from chaos
+
+Mercury Parser extracts the bits that humans care about from any URL you give it. That includes article content, titles, authors, published dates, excerpts, lead images, and more.
+
+Mercury Parser allows you to easily create custom parsers using simple JavaScript and CSS selectors. This allows you to proactively manage parsing and migration edge cases. There are [many examples available](https://github.com/jocmp/mercury-parser/tree/master/src/extractors/custom) along with [documentation](https://github.com/jocmp/mercury-parser/blob/master/src/extractors/custom/README.md).
+
+## How? Like this.
+
+### Installation
+
+```bash
+# If you're using yarn
+yarn add @jocmp/mercury-parser
+
+# If you're using npm
+npm install @jocmp/mercury-parser
+```
+
+### Usage
+
+```javascript
+import Parser from '@jocmp/mercury-parser';
+
+Parser.parse(url).then(result => console.log(result));
+
+// NOTE: When used in the browser, you can omit the URL argument
+// and simply run `Parser.parse()` to parse the current page.
+```
+
+The result looks like this:
+
+```json
+{
+  "title": "Thunder (mascot)",
+  "content": "... <p><b>Thunder</b> is the <a href=\"https://en.wikipedia.org/wiki/Stage_name\">stage name</a> for the...",
+  "author": "Wikipedia Contributors",
+  "date_published": "2016-09-16T20:56:00.000Z",
+  "lead_image_url": null,
+  "dek": null,
+  "next_page_url": null,
+  "url": "https://en.wikipedia.org/wiki/Thunder_(mascot)",
+  "domain": "en.wikipedia.org",
+  "excerpt": "Thunder Thunder is the stage name for the horse who is the official live animal mascot for the Denver Broncos",
+  "word_count": 4677,
+  "direction": "ltr",
+  "total_pages": 1,
+  "rendered_pages": 1
+}
+```
+
+If Parser is unable to find a field, that field will return `null`.
+
+#### `parse()` Options
+
+##### Content Formats
+
+By default, Postlight Parser returns the `content` field as HTML. However, you can override this behavior by passing in options to the `parse` function, specifying whether or not to scrape all pages of an article, and what type of output to return (valid values are `'html'`, `'markdown'`, and `'text'`). For example:
+
+```javascript
+Parser.parse(url, { contentType: 'markdown' }).then(result =>
+  console.log(result)
+);
+```
+
+This returns the the page's `content` as GitHub-flavored Markdown:
+
+```json
+"content": "...**Thunder** is the [stage name](https://en.wikipedia.org/wiki/Stage_name) for the..."
+```
+
+##### Custom Request Headers
+
+You can include custom headers in requests by passing name-value pairs to the `parse` function as follows:
+
+```javascript
+Parser.parse(url, {
+  headers: {
+    Cookie: 'name=value; name2=value2; name3=value3',
+    'User-Agent':
+      'Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1',
+  },
+}).then(result => console.log(result));
+```
+
+##### Pre-fetched HTML
+
+You can use Postlight Parser to parse custom or pre-fetched HTML by passing an HTML string to the `parse` function as follows:
+
+```javascript
+Parser.parse(url, {
+  html:
+    '<html><body><article><h1>Thunder (mascot)</h1><p>Thunder is the stage name for the horse who is the official live animal mascot for the Denver Broncos</p></article></body></html>',
+}).then(result => console.log(result));
+```
+
+Note that the URL argument is still supplied, in order to identify the web site and use its custom parser, if it has any, though it will not be used for fetching content.
+
+#### The command-line parser
+
+Postlight Parser also ships with a CLI, meaning you can use it from your command line like so:
+
+![Postlight Parser CLI Basic Usage](./assets/parser-basic-usage.gif)
+
+```bash
+# Install Postlight Parser globally
+yarn global add @jocmp/mercury-parser
+#   or
+npm -g install @jocmp/mercury-parser
+
+# Then
+postlight-parser https://postlight.com/trackchanges/mercury-goes-open-source
+
+# Pass optional --format argument to set content type (html|markdown|text)
+postlight-parser https://postlight.com/trackchanges/mercury-goes-open-source --format=markdown
+
+# Pass optional --header.name=value arguments to include custom headers in the request
+postlight-parser https://postlight.com/trackchanges/mercury-goes-open-source --header.Cookie="name=value; name2=value2; name3=value3" --header.User-Agent="Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1"
+
+# Pass optional --extend argument to add a custom type to the response
+postlight-parser https://postlight.com/trackchanges/mercury-goes-open-source --extend credit="p:last-child em"
+
+# Pass optional --extend-list argument to add a custom type with multiple matches
+postlight-parser https://postlight.com/trackchanges/mercury-goes-open-source --extend-list categories=".meta__tags-list a"
+
+# Get the value of attributes by adding a pipe to --extend or --extend-list
+postlight-parser https://postlight.com/trackchanges/mercury-goes-open-source --extend-list links=".body a|href"
+
+# Pass optional --add-extractor argument to add a custom extractor at runtime.
+postlight-parser https://postlight.com/trackchanges/mercury-goes-open-source --add-extractor ./src/extractors/fixtures/postlight.com/index.js
+```
+
+## License
+
+Licensed under either of the below, at your preference:
+
+- Apache License, Version 2.0
+  ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
+- MIT license
+  ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
+
+## Contributing
+
+For details on how to contribute to Postlight Parser, including how to write a custom content extractor for any site, see [CONTRIBUTING.md](./CONTRIBUTING.md)
+
+Unless it is explicitly stated otherwise, any contribution intentionally submitted for inclusion in the work, as defined in the Apache-2.0 license, shall be dual licensed as above without any additional terms or conditions.
+
+---
+
+🔬 A Labs project from your friends at [Postlight](https://postlight.com). Happy coding!

package/cli.js

@@ -0,0 +1,111 @@
+#!/usr/bin/env node
+/* eslint-disable */
+
+const Parser = require('./dist/mercury');
+const package_info = require('./package.json');
+const argv = require('yargs-parser')(process.argv.slice(2));
+
+const {
+  _: [url],
+  format,
+  f,
+  extend,
+  e,
+  extendList,
+  l,
+  header,
+  h,
+  addExtractor,
+  x,
+  version,
+} = argv;
+(async (
+  urlToParse,
+  contentType,
+  extendedTypes,
+  extendedListTypes,
+  headers,
+  addExtractor,
+  version
+) => {
+  if (version) {
+    console.log(package_info.version);
+    process.exit(0);
+  }
+
+  if (!urlToParse) {
+    console.log(
+      '\n\
+postlight-parser\n\n\
+    The Postlight Parser extracts semantic content from any url\n\n\
+Usage:\n\
+\n\
+    $ postlight-parser url-to-parse [--format=html|text|markdown] [--header.name=value]... [--extend type=selector]... [--extend-list type=selector]... [--add-extractor path_to_extractor.js]... \n\
+\n\
+'
+    );
+    return;
+  }
+  try {
+    const contentTypeMap = {
+      html: 'html',
+      markdown: 'markdown',
+      md: 'markdown',
+      text: 'text',
+      txt: 'text',
+    };
+
+    const extensions = {};
+    [].concat(extendedTypes || []).forEach(t => {
+      const [name, selector] = t.split('=');
+      const fullSelector =
+        selector.indexOf('|') > 0 ? selector.split('|') : selector;
+      extensions[name] = { selectors: [fullSelector] };
+    });
+    [].concat(extendedListTypes || []).forEach(t => {
+      const [name, selector] = t.split('=');
+      const fullSelector =
+        selector.indexOf('|') > 0 ? selector.split('|') : selector;
+      extensions[name] = {
+        selectors: [fullSelector],
+        allowMultiple: true,
+      };
+    });
+
+    // Attempt to load custom extractor from path.
+    let customExtractor;
+    if (addExtractor) {
+      customExtractor = require(addExtractor);
+    }
+
+    const result = await Parser.parse(urlToParse, {
+      contentType: contentTypeMap[contentType],
+      extend: extensions,
+      headers,
+      customExtractor,
+    });
+    console.log(JSON.stringify(result, null, 2));
+  } catch (e) {
+    if (e.message === 'ETIMEDOUT' && false) {
+      console.error(
+        '\nPostlight Parser encountered a timeout trying to load that resource.'
+      );
+    } else {
+      console.error(
+        '\nPostlight Parser encountered a problem trying to parse that resource.\n'
+      );
+      console.error(e);
+    }
+    const reportBug =
+      'If you believe this was an error, please file an issue at:\n\n    https://github.com/postlight/parser/issues/new';
+    console.error(`\n${reportBug}\n`);
+    process.exit(1);
+  }
+})(
+  url,
+  format || f,
+  extend || e,
+  extendList || l,
+  header || h,
+  addExtractor || x
+);