metascraper-defuddle 5.50.7 → 5.51.0

package/README.md

@@ -14,6 +14,25 @@
 $ npm install metascraper-defuddle --save
 ```
 
+## Usage
+
+```js
+const metascraper = require('metascraper')([require('metascraper-defuddle')()])
+```
+
+It runs a single [Defuddle](https://github.com/kepano/defuddle) `parseInternal` pass — not the high-level `Defuddle()`, whose `parse()` re-runs on low `wordCount` to recover _content_. This connector extracts metadata, which is read from meta tags/schema regardless of content removals, so those retries add nothing here.
+
+The Defuddle result is memoized by HTML, so the same page is defuddled once even when several rules (or another consumer, such as a markdown renderer) request it; the same URL re-fetched with different HTML still re-extracts.
+
+### Options
+
+```js
+require('metascraper-defuddle')({ preprocess, defuddleOpts })
+```
+
+- **preprocess** `(document, html) => void` — mutate the parsed DOM before extraction (e.g. strip noise, normalize flattened shadow-DOM content).
+- **defuddleOpts** `object` — options forwarded to Defuddle's `parseInternal` (e.g. `{ removeLowScoring: false }` to keep link-heavy blocks).
+
 ## License
 
 **metascraper-defuddle** © [Microlink](https://microlink.io), released under the [MIT](https://github.com/microlinkhq/metascraper/blob/master/LICENSE.md) License.<br>

package/package.json

@@ -2,7 +2,7 @@
   "name": "metascraper-defuddle",
   "description": "A Defuddle connector for metascraper",
   "homepage": "https://github.com/microlinkhq/metascraper/packages/metascraper-defuddle",
-  "version": "5.50.7",
+  "version": "5.51.0",
   "types": "src/index.d.ts",
   "main": "src/index.js",
   "author": {
@@ -45,5 +45,5 @@
   "ava": {
     "timeout": "15s"
   },
-  "gitHead": "ff358c29107b437528930e31d76bb9d29fead3e2"
+  "gitHead": "20f037012985c0a65acfc5d15c5147082746ea86"
 }

package/src/index.d.ts

@@ -1,2 +1,7 @@
-declare function rules(): import('metascraper').Rules
+type Options = {
+  preprocess?: (document: Document, html: string) => void,
+  defuddleOpts?: import('defuddle/node').DefuddleOptions
+}
+
+declare function rules(options?: Options): import('metascraper').Rules
 export = rules

package/src/index.js

@@ -1,24 +1,46 @@
 'use strict'
 
-const { memoizeOne, composeRule, getHtml } = require('@metascraper/helpers')
+const { composeRule, getHtml } = require('@metascraper/helpers')
+const debug = require('debug-logfmt')('metascraper-defuddle')
 const asyncMemoizeOne = require('async-memoize-one')
 const { parseHTML } = require('linkedom')
 
-const debug = require('debug-logfmt')('metascraper-defuddle')
-
-const defuddleExtract = asyncMemoizeOne(async (url, html) => {
-  const { Defuddle } = await import('defuddle/node')
-  try {
-    const { document } = parseHTML(html)
-    return await Defuddle(document, url, { useAsync: false })
-  } catch (err) {
-    debug('Defuddle failed, fallback to next rule', { message: err.message })
-    return undefined
-  }
-}, memoizeOne.EqualityFirstArgument)
-
-module.exports = () => {
-  const getDefuddle = composeRule(($, url) => defuddleExtract(url, getHtml($)))
+// Cache key = HTML + url + the identity of `preprocess`/`defuddleOpts`, so a
+// call with the same HTML but a different hook or options (e.g. another
+// metascraper-defuddle() instance or defuddleExtract consumer) re-extracts
+// instead of reusing a result produced with different ones.
+const isSameExtraction = (
+  [html, url, opts = {}],
+  [oldHtml, oldUrl, oldOpts = {}]
+) =>
+  html === oldHtml &&
+  url === oldUrl &&
+  opts.preprocess === oldOpts.preprocess &&
+  opts.defuddleOpts === oldOpts.defuddleOpts
+
+// Single Defuddle `parseInternal` pass, memoized. See README for the
+// `preprocess` / `defuddleOpts` options and the rationale.
+const extractMemo = asyncMemoizeOne(
+  async (html, url, { preprocess, defuddleOpts } = {}) => {
+    const { DefuddleClass } = await import('defuddle/node')
+    try {
+      const { document } = parseHTML(html)
+      if (preprocess) preprocess(document, html)
+      return new DefuddleClass(document, { url }).parseInternal(defuddleOpts)
+    } catch (err) {
+      debug('Defuddle failed, fallback to next rule', { message: err.message })
+      return undefined
+    }
+  },
+  isSameExtraction
+)
+
+const defuddleExtract = (url, html, options) => extractMemo(html, url, options)
+
+module.exports = ({ preprocess, defuddleOpts } = {}) => {
+  const getDefuddle = composeRule(($, url) =>
+    defuddleExtract(url, getHtml($), { preprocess, defuddleOpts })
+  )
 
   const rules = {
     author: getDefuddle({ from: 'author' }),