html-get 2.21.2 → 2.22.0-0

package/package.json

@@ -2,7 +2,7 @@
   "name": "html-get",
   "description": "Get the HTML from any website, fine-tuned for correction & speed",
   "homepage": "https://nicedoc.com/microlinkhq/html-get",
-  "version": "2.21.2",
+  "version": "2.22.0-0",
   "main": "src/index.js",
   "bin": {
     "html-get": "bin/index.js"
@@ -82,18 +82,6 @@
     "scripts",
     "src"
   ],
-  "scripts": {
-    "clean": "rm -rf node_modules",
-    "contributors": "(npx git-authors-cli && npx finepack && git add package.json && git commit -m 'build: contributors' --no-verify) || true",
-    "lint": "standard",
-    "postinstall": "node scripts/postinstall",
-    "postrelease": "npm run release:tags && npm run release:github && (ci-publish || npm publish --access=public)",
-    "pretest": "npm run lint",
-    "release": "standard-version -a",
-    "release:github": "github-generate-release",
-    "release:tags": "git push --follow-tags origin HEAD:master",
-    "test": "c8 ava"
-  },
   "license": "MIT",
   "ava": {
     "files": [
@@ -122,11 +110,20 @@
       "finepack"
     ]
   },
-  "pnpm": {
-    "neverBuiltDependencies": []
-  },
   "simple-git-hooks": {
     "commit-msg": "npx commitlint --edit",
     "pre-commit": "npx nano-staged"
+  },
+  "scripts": {
+    "clean": "rm -rf node_modules",
+    "contributors": "(npx git-authors-cli && npx finepack && git add package.json && git commit -m 'build: contributors' --no-verify) || true",
+    "lint": "standard",
+    "postinstall": "node scripts/postinstall",
+    "postrelease": "npm run release:tags && npm run release:github && (ci-publish || npm publish --access=public)",
+    "pretest": "npm run lint",
+    "release": "standard-version -a",
+    "release:github": "github-generate-release",
+    "release:tags": "git push --follow-tags origin HEAD:master",
+    "test": "c8 ava"
   }
-}
+}

package/src/auto-domains.json

@@ -1 +1 @@
-[[["domainWithoutSuffix","google"]],[["domainWithoutSuffix","youtube"]],[["domainWithoutSuffix","apple"]],[["domainWithoutSuffix","wordpress"]],[["domainWithoutSuffix","microsoft"]],[["domainWithoutSuffix","blogspot"]],[["domainWithoutSuffix","wikipedia"]],[["domainWithoutSuffix","github"]],[["domainWithoutSuffix","vimeo"]],[["domainWithoutSuffix","twitter"]],[["domainWithoutSuffix","theguardian"]],[["domainWithoutSuffix","nytimes"]],[["domain","x.com"]],[["domainWithoutSuffix","bbc"]],[["domainWithoutSuffix","imdb"]],[["domainWithoutSuffix","huffingtonpost"]],[["domainWithoutSuffix","pinterest"]],[["domainWithoutSuffix","telegraph"]],[["domainWithoutSuffix","instagram"]],[["domainWithoutSuffix","spotify"]],[["domainWithoutSuffix","slideshare"]],[["domainWithoutSuffix","yelp"]],[["domainWithoutSuffix","eventbrite"]],[["domainWithoutSuffix","arxiv"]],[["domain","abc.net.au"]],[["domainWithoutSuffix","soundcloud"]],[["domainWithoutSuffix","engadget"]],[["domainWithoutSuffix","zoom"]],[["domainWithoutSuffix","theverge"]],[["domainWithoutSuffix","techcrunch"]],[["domainWithoutSuffix","flickr"]],[["domainWithoutSuffix","stackoverflow"]],[["domainWithoutSuffix","reddit"]],[["domainWithoutSuffix","csdn"]],[["domainWithoutSuffix","deviantart"]],[["domainWithoutSuffix","digg"]],[["domainWithoutSuffix","dribbble"]],[["domainWithoutSuffix","etsy"]],[["domainWithoutSuffix","ghost"]],[["domainWithoutSuffix","giphy"]],[["domainWithoutSuffix","gitlab"]],[["domainWithoutSuffix","imgur"]],[["domainWithoutSuffix","meetup"]],[["domainWithoutSuffix","producthunt"]],[["domainWithoutSuffix","sourceforge"]],[["domainWithoutSuffix","substack"]],[["domainWithoutSuffix","tumblr"]],[["domainWithoutSuffix","ycombinator"]]]
+[[["domainWithoutSuffix","google"]],[["domainWithoutSuffix","youtube"]],[["domainWithoutSuffix","wordpress"]],[["domainWithoutSuffix","apple"]],[["domainWithoutSuffix","microsoft"]],[["domainWithoutSuffix","wikipedia"]],[["domainWithoutSuffix","vimeo"]],[["domainWithoutSuffix","blogspot"]],[["domainWithoutSuffix","github"]],[["domain","x.com"]],[["domainWithoutSuffix","nytimes"]],[["domainWithoutSuffix","theguardian"]],[["domainWithoutSuffix","twitter"]],[["domainWithoutSuffix","bbc"]],[["domainWithoutSuffix","imdb"]],[["domainWithoutSuffix","slideshare"]],[["domainWithoutSuffix","telegraph"]],[["domainWithoutSuffix","pinterest"]],[["domainWithoutSuffix","spotify"]],[["domainWithoutSuffix","huffingtonpost"]],[["domainWithoutSuffix","instagram"]],[["domainWithoutSuffix","techcrunch"]],[["domainWithoutSuffix","zoom"]],[["domain","abc.net.au"]],[["domainWithoutSuffix","arxiv"]],[["domainWithoutSuffix","yelp"]],[["domainWithoutSuffix","eventbrite"]],[["domainWithoutSuffix","engadget"]],[["domainWithoutSuffix","soundcloud"]],[["domainWithoutSuffix","theverge"]],[["domainWithoutSuffix","digg"]],[["domainWithoutSuffix","imgur"]],[["domainWithoutSuffix","csdn"]],[["domainWithoutSuffix","deviantart"]],[["domainWithoutSuffix","dribbble"]],[["domainWithoutSuffix","etsy"]],[["domainWithoutSuffix","flickr"]],[["domainWithoutSuffix","ghost"]],[["domainWithoutSuffix","giphy"]],[["domainWithoutSuffix","gitlab"]],[["domainWithoutSuffix","meetup"]],[["domainWithoutSuffix","producthunt"]],[["domainWithoutSuffix","reddit"]],[["domainWithoutSuffix","sourceforge"]],[["domainWithoutSuffix","stackoverflow"]],[["domainWithoutSuffix","substack"]],[["domainWithoutSuffix","tumblr"]],[["domainWithoutSuffix","ycombinator"]]]

package/src/index.js

@@ -139,9 +139,9 @@ const prerender = PCancelable.fn(
         async (page, response) => {
           if (!response) throw new AbortError('empty response')
 
-          return {
+          const duration = debug.duration()
+          const payload = {
             headers: response.headers(),
-            html: await page.content(),
             mode: 'prerender',
             url: response.url(),
             statusCode: response.status(),
@@ -153,6 +153,11 @@ const prerender = PCancelable.fn(
                 url: req.url()
               }))
           }
+
+          duration('payload:basic')
+          payload.html = await page.content('html')
+          duration('payload:html')
+          return payload
         },
         {
           timeout,
@@ -161,9 +166,10 @@ const prerender = PCancelable.fn(
         }
       )
 
+      const duration = debug.duration('prerender')
       const payload = await getPayload(url, opts)
       await fetchRes.cancel()
-      debug('prerender', { url, state: 'success' })
+      duration({ url, state: 'success' })
       return payload
     } catch (err) {
       const { isRejected, ...dataProps } = await fetchRes
@@ -216,9 +222,7 @@ const defaultGetTemporalFile = (input, ext) => {
 const defaultMutool = () =>
   (() => {
     try {
-      const mutoolPath = execSync('which mutool', { stdio: 'pipe' })
-        .toString()
-        .trim()
+      const mutoolPath = execSync('which mutool').toString().trim()
       return (...args) => $(`${mutoolPath} draw -q -F html ${args}`)
     } catch (_) {}
   })()