@unavatar/core 3.7.74 → 3.7.75

package/package.json

@@ -2,7 +2,7 @@
   "name": "@unavatar/core",
   "description": "Get unified user avatar from social networks, including Instagram, SoundCloud, Telegram, Twitter, YouTube & more.",
   "homepage": "https://unavatar.io",
-  "version": "3.7.74",
+  "version": "3.7.75",
   "main": "src/index.js",
   "exports": {
     ".": "./src/index.js",

package/src/providers/instagram.js

@@ -1,11 +1,8 @@
 'use strict'
 
-const randomCrawlerAgent = require('../util/crawler-agent')
-
 module.exports = ({ createHtmlProvider, getOgImage }) =>
   createHtmlProvider({
     name: 'instagram',
     url: input => `https://www.instagram.com/${input}`,
-    getter: getOgImage,
-    htmlOpts: () => ({ headers: { 'user-agent': randomCrawlerAgent() } })
+    getter: getOgImage
   })

package/src/providers/printables.js

@@ -1,7 +1,5 @@
 'use strict'
 
-const randomCrawlerAgent = require('../util/crawler-agent')
-
 module.exports = ({ createHtmlProvider, getOgImage }) =>
   createHtmlProvider({
     name: 'printables',
@@ -9,6 +7,5 @@ module.exports = ({ createHtmlProvider, getOgImage }) =>
       `https://www.printables.com/${
         input.startsWith('@') ? input : `@${input}`
       }`,
-    getter: getOgImage,
-    htmlOpts: () => ({ headers: { 'user-agent': randomCrawlerAgent() } })
+    getter: getOgImage
   })

package/src/providers/x.js

@@ -2,8 +2,6 @@
 
 const { $jsonld } = require('@metascraper/helpers')
 
-const randomCrawlerAgent = require('../util/crawler-agent')
-
 const toHighResolution = url => {
   if (url?.endsWith('_200x200.jpg')) {
     return url.replace('_200x200.jpg', '_400x400.jpg')
@@ -16,15 +14,15 @@ const toHighResolution = url => {
 
 const getProfileImage = $ =>
   toHighResolution(
-    $jsonld('mainEntity.image.contentUrl')($) || $('meta[property="og:image"]').attr('content')
+    $jsonld('mainEntity.image.contentUrl')($) ||
+      $('meta[property="og:image"]').attr('content')
   )
 
 const factory = ({ createHtmlProvider }) =>
   createHtmlProvider({
     name: 'x',
     url: input => `https://x.com/${input}`,
-    getter: getProfileImage,
-    htmlOpts: () => ({ headers: { 'user-agent': randomCrawlerAgent() } })
+    getter: getProfileImage
   })
 
 factory.getProfileImage = getProfileImage

package/src/util/crawler-agent.js

@@ -2,7 +2,4 @@
 
 const uniqueRandomArray = require('unique-random-array')
 
-// TODO: update top-crawler-agents to don't make necessary to filter.
-module.exports = uniqueRandomArray(
-  require('top-crawler-agents').filter(agent => agent.startsWith('Slackbot'))
-)
+module.exports = uniqueRandomArray(require('top-crawler-agents'))

package/src/util/html-provider.js

@@ -4,6 +4,7 @@ const { normalizeUrl } = require('@metascraper/helpers')
 const debug = require('debug-logfmt')('html-provider')
 const isAntibot = require('is-antibot')
 
+const randomCrawlerAgent = require('./crawler-agent')
 const httpStatus = require('./http-status')
 const ExtendableError = require('./error')
 
@@ -43,7 +44,15 @@ module.exports = ({ PROXY_TIMEOUT, getHTML, onFetchHTML }) => {
       const context = { provider: name, input, providerUrl }
 
       const attempt = async gotOpts => {
-        const defaultOpts = { ...htmlOpts?.(), timeout: PROXY_TIMEOUT }
+        const providerOpts = htmlOpts?.() ?? {}
+        const defaultOpts = {
+          ...providerOpts,
+          headers: {
+            'user-agent': randomCrawlerAgent(),
+            ...providerOpts.headers
+          },
+          timeout: PROXY_TIMEOUT
+        }
         const fetchOpts = gotOpts ? { ...defaultOpts, ...gotOpts } : defaultOpts
         const tier = fetchOpts.tier ?? 'origin'