@unavatar/core 3.7.72 → 3.7.74

package/package.json

@@ -2,7 +2,7 @@
   "name": "@unavatar/core",
   "description": "Get unified user avatar from social networks, including Instagram, SoundCloud, Telegram, Twitter, YouTube & more.",
   "homepage": "https://unavatar.io",
-  "version": "3.7.72",
+  "version": "3.7.74",
   "main": "src/index.js",
   "exports": {
     ".": "./src/index.js",
@@ -132,6 +132,7 @@
     "html-get": "~2.22.0",
     "https-tls": "~1.0.24",
     "is-absolute-url": "~3.0.3",
+    "is-antibot": "~1.3.0",
     "is-email-like": "~1.0.0",
     "lodash": "~4.17.23",
     "ms": "~2.1.3",

package/src/providers/instagram.js

@@ -2,12 +2,10 @@
 
 const randomCrawlerAgent = require('../util/crawler-agent')
 
-const isBlocked = $ => $('title').text().includes('Login')
-
 module.exports = ({ createHtmlProvider, getOgImage }) =>
   createHtmlProvider({
     name: 'instagram',
     url: input => `https://www.instagram.com/${input}`,
-    getter: $ => !isBlocked($) && getOgImage($),
+    getter: getOgImage,
     htmlOpts: () => ({ headers: { 'user-agent': randomCrawlerAgent() } })
   })

package/src/providers/linkedin.js

@@ -4,5 +4,5 @@ module.exports = ({ createHtmlProvider, getOgImage }) =>
   createHtmlProvider({
     name: 'linkedin',
     url: input => `https://www.linkedin.com/in/${input}`,
-    getter: $ => getOgImage($) || false
+    getter: getOgImage
   })

package/src/util/html-provider.js

@@ -2,6 +2,7 @@
 
 const { normalizeUrl } = require('@metascraper/helpers')
 const debug = require('debug-logfmt')('html-provider')
+const isAntibot = require('is-antibot')
 
 const httpStatus = require('./http-status')
 const ExtendableError = require('./error')
@@ -18,7 +19,6 @@ const createEmptyProviderValueError = ({ provider, statusCode }) =>
     message: 'Empty value returned by the provider.'
   })
 
-
 const getOgImage = $ =>
   $('meta[property="og:image"]').attr('content') ||
   $('meta[name="og:image"]').attr('content')
@@ -28,14 +28,16 @@ module.exports = ({ PROXY_TIMEOUT, getHTML, onFetchHTML }) => {
    * @param {object} opts
    * @param {string} opts.name - Provider identifier used in logs and metrics.
    * @param {(input: string) => string | Promise<string>} opts.url - Builds the URL to fetch for a given input.
-   * @param {($: cheerio.CheerioAPI) => string | false | undefined} opts.getter
+   * @param {($: cheerio.CheerioAPI) => string | undefined} opts.getter
    *   Extracts the avatar URL from the fetched HTML.
    *   - `string`    — avatar URL found (success).
    *   - `undefined`  — avatar not found (normal failure, no retry).
-   *   - `false`     — page is blocked; error.blocked will be set to true.
+   * @param {($: cheerio.CheerioAPI) => boolean} [opts.isBlocked]
+   *   Optional provider-specific blocked-page detector, checked after the
+   *   default `is-antibot` check when getter returns empty/undefined.
    * @param {() => object} [opts.htmlOpts] - Returns extra options merged into the fetch call.
    */
-  const createHtmlProvider = ({ name, url, getter, htmlOpts }) => {
+  const createHtmlProvider = ({ name, url, getter, isBlocked, htmlOpts }) => {
     const provider = async function ({ input, req = {}, res = {} }) {
       const providerUrl = await url(input)
       const context = { provider: name, input, providerUrl }
@@ -47,7 +49,11 @@ module.exports = ({ PROXY_TIMEOUT, getHTML, onFetchHTML }) => {
 
         const log = debug.duration({ ...context, tier })
 
-        const { $, statusCode } = await getHTML(providerUrl, fetchOpts)
+        const {
+          $,
+          statusCode,
+          headers: responseHeaders = {}
+        } = await getHTML(providerUrl, fetchOpts)
 
         attempt.lastHtml =
           typeof $ === 'function' && typeof $.html === 'function'
@@ -71,9 +77,18 @@ module.exports = ({ PROXY_TIMEOUT, getHTML, onFetchHTML }) => {
             statusCode
           })
           error.html = attempt.lastHtml
-          if (result === false) error.blocked = true
 
-          log.error({ statusCode, status: result === false ? 'blocked' : undefined })
+          const { detected: antibotDetected } = isAntibot({
+            url: providerUrl,
+            headers: responseHeaders,
+            body: attempt.lastHtml
+          })
+          if (antibotDetected || isBlocked?.($) === true) error.blocked = true
+
+          log.error({
+            statusCode,
+            status: error.blocked ? 'blocked' : undefined
+          })
 
           throw error
         }