@unavatar/core 3.7.66 → 3.7.68

package/bin/index.js

@@ -7,11 +7,26 @@ const mri = require('mri')
 
 module.exports = ({ baseUrl }) => {
   const { _, ...flags } = mri(process.argv.slice(2), {
-    default: {}
+    default: {},
+    alias: { H: 'header' }
   })
 
   const [input] = _
 
+  const parseHeaders = raw => {
+    const entries = raw == null ? [] : Array.isArray(raw) ? raw : [raw]
+    return entries.reduce((headers, entry) => {
+      const idx = entry.indexOf(':')
+      if (idx === -1) return headers
+      const name = entry.slice(0, idx).trim()
+      const value = entry.slice(idx + 1).trim()
+      if (name) headers[name.toLowerCase()] = value
+      return headers
+    }, {})
+  }
+
+  const customHeaders = parseHeaders(flags.header)
+
   if (!input) {
     console.error('Usage: unavatar <input> | unavatar <provider>/<key> | unavatar ping')
     console.error(
@@ -87,7 +102,10 @@ module.exports = ({ baseUrl }) => {
     }
   }
 
-  got(apiUrl.toString(), { headers: { 'x-api-key': flags.apiKey }, responseType: 'json' })
+  const requestHeaders = { ...customHeaders }
+  if (flags.apiKey) requestHeaders['x-api-key'] = flags.apiKey
+
+  got(apiUrl.toString(), { headers: requestHeaders, responseType: 'json' })
     .then(({ body, headers, statusCode, timings }) => {
       apiHeaders = headers
       apiStatusCode = statusCode

package/package.json

@@ -2,7 +2,7 @@
   "name": "@unavatar/core",
   "description": "Get unified user avatar from social networks, including Instagram, SoundCloud, Telegram, Twitter, YouTube & more.",
   "homepage": "https://unavatar.io",
-  "version": "3.7.66",
+  "version": "3.7.68",
   "main": "src/index.js",
   "exports": {
     ".": "./src/index.js",

package/src/providers/instagram.js

@@ -1,8 +1,11 @@
 'use strict'
 
+const randomCrawlerAgent = require('../util/crawler-agent')
+
 module.exports = ({ createHtmlProvider, getOgImage }) =>
   createHtmlProvider({
     name: 'instagram',
     url: input => `https://www.instagram.com/${input}`,
-    getter: getOgImage
+    getter: getOgImage,
+    htmlOpts: () => ({ headers: { 'user-agent': randomCrawlerAgent() } })
   })

package/src/providers/patreon.js

@@ -2,9 +2,37 @@
 
 const { $jsonld } = require('@metascraper/helpers')
 
-module.exports = ({ createHtmlProvider }) =>
+const unescapeUnicode = str =>
+  str.replace(/\\u([0-9a-fA-F]{4})/g, (_, hex) =>
+    String.fromCharCode(parseInt(hex, 16))
+  )
+
+const getRscAvatar = $ => {
+  let url
+  $('script').each((_, el) => {
+    const text = $(el).html() || ''
+    if (!text.includes('avatarPhotoImageUrls')) return
+    const match = text.match(
+      /avatarPhotoImageUrls[\s\S]*?\\"original\\":\\"((?:[^\\"]|\\u[0-9a-fA-F]{4})+)/
+    )
+    if (match) {
+      url = unescapeUnicode(match[1])
+      return false
+    }
+  })
+  return url
+}
+
+const getAvatar = $ =>
+  $jsonld('mainEntity.image.contentUrl')($) || getRscAvatar($)
+
+const factory = ({ createHtmlProvider }) =>
   createHtmlProvider({
     name: 'patreon',
     url: username => `https://www.patreon.com/${username}`,
-    getter: $ => $jsonld('mainEntity.image.thumbnailUrl')($)
+    getter: getAvatar
   })
+
+factory.getAvatar = getAvatar
+
+module.exports = factory

package/src/providers/x.js

@@ -1,11 +1,8 @@
 'use strict'
 
-const uniqueRandomArray = require('unique-random-array')
 const { $jsonld } = require('@metascraper/helpers')
 
-const randomCrawlerAgent = uniqueRandomArray(
-  require('top-crawler-agents').filter(agent => agent.startsWith('Slackbot'))
-)
+const randomCrawlerAgent = require('../util/crawler-agent')
 
 const toHighResolution = url => {
   if (url?.endsWith('_200x200.jpg')) {

package/src/util/crawler-agent.js

@@ -0,0 +1,8 @@
+'use strict'
+
+const uniqueRandomArray = require('unique-random-array')
+
+// TODO: update top-crawler-agents to don't make necessary to filter.
+module.exports = uniqueRandomArray(
+  require('top-crawler-agents').filter(agent => agent.startsWith('Slackbot'))
+)

package/src/util/html-provider.js

@@ -35,6 +35,11 @@ module.exports = ({ PROXY_TIMEOUT, getHTML, onFetchHTML }) => {
 
         const { $, statusCode } = await getHTML(providerUrl, fetchOpts)
 
+        attempt.lastHtml =
+          typeof $ === 'function' && typeof $.html === 'function'
+            ? $.html()
+            : undefined
+
         if (isStatusCodeMissing(statusCode)) {
           log.error({ statusCode, status: 'missing_status_code' })
           throw createEmptyProviderValueError({ provider: name, statusCode })
@@ -51,10 +56,7 @@ module.exports = ({ PROXY_TIMEOUT, getHTML, onFetchHTML }) => {
             provider: name,
             statusCode
           })
-          error.html =
-            typeof $ === 'function' && typeof $.html === 'function'
-              ? $.html()
-              : undefined
+          error.html = attempt.lastHtml
 
           log.error({ statusCode })