@unavatar/core 3.45.26 → 3.46.1

package/package.json

@@ -2,7 +2,7 @@
   "name": "@unavatar/core",
   "description": "Get unified user avatar from social networks, including Instagram, SoundCloud, Telegram, Twitter, YouTube & more.",
   "homepage": "https://unavatar.io",
-  "version": "3.45.26",
+  "version": "3.46.1",
   "main": "src/index.js",
   "exports": {
     ".": "./src/index.js",
@@ -128,6 +128,7 @@
     "@metascraper/helpers": "~5.52.0",
     "@microlink/mql": "~0.17.0",
     "@microlink/ping-url": "~1.4.16",
+    "bimi-url": "~1.0.0",
     "browserless": "~13.6.0",
     "cacheable-lookup": "~6.1.0",
     "data-uri-regex": "~0.1.4",

package/src/avatar/auto.js

@@ -8,6 +8,7 @@ const isEmail = require('is-email-like')
 const pTimeout = require('p-timeout')
 const pAny = require('p-any')
 
+const { RESERVED_ADDRESS_CODE } = require('../util/is-reserved-ip')
 const httpStatus = require('../util/http-status')
 const isIterable = require('../util/is-iterable')
 const ExtendableError = require('../util/error')
@@ -23,14 +24,28 @@ const getInputType = input => {
   return 'username'
 }
 
-const factory = ({ constants, providers, providersBy, reachableUrl }) => {
+const factory = ({
+  constants,
+  providers,
+  providerTiers,
+  reachableUrl,
+  isReservedIp
+}) => {
   const { REQUEST_TIMEOUT } = constants
-  const providerEntriesByType = Object.fromEntries(
-    Object.entries(providersBy).map(([inputType, providerNames]) => [
-      inputType,
-      providerNames.map(provider => [provider, providers[provider]])
-    ])
-  )
+
+  const refuseReservedAddress = provider => {
+    throw new ExtendableError({
+      message: 'The URL points to a reserved address.',
+      provider,
+      statusCode: httpStatus.FORBIDDEN
+    })
+  }
+
+  const assertPublicUrl = async (url, provider) => {
+    if (await isReservedIp(new URL(url).hostname)) {
+      refuseReservedAddress(provider)
+    }
+  }
 
   const getAvatarContent = provider => async output => {
     if (typeof output !== 'string' || output === '') {
@@ -55,7 +70,12 @@ const factory = ({ constants, providers, providersBy, reachableUrl }) => {
       })
     }
 
-    const { statusCode, url } = await reachableUrl(output)
+    await assertPublicUrl(output, provider)
+
+    const { statusCode, url, reservedAddress } = await reachableUrl(output)
+
+    if (reservedAddress) refuseReservedAddress(provider)
+    await assertPublicUrl(url, provider)
 
     if (!reachableUrl.isReachable({ statusCode })) {
       throw new ExtendableError({
@@ -68,43 +88,69 @@ const factory = ({ constants, providers, providersBy, reachableUrl }) => {
     return { type: 'url', data: url, provider }
   }
 
-  const getAvatar = async (fn, provider, input, context) => {
+  const getAvatar = async (
+    fn,
+    provider,
+    input,
+    context,
+    deadline = Date.now() + REQUEST_TIMEOUT
+  ) => {
     const promise = Promise.resolve(fn(input, context))
       .then(getAvatarContent(provider))
       .catch(error => {
         isIterable.forEach(error, error => {
+          if (error.code === RESERVED_ADDRESS_CODE) {
+            error.statusCode = httpStatus.FORBIDDEN
+          }
           error.statusCode = error.statusCode ?? error.response?.statusCode
           error.provider = provider
         })
         throw error
       })
 
-    return pTimeout(promise, REQUEST_TIMEOUT).catch(error => {
-      error.provider = provider
-      throw error
-    })
+    return pTimeout(promise, Math.max(0, deadline - Date.now())).catch(
+      error => {
+        error.provider = provider
+        throw error
+      }
+    )
   }
 
-  const resolveAutoByType = async (inputType, input, context) => {
-    let collection = providerEntriesByType[inputType]
+  const raceTier = (tier, input, context, deadline) =>
+    pAny(
+      tier.map(provider =>
+        getAvatar(providers[provider], provider, input, context, deadline)
+      )
+    )
 
-    if (inputType === 'email' && isHash(input)) {
-      collection = collection.filter(([provider]) => provider === 'gravatar')
+  const auto = inputType => async (input, context) => {
+    const tierKey =
+      inputType === 'email' && isHash(input) ? 'emailHash' : inputType
+    const tiers = providerTiers[tierKey]
+
+    if (!tiers?.length) {
+      throw new ExtendableError({
+        message: `No providers declared for \`${tierKey}\`.`,
+        statusCode: httpStatus.NOT_FOUND
+      })
     }
 
-    const promises = new Array(collection.length)
+    const deadline = Date.now() + REQUEST_TIMEOUT
+
+    let firstError
 
-    for (let index = 0; index < collection.length; index++) {
-      const [provider, fn] = collection[index]
-      promises[index] = getAvatar(fn, provider, input, context)
+    for (const tier of tiers) {
+      try {
+        return await raceTier(tier, input, context, deadline)
+      } catch (error) {
+        firstError ??= error
+        if (Date.now() >= deadline) break
+      }
     }
 
-    return pAny(promises)
+    throw firstError
   }
 
-  const auto = inputType => (input, context) =>
-    resolveAutoByType(inputType, input, context)
-
   return { auto, getInputType, getAvatar }
 }
 

package/src/index.js

@@ -22,12 +22,14 @@ module.exports = ({
     redis
   })
   const cache = require('./util/cache')({ createMultiCache, createRedisCache })
+  const dnsResolver = require('./util/dns-resolver')(constants)
   const cacheableLookup = require('./util/cacheable-lookup')({
     ...constants,
-    cache: cache.dnsCache
+    cache: cache.dnsCache,
+    resolver: dnsResolver
   })
   const isReservedIp = require('./util/is-reserved-ip')({ cacheableLookup })
-  const got = require('./util/got')({ cacheableLookup })
+  const got = require('./util/got')({ cacheableLookup, isReservedIp })
   const reachableUrl = require('./util/reachable-url')({
     got,
     pingCache: cache.pingCache
@@ -50,16 +52,20 @@ module.exports = ({
     got,
     reachableUrl,
     isReservedIp,
+    dnsResolver,
+    bimiCache: cache.bimiCache,
     githubSearchCache: cache.githubSearchCache,
     itunesSearchCache: cache.itunesSearchCache
   }
-  const { providers, providersBy } = require('./providers')(providerCtx)
+  const { providers, providerTiers, providersBy } =
+    require('./providers')(providerCtx)
 
   const { auto, getInputType, getAvatar } = require('./avatar/auto')({
     constants,
     providers,
-    providersBy,
-    reachableUrl
+    providerTiers,
+    reachableUrl,
+    isReservedIp
   })
 
   const unavatar = input => auto(getInputType(input))(input, {})

package/src/providers/bimi.js

@@ -0,0 +1,19 @@
+'use strict'
+
+const createGetLogo = require('bimi-url')
+
+const parseInput = input => input.slice(input.lastIndexOf('@') + 1)
+
+module.exports = ({ bimiCache, dnsResolver, got }) => {
+  const getLogo = createGetLogo({
+    gotOpts: got.gotOpts,
+    keyvOpts: bimiCache,
+    resolveTxt: hostname => dnsResolver.resolveTxt(hostname)
+  })
+
+  return function bimi (input) {
+    return getLogo(parseInput(input))
+  }
+}
+
+module.exports.parseInput = parseInput

package/src/providers/index.js

@@ -1,86 +1,99 @@
 'use strict'
 
-const providersBy = {
-  email: ['gravatar', 'github'],
+const providerTiers = {
+  email: [['gravatar', 'github'], ['bimi']],
+  emailHash: [['gravatar']],
   username: [
-    'apple-music',
-    'apple-store',
-    'behance',
-    'bilibili',
-    'bluesky',
-    'buymeacoffee',
-    'cashapp',
-    'codepen',
-    'cults3d',
-    'cursor',
-    'deezer',
-    'deviantart',
-    'discord',
-    'dockerhub',
-    'dribbble',
-    'facebook',
-    'flickr',
-    'gitee',
-    'github',
-    'gitlab',
-    'google-play',
-    'huggingface',
-    'hevy',
-    'instagram',
-    'juejin',
-    'ko-fi',
-    'linkedin',
-    'mastodon',
-    'medium',
-    'netease-music',
-    'npm',
-    'onlyfans',
-    'openstreetmap',
-    'patreon',
-    'paypal',
-    'pinterest',
-    'printables',
-    'primal',
-    'producthunt',
-    'psnprofiles',
-    'qq',
-    'raycast',
-    'reddit',
-    'revolut',
-    'snapchat',
-    'soundcloud',
-    'spotify',
-    'stackoverflow',
-    'steam',
-    'strava',
-    'substack',
-    'telegram',
-    'thingiverse',
-    'threads',
-    'tidal',
-    'tieba',
-    'tiktok',
-    'tumblr',
-    'twitch',
-    'venmo',
-    'vimeo',
-    'weibo',
-    'whatsapp',
-    'wise',
-    'x',
-    'xboxgamertag',
-    'youtube',
-    'zhihu'
+    [
+      'apple-music',
+      'apple-store',
+      'behance',
+      'bilibili',
+      'bluesky',
+      'buymeacoffee',
+      'cashapp',
+      'codepen',
+      'cults3d',
+      'cursor',
+      'deezer',
+      'deviantart',
+      'discord',
+      'dockerhub',
+      'dribbble',
+      'facebook',
+      'flickr',
+      'gitee',
+      'github',
+      'gitlab',
+      'google-play',
+      'huggingface',
+      'hevy',
+      'instagram',
+      'juejin',
+      'ko-fi',
+      'linkedin',
+      'mastodon',
+      'medium',
+      'netease-music',
+      'npm',
+      'onlyfans',
+      'openstreetmap',
+      'patreon',
+      'paypal',
+      'pinterest',
+      'printables',
+      'primal',
+      'producthunt',
+      'psnprofiles',
+      'qq',
+      'raycast',
+      'reddit',
+      'revolut',
+      'snapchat',
+      'soundcloud',
+      'spotify',
+      'stackoverflow',
+      'steam',
+      'strava',
+      'substack',
+      'telegram',
+      'thingiverse',
+      'threads',
+      'tidal',
+      'tieba',
+      'tiktok',
+      'tumblr',
+      'twitch',
+      'venmo',
+      'vimeo',
+      'weibo',
+      'whatsapp',
+      'wise',
+      'x',
+      'xboxgamertag',
+      'youtube',
+      'zhihu'
+    ]
   ],
-  domain: ['duckduckgo', 'google', 'microlink']
+  domain: [['bimi'], ['duckduckgo', 'google', 'microlink']]
 }
 
+const { emailHash, ...publicTiers } = providerTiers
+
+const providersBy = Object.fromEntries(
+  Object.entries(publicTiers).map(([inputType, tiers]) => [
+    inputType,
+    tiers.flat()
+  ])
+)
+
 module.exports = ctx => {
   const providers = {
     'apple-music': require('./apple-music')(ctx),
     'apple-store': require('./apple-store')(ctx),
     behance: require('./behance')(ctx),
     bilibili: require('./bilibili')(ctx),
+    bimi: require('./bimi')(ctx),
     bluesky: require('./bluesky')(ctx),
     buymeacoffee: require('./buymeacoffee')(ctx),
     cashapp: require('./cashapp')(ctx),
@@ -151,5 +164,5 @@ module.exports = ctx => {
     zhihu: require('./zhihu')(ctx)
   }
 
-  return { providers, providersBy }
+  return { providers, providerTiers, providersBy }
 }

package/src/util/cache.js

@@ -3,11 +3,21 @@
 const ms = require('ms')
 
 module.exports = ({ createMultiCache, createRedisCache }) => ({
-  dnsCache: createMultiCache(createRedisCache({ namespace: 'dns', ttl: ms('1d') })),
+  bimiCache: createMultiCache(
+    createRedisCache({ namespace: 'bimi', ttl: ms('1d') })
+  ),
+  dnsCache: createMultiCache(
+    createRedisCache({ namespace: 'dns', ttl: ms('1d') })
+  ),
   githubSearchCache: createRedisCache({
     namespace: 'github-search',
     ttl: ms('1d')
   }),
-  pingCache: createMultiCache(createRedisCache({ namespace: 'ping', ttl: ms('1d') })),
-  itunesSearchCache: createRedisCache({ namespace: 'itunes-search', ttl: ms('7d') })
+  pingCache: createMultiCache(
+    createRedisCache({ namespace: 'ping', ttl: ms('1d') })
+  ),
+  itunesSearchCache: createRedisCache({
+    namespace: 'itunes-search',
+    ttl: ms('7d')
+  })
 })

package/src/util/cacheable-lookup.js

@@ -1,22 +1,10 @@
 'use strict'
 
 const CacheableLookup = require('cacheable-lookup')
-const Tangerine = require('tangerine')
 
-module.exports = ({ TTL_DEFAULT, DNS_TIMEOUT, DNS_SERVERS, cache }) =>
+module.exports = ({ TTL_DEFAULT, cache, resolver }) =>
   new CacheableLookup({
     maxTtl: TTL_DEFAULT,
     cache,
-    resolver: new Tangerine(
-      {
-        cache: false,
-        timeout: DNS_TIMEOUT,
-        servers: DNS_SERVERS
-      },
-      require('got').extend({
-        responseType: 'buffer',
-        decompress: false,
-        retry: 0
-      })
-    )
+    resolver
   })

package/src/util/dns-resolver.js

@@ -0,0 +1,18 @@
+'use strict'
+
+const Tangerine = require('tangerine')
+const got = require('got')
+
+module.exports = ({ DNS_TIMEOUT, DNS_SERVERS }) =>
+  new Tangerine(
+    {
+      cache: false,
+      timeout: DNS_TIMEOUT,
+      servers: DNS_SERVERS
+    },
+    got.extend({
+      responseType: 'buffer',
+      decompress: false,
+      retry: 0
+    })
+  )

package/src/util/got.js

@@ -5,6 +5,8 @@ const tlsHook = require('https-tls/hook')
 const uaHints = require('ua-hints')
 const got = require('got')
 
+const { RESERVED_ADDRESS_CODE } = require('./is-reserved-ip')
+
 const topUserAgents = require('top-user-agents')
 const randomUserAgent = uniqueRandomArray(topUserAgents)
 
@@ -36,11 +38,40 @@ const userAgentHook = options => {
   }
 }
 
-module.exports = ({ cacheableLookup }) => {
+module.exports = ({ cacheableLookup, isReservedIp }) => {
+  /**
+   * The refused address is reported on `options.context` because a caller
+   * reading the outcome through `reachable-url` never sees the rejection: it is
+   * swallowed there and served back as a response.
+   *
+   * Only a context the caller owns is written: the one got defaults to is
+   * frozen and shared between requests, so asking to be told is what passing a
+   * context means.
+   *
+   * got replaces the error with a RequestError that keeps the message and the
+   * code, so the refusal is named by a code rather than by its type.
+   */
+  const reservedAddressHook = async options => {
+    const hostname = options.url?.hostname
+    if (hostname && (await isReservedIp(hostname))) {
+      if (Object.isExtensible(options.context)) {
+        options.context.reservedAddress = hostname
+      }
+      const error = new Error(
+        `Refusing to request a reserved address: ${hostname}`
+      )
+      error.code = RESERVED_ADDRESS_CODE
+      throw error
+    }
+  }
+
   const gotOpts = {
     dnsCache: cacheableLookup,
     https: { rejectUnauthorized: false },
-    hooks: { beforeRequest: [userAgentHook, tlsHook] }
+    hooks: {
+      beforeRequest: [reservedAddressHook, userAgentHook, tlsHook],
+      beforeRedirect: [reservedAddressHook]
+    }
   }
 
   const instance = got.extend(gotOpts)

package/src/util/is-reserved-ip.js

@@ -2,16 +2,16 @@
 
 const ip = require('ipaddr.js')
 
+const unbracket = hostname =>
+  hostname.startsWith('[') && hostname.endsWith(']')
+    ? hostname.slice(1, -1)
+    : hostname
+
 module.exports = ({ cacheableLookup }) => {
   const getIpAddress = async hostname => {
     if (ip.IPv4.isIPv4(hostname)) return hostname
-    if (
-      hostname.startsWith('[') &&
-      hostname.endsWith(']') &&
-      ip.IPv6.isIPv6(hostname.slice(1, -1))
-    ) {
-      return hostname.slice(1, -1)
-    }
+    const literal = unbracket(hostname)
+    if (ip.IPv6.isIPv6(literal)) return literal
     const { address } = await cacheableLookup.lookupAsync(hostname)
     return address
   }
@@ -21,3 +21,5 @@ module.exports = ({ cacheableLookup }) => {
     return ip.process(ipAddress).range() !== 'unicast'
   }
 }
+
+module.exports.RESERVED_ADDRESS_CODE = 'ERESERVEDADDRESSRANGE'

package/src/util/reachable-url.js

@@ -7,11 +7,12 @@ module.exports = ({ got, pingCache }) => {
     value: ({ url, statusCode }) => ({ url, statusCode })
   })
 
-  const reachableUrl = (url, opts) =>
-    pingUrl(url, {
-      ...got.gotOpts,
-      ...opts
-    })
+  const reachableUrl = async (url, opts) => {
+    const context = {}
+    const response = await pingUrl(url, { ...got.gotOpts, ...opts, context })
+    const { reservedAddress } = context
+    return reservedAddress ? { ...response, reservedAddress } : response
+  }
 
   reachableUrl.isReachable = createPingUrl.isReachable