@unavatar/core 3.19.2 → 3.19.4

package/package.json

@@ -2,7 +2,7 @@
   "name": "@unavatar/core",
   "description": "Get unified user avatar from social networks, including Instagram, SoundCloud, Telegram, Twitter, YouTube & more.",
   "homepage": "https://unavatar.io",
-  "version": "3.19.2",
+  "version": "3.19.4",
   "main": "src/index.js",
   "exports": {
     ".": "./src/index.js",
@@ -139,7 +139,7 @@
     "ms": "~2.1.3",
     "p-any": "~3.0.0",
     "p-timeout": "~4.1.0",
-    "puppeteer": "~24.40.0",
+    "puppeteer": "~24.41.0",
     "re2": "~1.24.0",
     "srcset": "~4.0.0",
     "stable-regex": "~1.0.1",

package/src/index.js

@@ -29,18 +29,21 @@ module.exports = ({ constants: userConstants, redis, onFetchHTML } = {}) => {
   })
   const createBrowser = require('./util/browserless')(constants)
   const getHTML = require('./util/html-get')({ createBrowser, got })
-  const { createHtmlProvider, getOgImage } = require('./util/html-provider')({
-    ...constants,
-    getHTML,
-    onFetchHTML
-  })
+  const { createHtmlProvider, getOgImage, NOT_FOUND } =
+    require('./util/html-provider')({
+      ...constants,
+      getHTML,
+      onFetchHTML
+    })
 
   const providerCtx = {
     constants,
     createHtmlProvider,
     getOgImage,
+    NOT_FOUND,
     got,
     isReservedIp,
+    githubSearchCache: cache.githubSearchCache,
     itunesSearchCache: cache.itunesSearchCache
   }
   const { providers, providersBy } = require('./providers')(providerCtx)

package/src/providers/github.js

@@ -1,10 +1,146 @@
 'use strict'
 
+const memoize = require('@keyvhq/memoize')
 const { stringify } = require('querystring')
+const isEmail = require('is-email-like')
 
-module.exports = ({ constants }) =>
-  function github (input) {
-    return `https://github.com/${input}.png?${stringify({
-      size: constants.AVATAR_SIZE
-    })}`
+const GITHUB_API_URL = 'https://api.github.com'
+const SEARCH_USERS_PER_PAGE = 10
+const SEARCH_COMMITS_PER_PAGE = 20
+const COMMIT_SEARCH_ACCEPT_HEADER = 'application/vnd.github+json'
+
+const normalizeCacheKey = value => value.trim().toLowerCase()
+const createLookupCacheKey = prefix => value =>
+  `${prefix}:${normalizeCacheKey(value)}`
+
+const fetchJsonBody = async ({ got, url, options }) => {
+  const { body } = await got(url, {
+    responseType: 'json',
+    ...options
+  })
+
+  return body
+}
+
+const getUsernameAvatarUrl = ({ constants, input }) =>
+  `https://github.com/${input}.png?${stringify({
+    size: constants.AVATAR_SIZE
+  })}`
+
+const createSearchUsersByEmail = ({ githubSearchCache, got }) =>
+  memoize(
+    async email => {
+      const body = await fetchJsonBody({
+        got,
+        url: `${GITHUB_API_URL}/search/users?q=${encodeURIComponent(
+          email
+        )}&per_page=${SEARCH_USERS_PER_PAGE}`
+      })
+
+      return body?.items ?? []
+    },
+    githubSearchCache,
+    { key: createLookupCacheKey('search-users') }
+  )
+
+const createGetUser = ({ githubSearchCache, got }) =>
+  memoize(
+    login =>
+      fetchJsonBody({
+        got,
+        url: `${GITHUB_API_URL}/users/${encodeURIComponent(login)}`
+      }),
+    githubSearchCache,
+    { key: createLookupCacheKey('user') }
+  )
+
+const createSearchCommitsByEmail = ({ githubSearchCache, got }) =>
+  memoize(
+    async email => {
+      const body = await fetchJsonBody({
+        got,
+        url: `${GITHUB_API_URL}/search/commits?q=${encodeURIComponent(
+          `author-email:${email}`
+        )}&per_page=${SEARCH_COMMITS_PER_PAGE}`,
+        options: {
+          headers: { accept: COMMIT_SEARCH_ACCEPT_HEADER }
+        }
+      })
+
+      return body?.items ?? []
+    },
+    githubSearchCache,
+    { key: createLookupCacheKey('search-commits') }
+  )
+
+const findExactPublicProfileMatch = async ({
+  email,
+  getUser,
+  searchUsersByEmail
+}) => {
+  const candidates = await searchUsersByEmail(email)
+  const normalizedEmail = email.toLowerCase()
+
+  for (const candidate of candidates) {
+    const user = await getUser(candidate.login)
+
+    if (user?.email?.toLowerCase() === normalizedEmail) {
+      return user.avatar_url
+    }
   }
+}
+
+const findCommitConsensusMatch = async ({ email, searchCommitsByEmail }) => {
+  const commits = await searchCommitsByEmail(email)
+  const counts = new Map()
+
+  for (const item of commits) {
+    const linkedUser = item.author ?? item.committer
+    if (!linkedUser?.login || !linkedUser?.avatar_url) continue
+
+    const entry = counts.get(linkedUser.login) ?? {
+      avatarUrl: linkedUser.avatar_url,
+      count: 0
+    }
+
+    entry.count += 1
+    counts.set(linkedUser.login, entry)
+  }
+
+  let winner
+  for (const entry of counts.values()) {
+    if (!winner || entry.count > winner.count) winner = entry
+  }
+
+  return winner?.avatarUrl
+}
+
+module.exports = ({ constants, githubSearchCache, got }) => {
+  const searchUsersByEmail = createSearchUsersByEmail({
+    githubSearchCache,
+    got
+  })
+  const getUser = createGetUser({ githubSearchCache, got })
+  const searchCommitsByEmail = createSearchCommitsByEmail({
+    githubSearchCache,
+    got
+  })
+
+  return async function github (input) {
+    if (!isEmail(input)) return getUsernameAvatarUrl({ constants, input })
+
+    const exactMatch = await findExactPublicProfileMatch({
+      email: input,
+      getUser,
+      searchUsersByEmail
+    })
+
+    if (exactMatch) return exactMatch
+
+    return findCommitConsensusMatch({ email: input, searchCommitsByEmail })
+  }
+}
+
+module.exports.getUsernameAvatarUrl = getUsernameAvatarUrl
+module.exports.findExactPublicProfileMatch = findExactPublicProfileMatch
+module.exports.findCommitConsensusMatch = findCommitConsensusMatch

package/src/providers/index.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const providersBy = {
-  email: ['gravatar'],
+  email: ['gravatar', 'github'],
   username: [
     'apple-music',
     'behance',

package/src/providers/psnprofiles.js

@@ -1,8 +1,15 @@
 'use strict'
 
-module.exports = ({ createHtmlProvider, getOgImage }) =>
+const getAvatarUrl = ({ $, getOgImage, NOT_FOUND }) => {
+  const ogImage = getOgImage($)
+  return ogImage === undefined ? NOT_FOUND : ogImage
+}
+
+module.exports = ({ createHtmlProvider, getOgImage, NOT_FOUND }) =>
   createHtmlProvider({
     name: 'psnprofiles',
     url: input => `https://psnprofiles.com/${input}`,
-    getter: getOgImage
+    getter: $ => getAvatarUrl({ $, getOgImage, NOT_FOUND })
   })
+
+module.exports.getAvatarUrl = getAvatarUrl

package/src/util/cache.js

@@ -4,6 +4,10 @@ const ms = require('ms')
 
 module.exports = ({ createMultiCache, createRedisCache }) => ({
   dnsCache: createMultiCache(createRedisCache({ namespace: 'dns', ttl: ms('1d') })),
+  githubSearchCache: createRedisCache({
+    namespace: 'github-search',
+    ttl: ms('1d')
+  }),
   pingCache: createMultiCache(createRedisCache({ namespace: 'ping', ttl: ms('1d') })),
   itunesSearchCache: createRedisCache({ namespace: 'itunes-search', ttl: ms('7d') })
 })

package/src/util/html-provider.js

@@ -28,22 +28,26 @@ const createProviderError = ({ provider, statusCode, cause, code }) =>
     message: 'Empty value returned by the provider.'
   })
 
+const createErrorCause = ({ html, headers, statusCode }) => ({
+  html,
+  headers,
+  statusCode
+})
+
 module.exports = ({ PROXY_TIMEOUT, getHTML, onFetchHTML }) => {
   /**
    * @param {object} opts
    * @param {string} opts.name - Provider identifier used in logs and metrics.
    * @param {(input: string) => string | Promise<string>} opts.url - Builds the URL to fetch for a given input.
-   * @param {($: cheerio.CheerioAPI) => string | undefined} opts.getter
+   * @param {($: cheerio.CheerioAPI) => string | symbol | undefined} opts.getter
    *   Extracts the avatar URL from the fetched HTML.
-   *   - `string`    — avatar URL found (success).
+   *   - `string`     — avatar URL found (success).
+   *   - `NOT_FOUND`  — provider-level miss.
    *   - `undefined`  — avatar not found (normal failure, no retry).
-   * @param {(context: { $: cheerio.CheerioAPI, statusCode: number }) => boolean} [opts.isBlocked]
-   *   Optional provider-specific blocked-page detector, checked after the
-   *   default `is-antibot` check when getter returns empty/undefined.
    * @param {() => object} [opts.htmlOpts] - Returns extra options merged into the fetch call.
    */
-  const createHtmlProvider = ({ name, url, getter, isBlocked, htmlOpts }) => {
-    const provider = async function (input, context) {
+  const createHtmlProvider = ({ name, url, getter, htmlOpts }) => {
+    async function provider (input, context) {
       const providerUrl = await url(input)
 
       const attempt = async gotOpts => {
@@ -56,7 +60,8 @@ module.exports = ({ PROXY_TIMEOUT, getHTML, onFetchHTML }) => {
           },
           timeout: PROXY_TIMEOUT
         }
-        const fetchOpts = gotOpts ? { ...defaultOpts, ...gotOpts } : defaultOpts
+        const fetchOpts = { ...defaultOpts, ...gotOpts }
+        const userAgent = fetchOpts.headers['user-agent']
         const tier = fetchOpts.tier ?? 'origin'
 
         const log = debug.duration({ provider: name, input, providerUrl, tier })
@@ -73,6 +78,11 @@ module.exports = ({ PROXY_TIMEOUT, getHTML, onFetchHTML }) => {
             : undefined
         attempt.lastHeaders = responseHeaders
         attempt.lastStatusCode = statusCode
+        const errorCause = createErrorCause({
+          html: attempt.lastHtml,
+          headers: responseHeaders,
+          statusCode
+        })
 
         if (isStatusCodeMissing(statusCode)) {
           const code = EMPTY_PROVIDER_VALUE_CODE.MISSING_STATUS_CODE
@@ -80,67 +90,87 @@ module.exports = ({ PROXY_TIMEOUT, getHTML, onFetchHTML }) => {
           throw createProviderError({
             provider: name,
             statusCode,
-            cause: {
-              html: attempt.lastHtml,
-              headers: responseHeaders,
-              statusCode
-            },
+            cause: errorCause,
             code
           })
         }
 
+        const result = getter($)
         if (statusCode === httpStatus.NOT_FOUND) {
           log.error({ statusCode, status: 'not_found' })
           return NOT_FOUND
         }
 
-        const result = getter($)
-        if (typeof result !== 'string' || result === '') {
-          const error = createProviderError({
+        function createEmptyGetterResultError () {
+          return createProviderError({
             provider: name,
             statusCode,
             code: EMPTY_PROVIDER_VALUE_CODE.EMPTY_GETTER_RESULT,
-            cause: {
-              html: attempt.lastHtml,
-              headers: responseHeaders,
-              statusCode
-            }
+            cause: errorCause
           })
+        }
 
+        function getBlockedStatus () {
           const isRateLimited = statusCode === httpStatus.TOO_MANY_REQUESTS
-          const providerBlocked = isBlocked?.({ $, statusCode })
+          if (isRateLimited) return { isBlocked: true, antibotProvider: null }
 
           const { detected: antibotDetected, provider: antibotProvider } =
-            isRateLimited || providerBlocked
-              ? { detected: false, provider: null }
-              : isAntibot({
-                url: providerUrl,
-                statusCode,
-                headers: responseHeaders,
-                body: attempt.lastHtml
-              })
-
-          if (isRateLimited || providerBlocked || antibotDetected) {
-            error.blocked = true
+            isAntibot({
+              url: providerUrl,
+              statusCode,
+              headers: responseHeaders,
+              body: attempt.lastHtml
+            })
+
+          return {
+            isBlocked: antibotDetected,
+            antibotProvider
           }
+        }
+
+        if (typeof result === 'string' && result !== '') {
+          const normalizedResult = normalizeUrl(providerUrl, result)
+          log({
+            statusCode,
+            status: 'success',
+            result: normalizedResult
+          })
+          return normalizedResult
+        }
+
+        // Some providers encode not-found via getter output. Check antibot
+        // first so challenge pages are retried as blocked, not treated as 404.
+        const { isBlocked: shouldMarkBlocked, antibotProvider } =
+          getBlockedStatus()
+
+        if (shouldMarkBlocked) {
+          const error = createEmptyGetterResultError()
+          error.blocked = true
 
           log.error({
             statusCode,
-            status: error.blocked ? 'blocked' : undefined,
+            status: 'blocked',
             antibot: antibotProvider ?? undefined,
-            userAgent: fetchOpts.headers['user-agent']
+            userAgent
           })
 
           throw error
         }
 
-        const normalizedResult = normalizeUrl(providerUrl, result)
-        log({
+        if (result === NOT_FOUND) {
+          log.error({ statusCode, status: 'not_found' })
+          return NOT_FOUND
+        }
+
+        const error = createEmptyGetterResultError()
+
+        log.error({
           statusCode,
-          status: 'success',
-          result: normalizedResult
+          antibot: antibotProvider ?? undefined,
+          userAgent
         })
-        return normalizedResult
+
+        throw error
       }
 
       if (typeof onFetchHTML === 'function') {
@@ -148,8 +178,7 @@ module.exports = ({ PROXY_TIMEOUT, getHTML, onFetchHTML }) => {
       }
 
       const result = await attempt()
-      if (result === NOT_FOUND) return
-      return result
+      return result === NOT_FOUND ? undefined : result
     }
 
     provider.getUrl = url