@unavatar/core 3.7.57 → 3.7.58

package/package.json

@@ -2,7 +2,7 @@
   "name": "@unavatar/core",
   "description": "Get unified user avatar from social networks, including Instagram, SoundCloud, Telegram, Twitter, YouTube & more.",
   "homepage": "https://unavatar.io",
-  "version": "3.7.57",
+  "version": "3.7.58",
   "main": "src/index.js",
   "exports": {
     ".": "./src/index.js",
@@ -115,6 +115,7 @@
     "@keyvhq/core": "~2.1.15",
     "@keyvhq/memoize": "~2.1.16",
     "@keyvhq/multi": "~2.1.15",
+    "@keyvhq/offline": "~2.2.0",
     "@keyvhq/redis": "~2.1.15",
     "@metascraper/helpers": "~5.49.24",
     "@microlink/mql": "~0.14.2",

package/src/index.js

@@ -2,7 +2,7 @@
 
 const DEFAULTS = require('./constant')
 
-module.exports = ({ constants: userConstants } = {}) => {
+module.exports = ({ constants: userConstants, redis, onFetchHTML } = {}) => {
   const constants = { ...DEFAULTS, ...userConstants }
 
   if (userConstants?.REQUEST_TIMEOUT && !userConstants?.PROXY_TIMEOUT) {
@@ -12,15 +12,20 @@ module.exports = ({ constants: userConstants } = {}) => {
     constants.DNS_TIMEOUT = Math.floor(constants.REQUEST_TIMEOUT * (1 / 5))
   }
 
-  const { createMemoryCache, createRedisCache } = require('./util/keyv')(constants)
-  const cacheableLookup = require('./util/cacheable-lookup')({ ...constants, createMemoryCache })
+  const { createMultiCache, createRedisCache } = require('./util/keyv')({ ...constants, redis })
+  const cache = require('./util/cache')({ createMultiCache, createRedisCache })
+  const cacheableLookup = require('./util/cacheable-lookup')({ ...constants, cache: cache.dnsCache })
   const got = require('./util/got')({ cacheableLookup })
-  const reachableUrl = require('./util/reachable-url')({ got, createMemoryCache })
+  const reachableUrl = require('./util/reachable-url')({ got, pingCache: cache.pingCache })
   const createBrowser = require('./util/browserless')(constants)
   const getHTML = require('./util/html-get')({ createBrowser, got })
-  const { createHtmlProvider, getOgImage } = require('./util/html-provider')({ ...constants, getHTML })
+  const { createHtmlProvider, getOgImage } = require('./util/html-provider')({
+    ...constants,
+    getHTML,
+    onFetchHTML
+  })
 
-  const providerCtx = { constants, createHtmlProvider, getOgImage, got, createRedisCache }
+  const providerCtx = { constants, createHtmlProvider, getOgImage, got, itunesSearchCache: cache.itunesSearchCache }
   const { providers, providersBy } = require('./providers')(providerCtx)
 
   const { auto, getInputType, getAvatar } = require('./avatar/auto')({

package/src/providers/apple-music.js

@@ -2,7 +2,6 @@
 
 const { $jsonld } = require('@metascraper/helpers')
 const memoize = require('@keyvhq/memoize')
-const ms = require('ms')
 
 const APPLE_MUSIC_ID_REGEX = /^\d+$/
 const APPLE_MUSIC_STOREFRONT = 'us'
@@ -13,7 +12,6 @@ const APPLE_MUSIC_ENTITY_TYPES = {
 }
 
 const APPLE_MUSIC_SEARCH_TYPES = ['artist', 'song', 'album']
-const ITUNES_SEARCH_CACHE_TTL = ms('7d')
 
 const isNumericId = value => {
   if (typeof value === 'number') {
@@ -37,12 +35,7 @@ const parseInput = input => {
   return { hasExplicitType, type, id }
 }
 
-module.exports = ({ createHtmlProvider, createRedisCache, got }) => {
-  const itunesSearchCache = createRedisCache({
-    namespace: 'itunes-search',
-    ttl: ITUNES_SEARCH_CACHE_TTL
-  })
-
+module.exports = ({ createHtmlProvider, itunesSearchCache, got }) => {
   const searchEntityId = memoize(
     async ({ query, type }) => {
       const entityConfig = APPLE_MUSIC_ENTITY_TYPES[type]
@@ -61,10 +54,7 @@ module.exports = ({ createHtmlProvider, createRedisCache, got }) => {
       return isNumericId(entityId) ? String(entityId) : null
     },
     itunesSearchCache,
-    {
-      key: ({ query, type }) => `${type}:${query.trim().toLowerCase()}`,
-      ttl: ITUNES_SEARCH_CACHE_TTL
-    }
+    { key: ({ query, type }) => `${type}:${query.trim().toLowerCase()}` }
   )
 
   const appleMusicURI = async input => {

package/src/util/cache.js

@@ -0,0 +1,9 @@
+'use strict'
+
+const ms = require('ms')
+
+module.exports = ({ createMultiCache, createRedisCache }) => ({
+  dnsCache: createMultiCache(createRedisCache({ namespace: 'dns', ttl: ms('1d') })),
+  pingCache: createMultiCache(createRedisCache({ namespace: 'ping', ttl: ms('1d') })),
+  itunesSearchCache: createRedisCache({ namespace: 'itunes-search', ttl: ms('7d') })
+})

package/src/util/cacheable-lookup.js

@@ -3,10 +3,10 @@
 const CacheableLookup = require('cacheable-lookup')
 const Tangerine = require('tangerine')
 
-module.exports = ({ TTL_DEFAULT, DNS_TIMEOUT, DNS_SERVERS, createMemoryCache }) =>
+module.exports = ({ TTL_DEFAULT, DNS_TIMEOUT, DNS_SERVERS, cache }) =>
   new CacheableLookup({
     maxTtl: TTL_DEFAULT,
-    cache: createMemoryCache({ namespace: 'dns' }),
+    cache,
     resolver: new Tangerine(
       {
         cache: false,

package/src/util/html-provider.js

@@ -1,14 +1,12 @@
 'use strict'
 
-const { mkdir, writeFile } = require('fs/promises')
 const { normalizeUrl } = require('@metascraper/helpers')
 const debug = require('debug-logfmt')('html-provider')
-const path = require('path')
 
 const httpStatus = require('./http-status')
 const ExtendableError = require('./error')
 
-const HTML_DEBUG_DIR = '/tmp/html'
+const NOT_FOUND = Symbol('NOT_FOUND')
 
 const isStatusCodeMissing = statusCode =>
   statusCode === undefined || statusCode === null || statusCode === ''
@@ -20,82 +18,49 @@ const createEmptyProviderValueError = ({ provider, statusCode }) =>
     message: 'Empty value returned by the provider.'
   })
 
-const NOT_FOUND = Symbol('NOT_FOUND')
-
-const UNRESOLVED = Symbol('UNRESOLVED')
-
 const getOgImage = $ => $('meta[property="og:image"]').attr('content')
 
-const sanitizeFileToken = input =>
-  String(input || '')
-    .trim()
-    .toLowerCase()
-    .replace(/[^a-z0-9._-]+/g, '-')
-    .replace(/^-+|-+$/g, '') || 'unknown'
-
-const writeHtmlDebugFile = async ({ debugEnabled, provider, tier, requestId, html }) => {
-  if (!debugEnabled || typeof html !== 'string' || html === '') return
-
-  const fileName = `${sanitizeFileToken(provider)}-${sanitizeFileToken(tier)}-${sanitizeFileToken(
-    requestId
-  )}.html`
-  const filePath = path.join(HTML_DEBUG_DIR, fileName)
-  await mkdir(path.dirname(filePath), { recursive: true })
-  await writeFile(filePath, html, 'utf8')
-
-  return filePath
-}
-
-const getHtmlDebugInfo = async $ => {
-  if (typeof $ !== 'function') return {}
-
-  const html = typeof $.html === 'function' ? $.html() : ''
-
-  return {
-    html,
-    htmlLength: html.length
-  }
-}
-
-module.exports = ({ PROXY_TIMEOUT, DEBUG_HTML_TO_FILE, getHTML }) => {
+module.exports = ({ PROXY_TIMEOUT, getHTML, onFetchHTML }) => {
   const createHtmlProvider = ({ name, url, getter, htmlOpts }) => {
-    const provider = async function ({ input, opts, req = {}, res = {} }) {
+    const provider = async function ({ input, req = {}, res = {} }) {
       const providerUrl = await url(input)
       const context = { provider: name, input, providerUrl }
 
-      const forceProxy = req.query?.proxy === true && typeof opts === 'function'
+      const attempt = async gotOpts => {
+        const defaultOpts = { ...htmlOpts?.(), timeout: PROXY_TIMEOUT }
+        const fetchOpts = gotOpts ? { ...defaultOpts, ...gotOpts } : defaultOpts
+        const tier = fetchOpts.tier ?? 'origin'
 
-      const logProviderError = payload => debug.error({ ...context, ...payload })
+        const log = debug.duration({ ...context, tier })
 
-      const logProviderLookup = payload => debug({ ...context, ...payload })
+        const { $, statusCode } = await getHTML(providerUrl, fetchOpts)
+
+        if (isStatusCodeMissing(statusCode)) {
+          log.error({ statusCode, status: 'missing_status_code' })
+          throw createEmptyProviderValueError({ provider: name, statusCode })
+        }
+
+        if (statusCode === httpStatus.NOT_FOUND) {
+          log.error({ statusCode, status: 'not_found' })
+          return NOT_FOUND
+        }
 
-      const getResult = async ($, statusCode, log, tier) => {
         const result = getter($)
         if (typeof result !== 'string' || result === '') {
-          let htmlDebugInfo = {}
-
-          if (DEBUG_HTML_TO_FILE) {
-            const { html, ...info } = await getHtmlDebugInfo($)
-            const requestId = typeof res.getHeader === 'function'
-              ? res.getHeader('x-request-id')
-              : undefined
-            const htmlFile = await writeHtmlDebugFile({
-              debugEnabled: true,
-              provider: name,
-              tier,
-              requestId,
-              html
-            }).catch(() => undefined)
-            htmlDebugInfo = { ...info, ...(htmlFile ? { htmlFile } : {}) }
-          }
-
-          log.error({
-            statusCode,
-            ...htmlDebugInfo
+          const error = createEmptyProviderValueError({
+            provider: name,
+            statusCode
           })
+          error.html =
+            typeof $ === 'function' && typeof $.html === 'function'
+              ? $.html()
+              : undefined
 
-          throw createEmptyProviderValueError({ provider: name, statusCode })
+          log.error({ statusCode })
+
+          throw error
         }
+
         const normalizedResult = normalizeUrl(providerUrl, result)
         log({
           statusCode,
@@ -105,90 +70,20 @@ module.exports = ({ PROXY_TIMEOUT, DEBUG_HTML_TO_FILE, getHTML }) => {
         return normalizedResult
       }
 
-      const reportSuccess = (requestType, result) => {
-        if (typeof res.setHeader === 'function') {
-          res.setHeader('x-proxy-tier', requestType)
-        }
-        return result
-      }
-
-      const getResultOrUndefined = async ({
-        tier,
-        resolve,
-        onError
-      } = {}) => {
-        const log = debug.duration({ ...context, tier })
-        try {
-          return await resolve(log)
-        } catch (error) {
-          if (error?.provider !== name && error?.name !== 'TimeoutError') {
-            logProviderError({ tier, status: 'failed', message: error.message })
-          }
-          onError?.(error)
-          return UNRESOLVED
-        }
-      }
-
-      const getProxyResultOrUndefined = async ({ superProxy = false } = {}) => {
-        const tier = superProxy ? 'residential' : 'datacenter'
-        return getResultOrUndefined({
-          tier,
-          resolve: async log => {
-            const { $, statusCode } = await getHTML(
-              providerUrl,
-              await opts(providerUrl, { superProxy, timeout: PROXY_TIMEOUT })
-            )
-            return getResult($, statusCode, log, tier)
-          }
-        })
-      }
-
-      const getOriginResultOrUndefined = () =>
-        getResultOrUndefined({
-          tier: 'origin',
-          resolve: async log => {
-            const { $, statusCode } = await getHTML(providerUrl, {
-              ...htmlOpts?.(),
-              timeout: PROXY_TIMEOUT
-            })
-
-            if (isStatusCodeMissing(statusCode)) {
-              log.error({ statusCode, status: 'missing_status_code' })
-              return UNRESOLVED
-            }
-
-            if (statusCode === httpStatus.NOT_FOUND) {
-              log.error({ statusCode, status: 'not_found' })
-              return NOT_FOUND
-            }
-
-            return getResult($, statusCode, log, 'origin')
-          }
-        })
-
-      if (forceProxy) {
-        logProviderLookup({ tier: 'origin', status: 'skipped', reason: 'force_proxy' })
+      if (typeof onFetchHTML === 'function') {
+        return onFetchHTML({ attempt, provider: name, providerUrl, req, res })
       }
 
-      if (!forceProxy) {
-        const resultOrigin = await getOriginResultOrUndefined()
-        if (resultOrigin === NOT_FOUND) return
-        if (resultOrigin !== UNRESOLVED) return reportSuccess('origin', resultOrigin)
-      }
-
-      if (typeof opts !== 'function') return
-
-      const resultDatacenter = await getProxyResultOrUndefined()
-      if (resultDatacenter !== UNRESOLVED) return reportSuccess('datacenter', resultDatacenter)
-
-      const resultResidential = await getProxyResultOrUndefined({ superProxy: true })
-      if (resultResidential === UNRESOLVED) return
-
-      return reportSuccess('residential', resultResidential)
+      const result = await attempt()
+      if (result === NOT_FOUND) return undefined
+      return result
     }
 
+    provider.getUrl = url
     return provider
   }
 
-  return { createHtmlProvider, getOgImage }
+  return { createHtmlProvider, getOgImage, NOT_FOUND }
 }
+
+module.exports.NOT_FOUND = NOT_FOUND

package/src/util/index.js

@@ -4,5 +4,6 @@ module.exports = {
   error: require('./error'),
   httpStatus: require('./http-status'),
   isIterable: require('./is-iterable'),
-  stringify: require('./stringify')
+  stringify: require('./stringify'),
+  NOT_FOUND: require('./html-provider').NOT_FOUND
 }

package/src/util/keyv.js

@@ -1,13 +1,15 @@
 'use strict'
 
 const keyvCompress = require('@keyvhq/compress')
+const KeyvOffline = require('@keyvhq/offline')
 const KeyvRedis = require('@keyvhq/redis')
 const KeyvMulti = require('@keyvhq/multi')
 const Keyv = require('@keyvhq/core')
 const assert = require('assert')
 
-module.exports = ({ TTL_DEFAULT }) => {
-  const createMultiCache = remote => new Keyv({ store: new KeyvMulti({ remote }) })
+module.exports = ({ TTL_DEFAULT, redis }) => {
+  const createMultiCache = remote =>
+    new Keyv({ store: new KeyvMulti({ remote }) })
 
   const createKeyv = opts => new Keyv({ ttl: TTL_DEFAULT, ...opts })
 
@@ -16,12 +18,10 @@ module.exports = ({ TTL_DEFAULT }) => {
     return keyvCompress(createKeyv(opts))
   }
 
-  const createMemoryCache = opts => createKeyvNamespace({ ...opts, store: new Map() })
-
   const createRedisCache = (opts = {}) => {
-    const store = new Map()
+    const store = redis ? new KeyvOffline(new KeyvRedis(redis)) : new Map()
     return createKeyvNamespace({ ...opts, store })
   }
 
-  return { createMemoryCache, createMultiCache, createRedisCache }
+  return { createMultiCache, createRedisCache }
 }

package/src/util/reachable-url.js

@@ -2,9 +2,7 @@
 
 const createPingUrl = require('@microlink/ping-url')
 
-module.exports = ({ got, createMemoryCache }) => {
-  const pingCache = createMemoryCache({ namespace: 'ping' })
-
+module.exports = ({ got, pingCache }) => {
   const pingUrl = createPingUrl(pingCache, {
     value: ({ url, statusCode }) => ({ url, statusCode })
   })