@unavatar/core 0.0.1

package/src/providers/reddit.js

@@ -0,0 +1,9 @@
+'use strict'
+
+module.exports = ({ createHtmlProvider }) =>
+  createHtmlProvider({
+    name: 'reddit',
+    url: input => `https://www.reddit.com/user/${input}/`,
+    getter: $ => $('img[alt*="avatar"]').attr('src'),
+    htmlOpts: () => ({ headers: { 'accept-language': 'en' } })
+  })

package/src/providers/soundcloud.js

@@ -0,0 +1,17 @@
+'use strict'
+
+const uniqueRandomArray = require('unique-random-array')
+
+/**
+ * SoundCloud is serving an old app for desktop users,
+ * so we need to use a mobile user agent to get the avatar.
+ */
+const randomUserAgent = uniqueRandomArray(require('top-user-agents/mobile'))
+
+module.exports = ({ createHtmlProvider, getOgImage }) =>
+  createHtmlProvider({
+    name: 'soundcloud',
+    url: input => `https://soundcloud.com/${input}`,
+    getter: getOgImage,
+    htmlOpts: () => ({ headers: { 'user-agent': randomUserAgent() } })
+  })

package/src/providers/spotify.js

@@ -0,0 +1,18 @@
+'use strict'
+
+const spotifyUri = input => {
+  const [first, second] = input.split(':')
+  const type = second ? first : 'user'
+  const id = second ?? first
+  return `${type}/${id}`
+}
+
+module.exports = ({ createHtmlProvider, getOgImage }) =>
+  createHtmlProvider({
+    name: 'spotify',
+    url: input => `https://open.spotify.com/${spotifyUri(input)}`,
+    getter: getOgImage,
+    htmlOpts: () => ({
+      prerender: true
+    })
+  })

package/src/providers/substack.js

@@ -0,0 +1,35 @@
+'use strict'
+
+const { $jsonld } = require('@metascraper/helpers')
+const { parse: parseSrcset } = require('srcset')
+
+const getBestSrcsetUrl = srcset => {
+  if (typeof srcset !== 'string' || srcset.trim() === '') return
+
+  const candidates = parseSrcset(srcset).map(candidate => ({
+    url: candidate.url,
+    score: candidate.width ?? candidate.density ?? 0
+  }))
+
+  if (candidates.length === 0) return
+  return candidates.reduce((best, current) => (current.score > best.score ? current : best)).url
+}
+
+const getPictureAvatar = $ => {
+  const pictureImg = $('picture img')
+  const srcset = pictureImg.attr('srcset')
+  return getBestSrcsetUrl(srcset) || pictureImg.attr('src')
+}
+
+const getAvatarUrl = $ => $jsonld('publisher.logo.url')($) || getPictureAvatar($)
+
+const factory = ({ createHtmlProvider }) =>
+  createHtmlProvider({
+    name: 'substack',
+    url: input => `https://${input}.substack.com`,
+    getter: getAvatarUrl
+  })
+
+factory.getAvatarUrl = getAvatarUrl
+
+module.exports = factory

package/src/providers/telegram.js

@@ -0,0 +1,8 @@
+'use strict'
+
+module.exports = ({ createHtmlProvider }) =>
+  createHtmlProvider({
+    name: 'telegram',
+    url: input => `https://t.me/${input}`,
+    getter: $ => $('img.tgme_page_photo_image').attr('src')
+  })

package/src/providers/tiktok.js

@@ -0,0 +1,26 @@
+'use strict'
+
+const { get } = require('lodash')
+
+const getAvatarUrl = $ => {
+  const text = $('#__UNIVERSAL_DATA_FOR_REHYDRATION__').contents().text()
+  if (!text) return
+  return get(JSON.parse(text), [
+    '__DEFAULT_SCOPE__',
+    'webapp.user-detail',
+    'userInfo',
+    'user',
+    'avatarLarger'
+  ])
+}
+
+const factory = ({ createHtmlProvider }) =>
+  createHtmlProvider({
+    name: 'tiktok',
+    url: input => `https://www.tiktok.com/@${input}`,
+    getter: getAvatarUrl
+  })
+
+factory.getAvatarUrl = getAvatarUrl
+
+module.exports = factory

package/src/providers/twitch.js

@@ -0,0 +1,8 @@
+'use strict'
+
+module.exports = ({ createHtmlProvider, getOgImage }) =>
+  createHtmlProvider({
+    name: 'twitch',
+    url: input => `https://www.twitch.tv/${input}`,
+    getter: getOgImage
+  })

package/src/providers/vimeo.js

@@ -0,0 +1,8 @@
+'use strict'
+
+module.exports = ({ createHtmlProvider, getOgImage }) =>
+  createHtmlProvider({
+    name: 'vimeo',
+    url: input => `https://vimeo.com/${input}`,
+    getter: getOgImage
+  })

package/src/providers/whatsapp.js

@@ -0,0 +1,31 @@
+'use strict'
+
+const whatsappURI = input => {
+  const [first, second] = input.split(':')
+  return {
+    type: second ? first : 'phone',
+    id: second ?? first
+  }
+}
+
+const whatsappURL = input => {
+  const { type, id } = whatsappURI(input)
+  switch (type) {
+    case 'phone':
+      return `https://api.whatsapp.com/send/?phone=${id}`
+    case 'channel':
+      return `https://www.whatsapp.com/channel/${id}`
+    case 'chat':
+    case 'group':
+      return `https://chat.whatsapp.com/${id}`
+    default:
+      throw new Error(`Unsupported WhatsApp type: ${type}`)
+  }
+}
+
+module.exports = ({ createHtmlProvider, getOgImage }) =>
+  createHtmlProvider({
+    name: 'whatsapp',
+    url: whatsappURL,
+    getter: getOgImage
+  })

package/src/providers/x.js

@@ -0,0 +1,35 @@
+'use strict'
+
+const uniqueRandomArray = require('unique-random-array')
+const { $jsonld } = require('@metascraper/helpers')
+
+const randomCrawlerAgent = uniqueRandomArray(
+  require('top-crawler-agents').filter(agent => agent.startsWith('Slackbot'))
+)
+
+const toHighResolution = url => {
+  if (url?.endsWith('_200x200.jpg')) {
+    return url.replace('_200x200.jpg', '_400x400.jpg')
+  }
+  if (url?.endsWith('_normal.jpg')) {
+    return url.replace('_normal.jpg', '_400x400.jpg')
+  }
+  return url
+}
+
+const getProfileImage = $ =>
+  toHighResolution(
+    $jsonld('mainEntity.image.contentUrl')($) || $('meta[property="og:image"]').attr('content')
+  )
+
+const factory = ({ createHtmlProvider }) =>
+  createHtmlProvider({
+    name: 'x',
+    url: input => `https://x.com/${input}`,
+    getter: getProfileImage,
+    htmlOpts: () => ({ headers: { 'user-agent': randomCrawlerAgent() } })
+  })
+
+factory.getProfileImage = getProfileImage
+
+module.exports = factory

package/src/providers/youtube.js

@@ -0,0 +1,8 @@
+'use strict'
+
+module.exports = ({ createHtmlProvider, getOgImage }) =>
+  createHtmlProvider({
+    name: 'youtube',
+    url: input => `https://www.youtube.com/@${input}`,
+    getter: getOgImage
+  })

package/src/util/browserless.js

@@ -0,0 +1,38 @@
+'use strict'
+
+const createBrowser = require('browserless')
+const { randomUUID } = require('crypto')
+const puppeteer = require('puppeteer')
+const path = require('path')
+
+module.exports = ({ TMP_FOLDER, PUPPETEER_RANDOM_DIR }) => {
+  const PUPPETEER_BASE_DIR = path.join(TMP_FOLDER, 'puppeteer')
+
+  const getPuppeteerDir = PUPPETEER_RANDOM_DIR
+    ? () => `${PUPPETEER_BASE_DIR}-${randomUUID()}`
+    : () => PUPPETEER_BASE_DIR
+
+  const getArgs = () => {
+    const PUPPETEER_DIR = getPuppeteerDir()
+    const DATA_DIR = path.join(PUPPETEER_DIR, 'profile')
+    const CACHE_DIR = path.join(PUPPETEER_DIR, 'cache')
+
+    const args = createBrowser.driver.defaultArgs.concat([
+      '--allow-running-insecure-content',
+      `--disk-cache-dir=${CACHE_DIR}`,
+      `--user-data-dir=${DATA_DIR}`
+    ])
+
+    return { PUPPETEER_DIR, DATA_DIR, CACHE_DIR, args }
+  }
+
+  const browser = createBrowser({
+    args: getArgs().args,
+    dumpio: false,
+    pipe: true,
+    puppeteer,
+    waitForInitialPage: false
+  })
+
+  return () => browser
+}

package/src/util/cacheable-lookup.js

@@ -0,0 +1,22 @@
+'use strict'
+
+const CacheableLookup = require('cacheable-lookup')
+const Tangerine = require('tangerine')
+
+module.exports = ({ TTL_DEFAULT, DNS_TIMEOUT, createMemoryCache }) =>
+  new CacheableLookup({
+    maxTtl: TTL_DEFAULT,
+    cache: createMemoryCache({ namespace: 'dns' }),
+    resolver: new Tangerine(
+      {
+        cache: false,
+        timeout: DNS_TIMEOUT,
+        servers: ['1.1.1.1', '8.8.8.8']
+      },
+      require('got').extend({
+        responseType: 'buffer',
+        decompress: false,
+        retry: 0
+      })
+    )
+  })

package/src/util/error.js

@@ -0,0 +1,6 @@
+module.exports = class ExtendableError extends Error {
+  constructor (props) {
+    super()
+    Object.assign(this, props)
+  }
+}

package/src/util/got.js

@@ -0,0 +1,32 @@
+'use strict'
+
+const uniqueRandomArray = require('unique-random-array')
+const tlsHook = require('https-tls/hook')
+const uaHints = require('ua-hints')
+const got = require('got')
+
+const randomUserAgent = uniqueRandomArray(require('top-user-agents'))
+
+const userAgentHook = options => {
+  if (options.headers['user-agent'] === 'got (https://github.com/sindresorhus/got)') {
+    const userAgent = randomUserAgent()
+    options.headers['user-agent'] = userAgent
+  }
+
+  for (const [key, value] of Object.entries(uaHints(options.headers['user-agent']))) {
+    options.headers[key] = value
+  }
+}
+
+module.exports = ({ cacheableLookup }) => {
+  const gotOpts = {
+    dnsCache: cacheableLookup,
+    https: { rejectUnauthorized: false },
+    hooks: { beforeRequest: [userAgentHook, tlsHook] }
+  }
+
+  const instance = got.extend(gotOpts)
+  instance.gotOpts = gotOpts
+
+  return instance
+}

package/src/util/html-get.js

@@ -0,0 +1,25 @@
+'use strict'
+
+module.exports = ({ createBrowser, got }) =>
+  async function (url, { puppeteerOpts, timeout, gotOpts, ...opts } = {}) {
+    const browser = await createBrowser()
+    const browserContext = await browser.createContext()
+
+    const promise = require('html-get')(url, {
+      prerender: false,
+      ...opts,
+      getBrowserless: () => browserContext,
+      serializeHtml: $ => ({ $ }),
+      puppeteerOpts: {
+        timeout,
+        ...puppeteerOpts
+      },
+      gotOpts: {
+        timeout,
+        ...got.gotOpts,
+        ...gotOpts
+      }
+    })
+
+    return Promise.resolve(promise).finally(() => browserContext.destroyContext())
+  }

package/src/util/html-provider.js

@@ -0,0 +1,190 @@
+'use strict'
+
+const { mkdir, writeFile } = require('fs/promises')
+const { normalizeUrl } = require('@metascraper/helpers')
+const debug = require('debug-logfmt')('html-provider')
+const path = require('path')
+
+const httpStatus = require('./http-status')
+const ExtendableError = require('./error')
+
+const HTML_DEBUG_DIR = '/tmp/html'
+
+const isStatusCodeMissing = statusCode =>
+  statusCode === undefined || statusCode === null || statusCode === ''
+
+const createEmptyProviderValueError = ({ provider, statusCode }) =>
+  new ExtendableError({
+    provider,
+    statusCode,
+    message: 'Empty value returned by the provider.'
+  })
+
+const NOT_FOUND = Symbol('NOT_FOUND')
+
+const UNRESOLVED = Symbol('UNRESOLVED')
+
+const getOgImage = $ => $('meta[property="og:image"]').attr('content')
+
+const sanitizeFileToken = input =>
+  String(input || '')
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9._-]+/g, '-')
+    .replace(/^-+|-+$/g, '') || 'unknown'
+
+const writeHtmlDebugFile = async ({ debugEnabled, provider, tier, requestId, html }) => {
+  if (!debugEnabled || typeof html !== 'string' || html === '') return
+
+  const fileName = `${sanitizeFileToken(provider)}-${sanitizeFileToken(tier)}-${sanitizeFileToken(
+    requestId
+  )}.html`
+  const filePath = path.join(HTML_DEBUG_DIR, fileName)
+  await mkdir(path.dirname(filePath), { recursive: true })
+  await writeFile(filePath, html, 'utf8')
+
+  return filePath
+}
+
+const getHtmlDebugInfo = async $ => {
+  if (typeof $ !== 'function') return {}
+
+  const html = typeof $.html === 'function' ? $.html() : ''
+
+  return {
+    html,
+    htmlLength: html.length
+  }
+}
+
+module.exports = ({ PROXY_TIMEOUT, DEBUG_HTML_TO_FILE, getHTML }) => {
+  const createHtmlProvider = ({ name, url, getter, htmlOpts }) => {
+    const provider = async function ({ input, opts, req = {}, res = {} }) {
+      const providerUrl = await url(input)
+      const context = { provider: name, input, providerUrl }
+
+      const forceProxy = req.query?.proxy === true && typeof opts === 'function'
+
+      const logProviderError = payload => debug.error({ ...context, ...payload })
+
+      const logProviderLookup = payload => debug({ ...context, ...payload })
+
+      const getResult = async ($, statusCode, log, tier) => {
+        const result = getter($)
+        if (typeof result !== 'string' || result === '') {
+          const { html, ...htmlDebugInfo } = await getHtmlDebugInfo($)
+          const requestId = typeof res.getHeader === 'function'
+            ? res.getHeader('x-request-id')
+            : undefined
+          const htmlFile = await writeHtmlDebugFile({
+            debugEnabled: DEBUG_HTML_TO_FILE,
+            provider: name,
+            tier,
+            requestId,
+            html
+          }).catch(() => undefined)
+
+          log.error({
+            statusCode,
+            ...htmlDebugInfo,
+            ...(htmlFile ? { htmlFile } : {})
+          })
+
+          throw createEmptyProviderValueError({ provider: name, statusCode })
+        }
+        const normalizedResult = normalizeUrl(providerUrl, result)
+        log({
+          statusCode,
+          status: 'success',
+          result: normalizedResult
+        })
+        return normalizedResult
+      }
+
+      const reportSuccess = (requestType, result) => {
+        if (typeof res.setHeader === 'function') {
+          res.setHeader('x-proxy-tier', requestType)
+        }
+        return result
+      }
+
+      const getResultOrUndefined = async ({
+        tier,
+        resolve,
+        onError
+      } = {}) => {
+        const log = debug.duration({ ...context, tier })
+        try {
+          return await resolve(log)
+        } catch (error) {
+          if (error?.provider !== name && error?.name !== 'TimeoutError') {
+            logProviderError({ tier, status: 'failed', message: error.message })
+          }
+          onError?.(error)
+          return UNRESOLVED
+        }
+      }
+
+      const getProxyResultOrUndefined = async ({ superProxy = false } = {}) => {
+        const tier = superProxy ? 'residential' : 'datacenter'
+        return getResultOrUndefined({
+          tier,
+          resolve: async log => {
+            const { $, statusCode } = await getHTML(
+              providerUrl,
+              await opts(providerUrl, { superProxy, timeout: PROXY_TIMEOUT })
+            )
+            return getResult($, statusCode, log, tier)
+          }
+        })
+      }
+
+      const getOriginResultOrUndefined = () =>
+        getResultOrUndefined({
+          tier: 'origin',
+          resolve: async log => {
+            const { $, statusCode } = await getHTML(providerUrl, {
+              ...htmlOpts?.(),
+              timeout: PROXY_TIMEOUT
+            })
+
+            if (isStatusCodeMissing(statusCode)) {
+              log.error({ statusCode, status: 'missing_status_code' })
+              return UNRESOLVED
+            }
+
+            if (statusCode === httpStatus.NOT_FOUND) {
+              log.error({ statusCode, status: 'not_found' })
+              return NOT_FOUND
+            }
+
+            return getResult($, statusCode, log, 'origin')
+          }
+        })
+
+      if (forceProxy) {
+        logProviderLookup({ tier: 'origin', status: 'skipped', reason: 'force_proxy' })
+      }
+
+      if (!forceProxy) {
+        const resultOrigin = await getOriginResultOrUndefined()
+        if (resultOrigin === NOT_FOUND) return
+        if (resultOrigin !== UNRESOLVED) return reportSuccess('origin', resultOrigin)
+      }
+
+      if (typeof opts !== 'function') return
+
+      const resultDatacenter = await getProxyResultOrUndefined()
+      if (resultDatacenter !== UNRESOLVED) return reportSuccess('datacenter', resultDatacenter)
+
+      const resultResidential = await getProxyResultOrUndefined({ superProxy: true })
+      if (resultResidential === UNRESOLVED) return
+
+      return reportSuccess('residential', resultResidential)
+    }
+
+    return provider
+  }
+
+  return { createHtmlProvider, getOgImage }
+}

package/src/util/http-status.js

@@ -0,0 +1,17 @@
+'use strict'
+
+const { STATUS_CODES } = require('http')
+
+const byCode = Object.entries(STATUS_CODES).reduce((acc, [key, value]) => {
+  const name = value.replace(/[^a-zA-Z0-9]+/g, '_').toUpperCase()
+  acc[name] = key
+  return acc
+}, {})
+
+const fn = input => (typeof input === 'number' ? STATUS_CODES[input] ?? input : byCode[input])
+
+Object.keys(byCode).forEach(key => {
+  fn[key] = Number(byCode[key])
+})
+
+module.exports = fn

package/src/util/is-iterable.js

@@ -0,0 +1,9 @@
+const isIterable = input => typeof input[Symbol.iterator] === 'function'
+
+isIterable.forEach = (input, fn) => {
+  for (const item of isIterable(input) ? input : [].concat(input)) {
+    fn(item)
+  }
+}
+
+module.exports = isIterable

package/src/util/keyv.js

@@ -0,0 +1,27 @@
+'use strict'
+
+const keyvCompress = require('@keyvhq/compress')
+const KeyvRedis = require('@keyvhq/redis')
+const KeyvMulti = require('@keyvhq/multi')
+const Keyv = require('@keyvhq/core')
+const assert = require('assert')
+
+module.exports = ({ TTL_DEFAULT }) => {
+  const createMultiCache = remote => new Keyv({ store: new KeyvMulti({ remote }) })
+
+  const createKeyv = opts => new Keyv({ ttl: TTL_DEFAULT, ...opts })
+
+  const createKeyvNamespace = opts => {
+    assert(opts.namespace, '`opts.namespace` is required.')
+    return keyvCompress(createKeyv(opts))
+  }
+
+  const createMemoryCache = opts => createKeyvNamespace({ ...opts, store: new Map() })
+
+  const createRedisCache = (opts = {}) => {
+    const store = new Map()
+    return createKeyvNamespace({ ...opts, store })
+  }
+
+  return { createMemoryCache, createMultiCache, createRedisCache }
+}

package/src/util/reachable-url.js

@@ -0,0 +1,21 @@
+'use strict'
+
+const createPingUrl = require('@microlink/ping-url')
+
+module.exports = ({ got, createMemoryCache }) => {
+  const pingCache = createMemoryCache({ namespace: 'ping' })
+
+  const pingUrl = createPingUrl(pingCache, {
+    value: ({ url, statusCode }) => ({ url, statusCode })
+  })
+
+  const reachableUrl = (url, opts) =>
+    pingUrl(url, {
+      ...got.gotOpts,
+      ...opts
+    })
+
+  reachableUrl.isReachable = createPingUrl.isReachable
+
+  return reachableUrl
+}

package/src/util/stringify.js

@@ -0,0 +1,5 @@
+'use strict'
+
+const { URLSearchParams } = require('url')
+
+module.exports = obj => new URLSearchParams(obj).toString()