@unavatar/core 0.0.1

package/package.json

@@ -0,0 +1,192 @@
+{
+  "name": "@unavatar/core",
+  "description": "Get unified user avatar from social networks, including Instagram, SoundCloud, Telegram, Twitter, YouTube & more.",
+  "homepage": "https://unavatar.io",
+  "version": "0.0.1",
+  "main": "src/index.js",
+  "bin": {
+    "unavatar": "bin/unavatar",
+    "unavatar-dev": "bin/unavatar-dev"
+  },
+  "author": {
+    "email": "josefrancisco.verdu@gmail.com",
+    "name": "Kiko Beats",
+    "url": "https://kikobeats.com"
+  },
+  "contributors": [
+    {
+      "name": "Ben Croker",
+      "email": "57572400+bencroker@users.noreply.github.com"
+    },
+    {
+      "name": "David Refoua",
+      "email": "David@Refoua.me"
+    },
+    {
+      "name": "Alexander Schlindwein",
+      "email": "alexander.schlindwein@hotmail.de"
+    },
+    {
+      "name": "Zadkiel",
+      "email": "hello@zadkiel.fr"
+    },
+    {
+      "name": "Omid Nikrah",
+      "email": "omidnikrah@gmail.com"
+    },
+    {
+      "name": "Nicolas Hedger",
+      "email": "649677+nhedger@users.noreply.github.com"
+    },
+    {
+      "name": "Angel M De Miguel",
+      "email": "angel@bitnami.com"
+    },
+    {
+      "name": "Tom Witkowski",
+      "email": "dev.gummibeer@gmail.com"
+    },
+    {
+      "name": "Terence Eden",
+      "email": "edent@users.noreply.github.com"
+    },
+    {
+      "name": "Hameed Rahamathullah",
+      "email": "hameedraha@gmail.com"
+    },
+    {
+      "name": "Jahir Fiquitiva",
+      "email": "jahir.fiquitiva@gmail.com"
+    },
+    {
+      "name": "Mads Hougesen",
+      "email": "mads@mhouge.dk"
+    },
+    {
+      "name": "ぶーと / Yoshiaki Ueda",
+      "email": "oh@bootjp.me"
+    },
+    {
+      "name": "Reed Jones",
+      "email": "reedjones@reedjones.com"
+    },
+    {
+      "name": "Rodrigo Reis",
+      "email": "rodrigoreis22@yahoo.com.br"
+    }
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Kikobeats/unavatar.git"
+  },
+  "bugs": {
+    "url": "https://github.com/Kikobeats/unavatar/issues"
+  },
+  "keywords": [
+    "avatar",
+    "domain",
+    "email",
+    "github",
+    "gravatar",
+    "instagram",
+    "microlink",
+    "telegram",
+    "twitter",
+    "youtube"
+  ],
+  "dependencies": {
+    "@keyvhq/compress": "~2.1.15",
+    "@keyvhq/core": "~2.1.15",
+    "@keyvhq/memoize": "~2.1.16",
+    "@keyvhq/multi": "~2.1.15",
+    "@keyvhq/redis": "~2.1.15",
+    "@kikobeats/got-scraping": "~4.2.0",
+    "@metascraper/helpers": "~5.49.24",
+    "@microlink/mql": "~0.14.2",
+    "@microlink/ping-url": "~1.4.16",
+    "browserless": "~10.10.0",
+    "cacheable-lookup": "~6.1.0",
+    "cheerio": "~1.2.0",
+    "data-uri-regex": "~0.1.4",
+    "debug-logfmt": "~1.4.7",
+    "got": "~11.8.6",
+    "html-get": "~2.21.21",
+    "https-tls": "~1.0.24",
+    "is-absolute-url": "~3.0.3",
+    "is-email-like": "~1.0.0",
+    "lodash": "~4.17.23",
+    "ms": "~2.1.3",
+    "p-any": "~3.0.0",
+    "p-cancelable": "2.1.1",
+    "p-timeout": "~4.1.0",
+    "puppeteer": "~24.37.5",
+    "srcset": "~4.0.0",
+    "tangerine": "~2.0.2",
+    "top-crawler-agents": "~1.0.34",
+    "top-user-agents": "~2.1.95",
+    "ua-hints": "~1.0.1",
+    "unique-random-array": "~2.0.0",
+    "url-regex": "~5.0.0"
+  },
+  "devDependencies": {
+    "@commitlint/cli": "latest",
+    "@commitlint/config-conventional": "latest",
+    "@ksmithut/prettier-standard": "latest",
+    "ava": "latest",
+    "c8": "latest",
+    "ci-publish": "latest",
+    "conventional-changelog-cli": "latest",
+    "finepack": "latest",
+    "git-authors-cli": "latest",
+    "github-generate-release": "latest",
+    "nano-staged": "latest",
+    "proxyquire": "latest",
+    "simple-git-hooks": "latest",
+    "sinon": "latest",
+    "standard": "latest",
+    "standard-version": "latest"
+  },
+  "engines": {
+    "node": "24"
+  },
+  "files": [
+    "bin",
+    "src"
+  ],
+  "license": "MIT",
+  "commitlint": {
+    "extends": [
+      "@commitlint/config-conventional"
+    ],
+    "rules": {
+      "body-max-line-length": [
+        0
+      ]
+    }
+  },
+  "nano-staged": {
+    "*.js": [
+      "prettier-standard",
+      "standard --fix"
+    ],
+    "package.json": [
+      "finepack"
+    ]
+  },
+  "simple-git-hooks": {
+    "commit-msg": "npx commitlint --edit",
+    "pre-commit": "npx nano-staged"
+  },
+  "scripts": {
+    "contributors": "(git-authors-cli && finepack && git add package.json && git commit -m 'build: contributors' --no-verify) || true",
+    "postrelease": "pnpm release:tags && pnpm release:github && (ci-publish || pnpm publish --access=public)",
+    "release": "pnpm release:version && pnpm release:changelog && pnpm release:commit && pnpm release:tag",
+    "release:changelog": "conventional-changelog -p conventionalcommits -i CHANGELOG.md -s",
+    "release:commit": "git add package.json CHANGELOG.md && git commit -m \"chore(release): $(node -p \"require('./package.json').version\")\"",
+    "release:github": "github-generate-release",
+    "release:tag": "git tag -a v$(node -p \"require('./package.json').version\") -m \"v$(node -p \"require('./package.json').version\")\"",
+    "release:tags": "git push origin HEAD:master --follow-tags",
+    "release:version": "standard-version --skip.changelog --skip.commit --skip.tag",
+    "test": "c8 ava"
+  }
+}

package/src/avatar/auto.js

@@ -0,0 +1,94 @@
+'use strict'
+
+const isAbsoluteUrl = require('is-absolute-url')
+const dataUriRegex = require('data-uri-regex')
+const isEmail = require('is-email-like')
+const pTimeout = require('p-timeout')
+const urlRegex = require('url-regex')
+const pAny = require('p-any')
+
+const httpStatus = require('../util/http-status')
+const isIterable = require('../util/is-iterable')
+const ExtendableError = require('../util/error')
+
+const getInputType = input => {
+  if (isEmail(input)) return 'email'
+  if (urlRegex({ strict: false }).test(input)) return 'domain'
+  return 'username'
+}
+
+const factory = ({ constants, providers, providersBy, reachableUrl }) => {
+  const { REQUEST_TIMEOUT } = constants
+
+  const getAvatarContent = provider => async output => {
+    if (typeof output !== 'string' || output === '') {
+      const message = output === undefined ? 'not found' : `\`${output}\` is invalid`
+      const statusCode = output === undefined ? httpStatus.NOT_FOUND : httpStatus.UNPROCESSABLE_ENTITY
+      throw new ExtendableError({ provider, message, statusCode })
+    }
+
+    if (dataUriRegex().test(output)) {
+      return { type: 'buffer', data: output }
+    }
+
+    if (!isAbsoluteUrl(output)) {
+      throw new ExtendableError({
+        message: 'The URL must to be absolute.',
+        provider,
+        statusCode: 400
+      })
+    }
+
+    const { statusCode, url } = await reachableUrl(output)
+
+    if (!reachableUrl.isReachable({ statusCode })) {
+      throw new ExtendableError({
+        message: httpStatus(statusCode),
+        provider,
+        statusCode
+      })
+    }
+
+    return { type: 'url', data: url, provider }
+  }
+
+  const getAvatar = async (fn, provider, args) => {
+    const promise = Promise.resolve(fn(args))
+      .then(getAvatarContent(provider))
+      .catch(error => {
+        isIterable.forEach(error, error => {
+          error.statusCode = error.statusCode ?? error.response?.statusCode
+          error.provider = provider
+        })
+        throw error
+      })
+
+    return pTimeout(promise, REQUEST_TIMEOUT).catch(error => {
+      error.provider = provider
+      throw error
+    })
+  }
+
+  const resolveAutoByType = async (inputType, args) => {
+    const collection = providersBy[inputType]
+    const promises = collection.map(provider =>
+      pTimeout(getAvatar(providers[provider], provider, args), REQUEST_TIMEOUT)
+    )
+    return pAny(promises)
+  }
+
+  const auto = inputTypeOrArgs => {
+    if (typeof inputTypeOrArgs === 'string') {
+      return args => resolveAutoByType(inputTypeOrArgs, args)
+    }
+
+    const args = inputTypeOrArgs
+    return resolveAutoByType(getInputType(args.input), args)
+  }
+
+  return { auto, getInputType, getAvatar }
+}
+
+factory.getInputType = getInputType
+
+module.exports = factory

package/src/constant.js

@@ -0,0 +1,18 @@
+'use strict'
+
+const { existsSync } = require('fs')
+const ms = require('ms')
+
+const DEFAULTS = {
+  AVATAR_SIZE: 400,
+  REQUEST_TIMEOUT: 25000,
+  TTL_DEFAULT: ms('28d'),
+  TMP_FOLDER: existsSync('/dev/shm') ? '/dev/shm' : '/tmp',
+  DEBUG_HTML_TO_FILE: false,
+  PUPPETEER_RANDOM_DIR: false
+}
+
+DEFAULTS.PROXY_TIMEOUT = Math.floor(DEFAULTS.REQUEST_TIMEOUT * (1 / 3))
+DEFAULTS.DNS_TIMEOUT = Math.floor(DEFAULTS.REQUEST_TIMEOUT * (1 / 5))
+
+module.exports = DEFAULTS

package/src/index.js

@@ -0,0 +1,51 @@
+'use strict'
+
+const DEFAULTS = require('./constant')
+
+module.exports = ({ constants: userConstants } = {}) => {
+  const constants = { ...DEFAULTS, ...userConstants }
+
+  if (userConstants?.REQUEST_TIMEOUT && !userConstants?.PROXY_TIMEOUT) {
+    constants.PROXY_TIMEOUT = Math.floor(constants.REQUEST_TIMEOUT * (1 / 3))
+  }
+  if (userConstants?.REQUEST_TIMEOUT && !userConstants?.DNS_TIMEOUT) {
+    constants.DNS_TIMEOUT = Math.floor(constants.REQUEST_TIMEOUT * (1 / 5))
+  }
+
+  const { createMemoryCache, createRedisCache } = require('./util/keyv')(constants)
+  const cacheableLookup = require('./util/cacheable-lookup')({ ...constants, createMemoryCache })
+  const got = require('./util/got')({ cacheableLookup })
+  const reachableUrl = require('./util/reachable-url')({ got, createMemoryCache })
+  const createBrowser = require('./util/browserless')(constants)
+  const getHTML = require('./util/html-get')({ createBrowser, got })
+  const { createHtmlProvider, getOgImage } = require('./util/html-provider')({ ...constants, getHTML })
+
+  const providerCtx = { constants, createHtmlProvider, getOgImage, got, createRedisCache }
+  const { providers, providersBy } = require('./providers')(providerCtx)
+
+  const { auto, getInputType, getAvatar } = require('./avatar/auto')({
+    constants,
+    providers,
+    providersBy,
+    reachableUrl
+  })
+
+  const normalizeArgs = input =>
+    typeof input === 'string' ? { input } : input
+
+  const unavatar = input => auto(normalizeArgs(input))
+
+  Object.keys(providers).forEach(name => {
+    unavatar[name] = input => getAvatar(providers[name], name, normalizeArgs(input))
+  })
+
+  unavatar.auto = auto
+  unavatar.getInputType = getInputType
+  unavatar.getAvatar = getAvatar
+  unavatar.providers = providers
+  unavatar.providersBy = providersBy
+  unavatar.reachableUrl = reachableUrl
+  unavatar.got = got
+
+  return unavatar
+}

package/src/providers/apple-music.js

@@ -0,0 +1,97 @@
+'use strict'
+
+const { $jsonld } = require('@metascraper/helpers')
+const memoize = require('@keyvhq/memoize')
+const ms = require('ms')
+
+const APPLE_MUSIC_ID_REGEX = /^\d+$/
+const APPLE_MUSIC_STOREFRONT = 'us'
+const APPLE_MUSIC_ENTITY_TYPES = {
+  album: { entity: 'album', idKey: 'collectionId' },
+  artist: { entity: 'musicArtist', idKey: 'artistId' },
+  song: { entity: 'song', idKey: 'trackId' }
+}
+
+const APPLE_MUSIC_SEARCH_TYPES = ['artist', 'song', 'album']
+const ITUNES_SEARCH_CACHE_TTL = ms('7d')
+
+const isNumericId = value => {
+  if (typeof value === 'number') {
+    return Number.isSafeInteger(value) && value > 0
+  }
+
+  if (typeof value === 'string' && APPLE_MUSIC_ID_REGEX.test(value)) {
+    const parsed = Number.parseInt(value, 10)
+    return Number.isSafeInteger(parsed) && parsed > 0
+  }
+
+  return false
+}
+
+const parseInput = input => {
+  const separatorIndex = input.indexOf(':')
+  const hasExplicitType = separatorIndex !== -1
+  const type = hasExplicitType ? input.slice(0, separatorIndex) : undefined
+  const id = hasExplicitType ? input.slice(separatorIndex + 1) : input
+
+  return { hasExplicitType, type, id }
+}
+
+module.exports = ({ createHtmlProvider, createRedisCache, got }) => {
+  const itunesSearchCache = createRedisCache({
+    namespace: 'itunes-search',
+    ttl: ITUNES_SEARCH_CACHE_TTL
+  })
+
+  const searchEntityId = memoize(
+    async ({ query, type }) => {
+      const entityConfig = APPLE_MUSIC_ENTITY_TYPES[type]
+      if (!entityConfig) return null
+      const { entity, idKey } = entityConfig
+
+      const url = `https://itunes.apple.com/search?term=${encodeURIComponent(
+        query
+      )}&entity=${entity}&limit=1`
+      const body = await got(url, {
+        responseType: 'json',
+        resolveBodyOnly: true
+      })
+
+      const entityId = body?.results?.[0]?.[idKey]
+      return isNumericId(entityId) ? String(entityId) : null
+    },
+    itunesSearchCache,
+    {
+      key: ({ query, type }) => `${type}:${query.trim().toLowerCase()}`,
+      ttl: ITUNES_SEARCH_CACHE_TTL
+    }
+  )
+
+  const appleMusicURI = async input => {
+    const { hasExplicitType, type, id } = parseInput(input)
+
+    if (!hasExplicitType) {
+      for (const searchType of APPLE_MUSIC_SEARCH_TYPES) {
+        const entityId = await searchEntityId({ query: id, type: searchType })
+        if (entityId) return `${APPLE_MUSIC_STOREFRONT}/${searchType}/${entityId}`
+      }
+
+      return `${APPLE_MUSIC_STOREFRONT}/search?term=${encodeURIComponent(id)}`
+    }
+
+    if (!APPLE_MUSIC_ENTITY_TYPES[type]) {
+      return `${type}/${isNumericId(id) ? id : encodeURIComponent(id)}`
+    }
+
+    if (isNumericId(id)) return `${APPLE_MUSIC_STOREFRONT}/${type}/${id}`
+
+    const entityId = await searchEntityId({ query: id, type })
+    return `${APPLE_MUSIC_STOREFRONT}/${type}/${entityId || encodeURIComponent(id)}`
+  }
+
+  return createHtmlProvider({
+    name: 'apple-music',
+    url: async input => `https://music.apple.com/${await appleMusicURI(input)}`,
+    getter: $ => $jsonld('image')($)
+  })
+}

package/src/providers/bluesky.js

@@ -0,0 +1,10 @@
+'use strict'
+
+const { $jsonld } = require('@metascraper/helpers')
+
+module.exports = ({ createHtmlProvider }) =>
+  createHtmlProvider({
+    name: 'bluesky',
+    url: input => `https://bsky.app/profile/${input}`,
+    getter: $ => $jsonld('mainEntity.image')($)
+  })

package/src/providers/deviantart.js

@@ -0,0 +1,8 @@
+'use strict'
+
+module.exports = ({ createHtmlProvider, getOgImage }) =>
+  createHtmlProvider({
+    name: 'deviantart',
+    url: input => `https://www.deviantart.com/${input}`,
+    getter: getOgImage
+  })

package/src/providers/dribbble.js

@@ -0,0 +1,8 @@
+'use strict'
+
+module.exports = ({ createHtmlProvider }) =>
+  createHtmlProvider({
+    name: 'dribbble',
+    url: input => `https://dribbble.com/${input}`,
+    getter: $ => $('.profile-avatar').attr('src')
+  })

package/src/providers/duckduckgo.js

@@ -0,0 +1,6 @@
+'use strict'
+
+module.exports = () =>
+  function duckduckgo ({ input }) {
+    return `https://icons.duckduckgo.com/ip3/${input}.ico`
+  }

package/src/providers/github.js

@@ -0,0 +1,10 @@
+'use strict'
+
+const { stringify } = require('querystring')
+
+module.exports = ({ constants }) =>
+  async function github ({ input }) {
+    return `https://github.com/${input}.png?${stringify({
+      size: constants.AVATAR_SIZE
+    })}`
+  }

package/src/providers/gitlab.js

@@ -0,0 +1,8 @@
+'use strict'
+
+module.exports = ({ createHtmlProvider, getOgImage }) =>
+  createHtmlProvider({
+    name: 'gitlab',
+    url: input => `https://gitlab.com/${input}`,
+    getter: getOgImage
+  })

package/src/providers/google.js

@@ -0,0 +1,6 @@
+'use strict'
+
+module.exports = () =>
+  function google ({ input }) {
+    return `https://www.google.com/s2/favicons?domain_url=${input}&sz=128`
+  }

package/src/providers/gravatar.js

@@ -0,0 +1,15 @@
+'use strict'
+
+const crypto = require('crypto')
+
+const stringify = require('../util/stringify')
+
+const md5 = str => crypto.createHash('md5').update(str).digest('hex')
+
+module.exports = ({ constants }) =>
+  function gravatar ({ input }) {
+    return `https://gravatar.com/avatar/${md5(input.trim().toLowerCase())}?${stringify({
+      size: constants.AVATAR_SIZE,
+      d: '404'
+    })}`
+  }

package/src/providers/index.js

@@ -0,0 +1,60 @@
+'use strict'
+
+const providersBy = {
+  email: ['gravatar'],
+  username: [
+    'apple-music',
+    'bluesky',
+    'deviantart',
+    'dribbble',
+    'github',
+    'gitlab',
+    'instagram',
+    'onlyfans',
+    'openstreetmap',
+    'patreon',
+    'reddit',
+    'soundcloud',
+    'spotify',
+    'substack',
+    'telegram',
+    'tiktok',
+    'twitch',
+    'vimeo',
+    'x',
+    'youtube'
+  ],
+  domain: ['microlink']
+}
+
+module.exports = ctx => {
+  const providers = {
+    'apple-music': require('./apple-music')(ctx),
+    bluesky: require('./bluesky')(ctx),
+    deviantart: require('./deviantart')(ctx),
+    dribbble: require('./dribbble')(ctx),
+    duckduckgo: require('./duckduckgo')(ctx),
+    github: require('./github')(ctx),
+    gitlab: require('./gitlab')(ctx),
+    google: require('./google')(ctx),
+    gravatar: require('./gravatar')(ctx),
+    instagram: require('./instagram')(ctx),
+    microlink: require('./microlink')(ctx),
+    onlyfans: require('./onlyfans')(ctx),
+    openstreetmap: require('./openstreetmap')(ctx),
+    patreon: require('./patreon')(ctx),
+    reddit: require('./reddit')(ctx),
+    soundcloud: require('./soundcloud')(ctx),
+    spotify: require('./spotify')(ctx),
+    substack: require('./substack')(ctx),
+    telegram: require('./telegram')(ctx),
+    tiktok: require('./tiktok')(ctx),
+    twitch: require('./twitch')(ctx),
+    vimeo: require('./vimeo')(ctx),
+    whatsapp: require('./whatsapp')(ctx),
+    x: require('./x')(ctx),
+    youtube: require('./youtube')(ctx)
+  }
+
+  return { providers, providersBy }
+}

package/src/providers/instagram.js

@@ -0,0 +1,8 @@
+'use strict'
+
+module.exports = ({ createHtmlProvider, getOgImage }) =>
+  createHtmlProvider({
+    name: 'instagram',
+    url: input => `https://www.instagram.com/${input}`,
+    getter: getOgImage
+  })

package/src/providers/microlink.js

@@ -0,0 +1,15 @@
+'use strict'
+
+const PCancelable = require('p-cancelable')
+const mql = require('@microlink/mql')
+const { get } = require('lodash')
+
+module.exports = ({ constants }) =>
+  PCancelable.fn(async function microlink ({ input, req = {} }, onCancel) {
+    const promise = mql(`https://${input}`, {
+      apiKey: req.isPro ? constants.MICROLINK_API_KEY : req.headers?.['x-api-key']
+    })
+    onCancel(() => promise.onCancel())
+    const { data } = await promise
+    return get(data, 'logo.url')
+  })

package/src/providers/onlyfans.js

@@ -0,0 +1,20 @@
+'use strict'
+
+const { get } = require('lodash')
+
+const getAvatarUrl = $ => {
+  const text = $('script[type="application/ld+json"]').contents().text()
+  if (!text) return
+  return get(JSON.parse(text), 'mainEntity.image')
+}
+
+module.exports = ({ createHtmlProvider }) =>
+  createHtmlProvider({
+    name: 'onlyfans',
+    url: input => `https://onlyfans.com/${input}`,
+    getter: getAvatarUrl,
+    htmlOpts: () => ({
+      prerender: true,
+      puppeteerOpts: { waitUntil: 'networkidle2', abortTypes: ['other', 'image', 'font'] }
+    })
+  })

package/src/providers/openstreetmap.js

@@ -0,0 +1,20 @@
+'use strict'
+
+const OPENSTREETMAP_USER_ID_REGEX = /^\d+$/
+
+module.exports = ({ got, createHtmlProvider }) => {
+  const fromID = userId =>
+    got(`https://api.openstreetmap.org/api/0.6/user/${userId}.json`, {
+      responseType: 'json'
+    }).then(({ body }) => body?.user?.img?.href)
+
+  const fromUsername = createHtmlProvider({
+    name: 'openstreetmap',
+    url: username => `https://www.openstreetmap.org/user/${encodeURIComponent(username)}`,
+    getter: $ => $('img.user_image').attr('src')
+  })
+
+  return function openstreetmap ({ input }) {
+    return OPENSTREETMAP_USER_ID_REGEX.test(input) ? fromID(input) : fromUsername(...arguments)
+  }
+}

package/src/providers/patreon.js

@@ -0,0 +1,10 @@
+'use strict'
+
+const { $jsonld } = require('@metascraper/helpers')
+
+module.exports = ({ createHtmlProvider }) =>
+  createHtmlProvider({
+    name: 'patreon',
+    url: username => `https://www.patreon.com/${username}`,
+    getter: $ => $jsonld('mainEntity.image.thumbnailUrl')($)
+  })