@unavatar/core 3.72.2 → 3.72.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "name": "@unavatar/core",
3
3
  "description": "Get unified user avatar from social networks, including Instagram, SoundCloud, Telegram, Twitter, YouTube & more.",
4
4
  "homepage": "https://unavatar.io",
5
- "version": "3.72.2",
5
+ "version": "3.72.3",
6
6
  "main": "src/index.js",
7
7
  "exports": {
8
8
  ".": "./src/index.js",
@@ -127,7 +127,6 @@
127
127
  "@keyvhq/offline": "~2.2.0",
128
128
  "@keyvhq/redis": "~2.2.3",
129
129
  "@metascraper/helpers": "~5.58.4",
130
- "@microlink/ping-url": "~1.5.0",
131
130
  "bimi-url": "~1.0.0",
132
131
  "browserless": "~14.1.1",
133
132
  "cacheable-lookup": "~6.1.0",
@@ -147,6 +146,7 @@
147
146
  "p-timeout": "~4.1.0",
148
147
  "puppeteer": "~25.16.0",
149
148
  "re2": "~1.27.0",
149
+ "reachable-url": "~2.1.10",
150
150
  "srcset": "~4.0.0",
151
151
  "stable-regex": "~1.0.1",
152
152
  "tangerine": "~2.1.3",
@@ -1,17 +1,78 @@
1
1
  'use strict'
2
2
 
3
- const createPingUrl = require('@microlink/ping-url')
3
+ const memoize = require('@keyvhq/memoize')
4
+ const reachableUrl = require('reachable-url')
5
+ const ms = require('ms')
6
+
7
+ const RATE_LIMITED = 429
8
+ const UNCACHED_STATUS_CODES = new Set([403, 408])
9
+ const RATE_LIMIT_TTL_DEFAULT = ms('1m')
10
+ const RATE_LIMIT_TTL_MIN = ms('1s')
11
+ const RATE_LIMIT_TTL_MAX = ms('10m')
4
12
 
5
13
  const isRedirectStatus = statusCode => statusCode >= 300 && statusCode < 400
6
14
 
15
+ const isUncached = statusCode =>
16
+ UNCACHED_STATUS_CODES.has(statusCode) || statusCode >= 500
17
+
18
+ const parseRetryAfter = (value, now = Date.now()) => {
19
+ if (typeof value === 'string') value = value.trim()
20
+ if (!value) return undefined
21
+ const seconds = Number(value)
22
+ if (Number.isFinite(seconds)) return seconds * 1000
23
+ if (typeof value !== 'string') return undefined
24
+ if (/[a-z]/i.test(value)) {
25
+ const date = Date.parse(value)
26
+ if (!Number.isNaN(date)) return date - now
27
+ }
28
+ const first = value.split(',')[0]
29
+ if (!first) return undefined
30
+ const firstSeconds = Number(first)
31
+ return Number.isFinite(firstSeconds) ? firstSeconds * 1000 : undefined
32
+ }
33
+
34
+ const rateLimitTtl = retryAfter => {
35
+ const wait = parseRetryAfter(retryAfter)
36
+ if (wait === undefined) return RATE_LIMIT_TTL_DEFAULT
37
+ return Math.min(Math.max(wait, RATE_LIMIT_TTL_MIN), RATE_LIMIT_TTL_MAX)
38
+ }
39
+
40
+ class UncachedPing extends Error {
41
+ constructor (response) {
42
+ super(`ping not cached for ${response.statusCode}`)
43
+ this.response = response
44
+ }
45
+ }
46
+
47
+ const ping = async (url, opts) => {
48
+ const {
49
+ url: resolvedUrl,
50
+ statusCode,
51
+ headers
52
+ } = await reachableUrl(url, opts)
53
+ const response = { url: resolvedUrl, statusCode }
54
+ if (isUncached(statusCode)) throw new UncachedPing(response)
55
+ if (statusCode === RATE_LIMITED) {
56
+ return { ...response, ttl: rateLimitTtl(headers?.['retry-after']) }
57
+ }
58
+ return response
59
+ }
60
+
61
+ const unwrapUncached = error => {
62
+ if (error instanceof UncachedPing) return error.response
63
+ throw error
64
+ }
65
+
7
66
  module.exports = ({ got, pingCache }) => {
8
- const pingUrl = createPingUrl(pingCache, {
9
- value: ({ url, statusCode }) => ({ url, statusCode })
10
- })
67
+ const cachedPing = memoize(ping, pingCache, { ttl: value => value.ttl })
11
68
 
12
- const reachableUrl = async (url, opts) => {
69
+ const pingUrl = async (url, opts) => {
13
70
  const context = {}
14
- const response = await pingUrl(url, { ...got.gotOpts, ...opts, context })
71
+ const { ttl, ...response } = await cachedPing(url, {
72
+ ...got.gotOpts,
73
+ ...opts,
74
+ context
75
+ }).catch(unwrapUncached)
15
76
  const { reservedAddress } = context
16
77
 
17
78
  // reachable-url leaves the redirect hop as the status when a beforeRedirect
@@ -26,7 +87,9 @@ module.exports = ({ got, pingCache }) => {
26
87
  return { ...response, statusCode }
27
88
  }
28
89
 
29
- reachableUrl.isReachable = createPingUrl.isReachable
90
+ pingUrl.isReachable = reachableUrl.isReachable
30
91
 
31
- return reachableUrl
92
+ return pingUrl
32
93
  }
94
+
95
+ module.exports.rateLimitTtl = rateLimitTtl