@unavatar/core 3.72.1 → 3.72.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +2 -2
- package/src/util/reachable-url.js +71 -8
package/README.md
CHANGED
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "@unavatar/core",
|
|
3
3
|
"description": "Get unified user avatar from social networks, including Instagram, SoundCloud, Telegram, Twitter, YouTube & more.",
|
|
4
4
|
"homepage": "https://unavatar.io",
|
|
5
|
-
"version": "3.72.
|
|
5
|
+
"version": "3.72.3",
|
|
6
6
|
"main": "src/index.js",
|
|
7
7
|
"exports": {
|
|
8
8
|
".": "./src/index.js",
|
|
@@ -127,7 +127,6 @@
|
|
|
127
127
|
"@keyvhq/offline": "~2.2.0",
|
|
128
128
|
"@keyvhq/redis": "~2.2.3",
|
|
129
129
|
"@metascraper/helpers": "~5.58.4",
|
|
130
|
-
"@microlink/ping-url": "~1.5.0",
|
|
131
130
|
"bimi-url": "~1.0.0",
|
|
132
131
|
"browserless": "~14.1.1",
|
|
133
132
|
"cacheable-lookup": "~6.1.0",
|
|
@@ -147,6 +146,7 @@
|
|
|
147
146
|
"p-timeout": "~4.1.0",
|
|
148
147
|
"puppeteer": "~25.16.0",
|
|
149
148
|
"re2": "~1.27.0",
|
|
149
|
+
"reachable-url": "~2.1.10",
|
|
150
150
|
"srcset": "~4.0.0",
|
|
151
151
|
"stable-regex": "~1.0.1",
|
|
152
152
|
"tangerine": "~2.1.3",
|
|
@@ -1,17 +1,78 @@
|
|
|
1
1
|
'use strict'
|
|
2
2
|
|
|
3
|
-
const
|
|
3
|
+
const memoize = require('@keyvhq/memoize')
|
|
4
|
+
const reachableUrl = require('reachable-url')
|
|
5
|
+
const ms = require('ms')
|
|
6
|
+
|
|
7
|
+
const RATE_LIMITED = 429
|
|
8
|
+
const UNCACHED_STATUS_CODES = new Set([403, 408])
|
|
9
|
+
const RATE_LIMIT_TTL_DEFAULT = ms('1m')
|
|
10
|
+
const RATE_LIMIT_TTL_MIN = ms('1s')
|
|
11
|
+
const RATE_LIMIT_TTL_MAX = ms('10m')
|
|
4
12
|
|
|
5
13
|
const isRedirectStatus = statusCode => statusCode >= 300 && statusCode < 400
|
|
6
14
|
|
|
15
|
+
const isUncached = statusCode =>
|
|
16
|
+
UNCACHED_STATUS_CODES.has(statusCode) || statusCode >= 500
|
|
17
|
+
|
|
18
|
+
const parseRetryAfter = (value, now = Date.now()) => {
|
|
19
|
+
if (typeof value === 'string') value = value.trim()
|
|
20
|
+
if (!value) return undefined
|
|
21
|
+
const seconds = Number(value)
|
|
22
|
+
if (Number.isFinite(seconds)) return seconds * 1000
|
|
23
|
+
if (typeof value !== 'string') return undefined
|
|
24
|
+
if (/[a-z]/i.test(value)) {
|
|
25
|
+
const date = Date.parse(value)
|
|
26
|
+
if (!Number.isNaN(date)) return date - now
|
|
27
|
+
}
|
|
28
|
+
const first = value.split(',')[0]
|
|
29
|
+
if (!first) return undefined
|
|
30
|
+
const firstSeconds = Number(first)
|
|
31
|
+
return Number.isFinite(firstSeconds) ? firstSeconds * 1000 : undefined
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const rateLimitTtl = retryAfter => {
|
|
35
|
+
const wait = parseRetryAfter(retryAfter)
|
|
36
|
+
if (wait === undefined) return RATE_LIMIT_TTL_DEFAULT
|
|
37
|
+
return Math.min(Math.max(wait, RATE_LIMIT_TTL_MIN), RATE_LIMIT_TTL_MAX)
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
class UncachedPing extends Error {
|
|
41
|
+
constructor (response) {
|
|
42
|
+
super(`ping not cached for ${response.statusCode}`)
|
|
43
|
+
this.response = response
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const ping = async (url, opts) => {
|
|
48
|
+
const {
|
|
49
|
+
url: resolvedUrl,
|
|
50
|
+
statusCode,
|
|
51
|
+
headers
|
|
52
|
+
} = await reachableUrl(url, opts)
|
|
53
|
+
const response = { url: resolvedUrl, statusCode }
|
|
54
|
+
if (isUncached(statusCode)) throw new UncachedPing(response)
|
|
55
|
+
if (statusCode === RATE_LIMITED) {
|
|
56
|
+
return { ...response, ttl: rateLimitTtl(headers?.['retry-after']) }
|
|
57
|
+
}
|
|
58
|
+
return response
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const unwrapUncached = error => {
|
|
62
|
+
if (error instanceof UncachedPing) return error.response
|
|
63
|
+
throw error
|
|
64
|
+
}
|
|
65
|
+
|
|
7
66
|
module.exports = ({ got, pingCache }) => {
|
|
8
|
-
const
|
|
9
|
-
value: ({ url, statusCode }) => ({ url, statusCode })
|
|
10
|
-
})
|
|
67
|
+
const cachedPing = memoize(ping, pingCache, { ttl: value => value.ttl })
|
|
11
68
|
|
|
12
|
-
const
|
|
69
|
+
const pingUrl = async (url, opts) => {
|
|
13
70
|
const context = {}
|
|
14
|
-
const response = await
|
|
71
|
+
const { ttl, ...response } = await cachedPing(url, {
|
|
72
|
+
...got.gotOpts,
|
|
73
|
+
...opts,
|
|
74
|
+
context
|
|
75
|
+
}).catch(unwrapUncached)
|
|
15
76
|
const { reservedAddress } = context
|
|
16
77
|
|
|
17
78
|
// reachable-url leaves the redirect hop as the status when a beforeRedirect
|
|
@@ -26,7 +87,9 @@ module.exports = ({ got, pingCache }) => {
|
|
|
26
87
|
return { ...response, statusCode }
|
|
27
88
|
}
|
|
28
89
|
|
|
29
|
-
|
|
90
|
+
pingUrl.isReachable = reachableUrl.isReachable
|
|
30
91
|
|
|
31
|
-
return
|
|
92
|
+
return pingUrl
|
|
32
93
|
}
|
|
94
|
+
|
|
95
|
+
module.exports.rateLimitTtl = rateLimitTtl
|