@unavatar/core 3.71.12 → 3.72.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/index.js +9 -1
- package/src/providers/bilibili.js +24 -16
- package/src/providers/deezer.js +31 -16
- package/src/providers/opencollective.js +15 -11
- package/src/providers/zhihu.js +22 -18
- package/src/util/api-provider.js +153 -0
- package/src/util/html-provider.js +11 -5
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "@unavatar/core",
|
|
3
3
|
"description": "Get unified user avatar from social networks, including Instagram, SoundCloud, Telegram, Twitter, YouTube & more.",
|
|
4
4
|
"homepage": "https://unavatar.io",
|
|
5
|
-
"version": "3.
|
|
5
|
+
"version": "3.72.0",
|
|
6
6
|
"main": "src/index.js",
|
|
7
7
|
"exports": {
|
|
8
8
|
".": "./src/index.js",
|
package/src/index.js
CHANGED
|
@@ -5,10 +5,12 @@ const DEFAULTS = require('./constant')
|
|
|
5
5
|
module.exports = ({
|
|
6
6
|
constants: userConstants,
|
|
7
7
|
redis,
|
|
8
|
+
onFetch,
|
|
8
9
|
onFetchHTML,
|
|
9
10
|
userAgent
|
|
10
11
|
} = {}) => {
|
|
11
12
|
const constants = { ...DEFAULTS, ...userConstants }
|
|
13
|
+
const fetchHook = onFetch ?? onFetchHTML
|
|
12
14
|
|
|
13
15
|
if (userConstants?.REQUEST_TIMEOUT && !userConstants?.PROXY_TIMEOUT) {
|
|
14
16
|
constants.PROXY_TIMEOUT = Math.floor(constants.REQUEST_TIMEOUT * (1 / 3))
|
|
@@ -40,13 +42,19 @@ module.exports = ({
|
|
|
40
42
|
require('./util/html-provider')({
|
|
41
43
|
...constants,
|
|
42
44
|
getHTML,
|
|
43
|
-
|
|
45
|
+
onFetch: fetchHook,
|
|
44
46
|
userAgent
|
|
45
47
|
})
|
|
48
|
+
const { createApiProvider } = require('./util/api-provider')({
|
|
49
|
+
got,
|
|
50
|
+
PROXY_TIMEOUT: constants.PROXY_TIMEOUT,
|
|
51
|
+
onFetch: fetchHook
|
|
52
|
+
})
|
|
46
53
|
|
|
47
54
|
const providerCtx = {
|
|
48
55
|
constants,
|
|
49
56
|
createHtmlProvider,
|
|
57
|
+
createApiProvider,
|
|
50
58
|
getOgImage,
|
|
51
59
|
NOT_FOUND,
|
|
52
60
|
got,
|
|
@@ -2,26 +2,34 @@
|
|
|
2
2
|
|
|
3
3
|
const API_URL = 'https://api.bilibili.com/x/web-interface/card'
|
|
4
4
|
|
|
5
|
+
const OK_CODE = 0
|
|
6
|
+
const MISSING_CODES = new Set([-400, -404])
|
|
7
|
+
const ANTIBOT_CODES = new Set([-352, -412, -509, -799])
|
|
8
|
+
|
|
5
9
|
const getAvatarUrl = input => `${API_URL}?mid=${encodeURIComponent(input)}`
|
|
6
10
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
11
|
+
const isPlaceholderFace = face => face.endsWith('/noface.jpg')
|
|
12
|
+
|
|
13
|
+
const isBlocked = ({ body }) => ANTIBOT_CODES.has(body?.code)
|
|
14
|
+
|
|
15
|
+
const getAvatar = ({ body, NOT_FOUND }) => {
|
|
16
|
+
if (MISSING_CODES.has(body?.code)) return NOT_FOUND
|
|
17
|
+
if (body?.code !== OK_CODE) return undefined
|
|
10
18
|
const face = body.data?.card?.face
|
|
11
|
-
|
|
19
|
+
if (!face || isPlaceholderFace(face)) return NOT_FOUND
|
|
20
|
+
return face
|
|
12
21
|
}
|
|
13
22
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
})
|
|
20
|
-
|
|
21
|
-
if (statusCode >= 400) return
|
|
23
|
+
const factory = ({ createApiProvider, NOT_FOUND }) =>
|
|
24
|
+
createApiProvider({
|
|
25
|
+
name: 'bilibili',
|
|
26
|
+
url: getAvatarUrl,
|
|
27
|
+
isBlocked,
|
|
28
|
+
getter: ({ body }) => getAvatar({ body, NOT_FOUND })
|
|
29
|
+
})
|
|
22
30
|
|
|
23
|
-
|
|
24
|
-
|
|
31
|
+
factory.getAvatarUrl = getAvatarUrl
|
|
32
|
+
factory.getAvatar = getAvatar
|
|
33
|
+
factory.isBlocked = isBlocked
|
|
25
34
|
|
|
26
|
-
module.exports
|
|
27
|
-
module.exports.getAvatar = getAvatar
|
|
35
|
+
module.exports = factory
|
package/src/providers/deezer.js
CHANGED
|
@@ -2,6 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
const API_URL = 'https://api.deezer.com'
|
|
4
4
|
|
|
5
|
+
const MISSING_ERROR_CODES = new Set([600, 800])
|
|
6
|
+
const QUOTA_ERROR_CODE = 4
|
|
7
|
+
|
|
5
8
|
// Entities without artwork keep an empty image hash
|
|
6
9
|
// (cdn-images.dzcdn.net/images/artist//500x500.jpg), which resolves to a
|
|
7
10
|
// generic placeholder, so any empty hash segment is treated as a miss.
|
|
@@ -20,25 +23,37 @@ const getAvatarUrl = input => {
|
|
|
20
23
|
return `${API_URL}/${encodeURIComponent(type)}/${encodeURIComponent(id)}`
|
|
21
24
|
}
|
|
22
25
|
|
|
23
|
-
const
|
|
24
|
-
|
|
26
|
+
const isBlocked = ({ body }) => body?.error?.code === QUOTA_ERROR_CODE
|
|
27
|
+
|
|
28
|
+
const getAvatar = ({ body, NOT_FOUND }) => {
|
|
29
|
+
if (MISSING_ERROR_CODES.has(body?.error?.code)) return NOT_FOUND
|
|
30
|
+
if (body?.id === undefined) return undefined
|
|
25
31
|
|
|
26
|
-
const pic = body
|
|
27
|
-
|
|
32
|
+
const pic = body.picture_xl || body.cover_xl || body.album?.cover_xl
|
|
33
|
+
if (typeof pic !== 'string' || !pic || !isArtwork(pic)) return NOT_FOUND
|
|
34
|
+
return pic
|
|
28
35
|
}
|
|
29
36
|
|
|
30
|
-
|
|
31
|
-
async function deezer (input) {
|
|
32
|
-
const { body, statusCode } = await got(getAvatarUrl(input), {
|
|
33
|
-
responseType: 'json',
|
|
34
|
-
throwHttpErrors: false
|
|
35
|
-
})
|
|
37
|
+
const hasWhitespace = input => /\s/.test(input)
|
|
36
38
|
|
|
37
|
-
|
|
39
|
+
const factory = ({ createApiProvider, NOT_FOUND }) => {
|
|
40
|
+
const apiProvider = createApiProvider({
|
|
41
|
+
name: 'deezer',
|
|
42
|
+
url: getAvatarUrl,
|
|
43
|
+
isBlocked,
|
|
44
|
+
getter: ({ body }) => getAvatar({ body, NOT_FOUND })
|
|
45
|
+
})
|
|
38
46
|
|
|
39
|
-
|
|
40
|
-
|
|
47
|
+
const provider = (input, context) =>
|
|
48
|
+
hasWhitespace(input) ? undefined : apiProvider(input, context)
|
|
49
|
+
|
|
50
|
+
provider.getUrl = getAvatarUrl
|
|
51
|
+
return provider
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
factory.parseInput = parseInput
|
|
55
|
+
factory.getAvatarUrl = getAvatarUrl
|
|
56
|
+
factory.getAvatar = getAvatar
|
|
57
|
+
factory.isBlocked = isBlocked
|
|
41
58
|
|
|
42
|
-
module.exports
|
|
43
|
-
module.exports.getAvatarUrl = getAvatarUrl
|
|
44
|
-
module.exports.getAvatar = getAvatar
|
|
59
|
+
module.exports = factory
|
|
@@ -11,17 +11,21 @@ const getAvatarUrl = ({ constants, slug }) =>
|
|
|
11
11
|
constants.AVATAR_SIZE
|
|
12
12
|
}.png`
|
|
13
13
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
})
|
|
14
|
+
const getAvatar = ({ body, constants, NOT_FOUND }) => {
|
|
15
|
+
if (!body?.slug) return undefined
|
|
16
|
+
if (!body.image) return NOT_FOUND
|
|
17
|
+
return getAvatarUrl({ constants, slug: body.slug })
|
|
18
|
+
}
|
|
20
19
|
|
|
21
|
-
|
|
20
|
+
const factory = ({ constants, createApiProvider, NOT_FOUND }) =>
|
|
21
|
+
createApiProvider({
|
|
22
|
+
name: 'opencollective',
|
|
23
|
+
url: getProfileUrl,
|
|
24
|
+
getter: ({ body }) => getAvatar({ body, constants, NOT_FOUND })
|
|
25
|
+
})
|
|
22
26
|
|
|
23
|
-
|
|
24
|
-
|
|
27
|
+
factory.getProfileUrl = getProfileUrl
|
|
28
|
+
factory.getAvatarUrl = getAvatarUrl
|
|
29
|
+
factory.getAvatar = getAvatar
|
|
25
30
|
|
|
26
|
-
module.exports
|
|
27
|
-
module.exports.getAvatarUrl = getAvatarUrl
|
|
31
|
+
module.exports = factory
|
package/src/providers/zhihu.js
CHANGED
|
@@ -9,28 +9,32 @@ const randomDesktopUserAgent = uniqueRandomArray(
|
|
|
9
9
|
|
|
10
10
|
const API_URL = 'https://www.zhihu.com/api/v4/members'
|
|
11
11
|
|
|
12
|
+
const ANTIBOT_ERROR_CODE = 40352
|
|
13
|
+
const MISSING_MEMBER_ERROR_CODE = 4041
|
|
14
|
+
|
|
12
15
|
const getAvatarUrl = input =>
|
|
13
16
|
`${API_URL}/${encodeURIComponent(input)}?include=avatar_url`
|
|
14
17
|
|
|
15
|
-
|
|
16
|
-
// a plain 404, and default avatars are flagged via `use_default_avatar`.
|
|
17
|
-
const getAvatar = body => {
|
|
18
|
-
if (!body || body.error || body.use_default_avatar) return
|
|
19
|
-
return body.avatar_url || undefined
|
|
20
|
-
}
|
|
18
|
+
const isBlocked = ({ body }) => body?.error?.code === ANTIBOT_ERROR_CODE
|
|
21
19
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
})
|
|
20
|
+
const getAvatar = ({ body, NOT_FOUND }) => {
|
|
21
|
+
if (body?.error?.code === MISSING_MEMBER_ERROR_CODE) return NOT_FOUND
|
|
22
|
+
if (body?.error) return undefined
|
|
23
|
+
if (body?.use_default_avatar) return NOT_FOUND
|
|
24
|
+
return body?.avatar_url || undefined
|
|
25
|
+
}
|
|
29
26
|
|
|
30
|
-
|
|
27
|
+
const factory = ({ createApiProvider, NOT_FOUND }) =>
|
|
28
|
+
createApiProvider({
|
|
29
|
+
name: 'zhihu',
|
|
30
|
+
url: getAvatarUrl,
|
|
31
|
+
gotOpts: () => ({ headers: { 'user-agent': randomDesktopUserAgent() } }),
|
|
32
|
+
isBlocked,
|
|
33
|
+
getter: ({ body }) => getAvatar({ body, NOT_FOUND })
|
|
34
|
+
})
|
|
31
35
|
|
|
32
|
-
|
|
33
|
-
|
|
36
|
+
factory.getAvatarUrl = getAvatarUrl
|
|
37
|
+
factory.getAvatar = getAvatar
|
|
38
|
+
factory.isBlocked = isBlocked
|
|
34
39
|
|
|
35
|
-
module.exports
|
|
36
|
-
module.exports.getAvatar = getAvatar
|
|
40
|
+
module.exports = factory
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
const debug = require('debug-logfmt')('api-provider')
|
|
4
|
+
|
|
5
|
+
const httpStatus = require('./http-status')
|
|
6
|
+
const ExtendableError = require('./error')
|
|
7
|
+
const { NOT_FOUND } = require('./html-provider')
|
|
8
|
+
|
|
9
|
+
const BLOCKED_STATUS_CODES = new Set([
|
|
10
|
+
httpStatus.FORBIDDEN,
|
|
11
|
+
httpStatus.TOO_MANY_REQUESTS
|
|
12
|
+
])
|
|
13
|
+
|
|
14
|
+
const MISS = {
|
|
15
|
+
blocked: {
|
|
16
|
+
code: 'blocked',
|
|
17
|
+
blocked: true,
|
|
18
|
+
message: 'The provider blocked the request.'
|
|
19
|
+
},
|
|
20
|
+
upstreamError: {
|
|
21
|
+
code: 'upstream_error',
|
|
22
|
+
blocked: false,
|
|
23
|
+
message: 'The provider answered with an error.'
|
|
24
|
+
},
|
|
25
|
+
empty: {
|
|
26
|
+
code: 'empty_getter_result',
|
|
27
|
+
blocked: false,
|
|
28
|
+
message: 'Empty value returned by the provider.'
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const createProviderError = ({ provider, statusCode, miss }) =>
|
|
33
|
+
new ExtendableError({ provider, statusCode, ...miss })
|
|
34
|
+
|
|
35
|
+
const BODY_PARSE_FAILURE = 'ERR_BODY_PARSE_FAILURE'
|
|
36
|
+
|
|
37
|
+
const isProviderMiss = error =>
|
|
38
|
+
error instanceof ExtendableError && typeof error.blocked === 'boolean'
|
|
39
|
+
|
|
40
|
+
const swallowProviderMiss = error => {
|
|
41
|
+
if (isProviderMiss(error)) return undefined
|
|
42
|
+
throw error
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const toTierGotOpts = ({ timeout, agent } = {}) => ({
|
|
46
|
+
...(agent && { agent }),
|
|
47
|
+
...(timeout && { timeout: { request: timeout } })
|
|
48
|
+
})
|
|
49
|
+
|
|
50
|
+
module.exports = ({ got, PROXY_TIMEOUT, onFetch }) => {
|
|
51
|
+
const createApiProvider = ({ name, url, getter, isBlocked, gotOpts }) => {
|
|
52
|
+
async function provider (input, context) {
|
|
53
|
+
const providerUrl = await url(input)
|
|
54
|
+
|
|
55
|
+
const attempt = async ({
|
|
56
|
+
tier = 'origin',
|
|
57
|
+
gotOpts: tierGotOpts
|
|
58
|
+
} = {}) => {
|
|
59
|
+
const log = debug.duration({ provider: name, input, providerUrl, tier })
|
|
60
|
+
|
|
61
|
+
const { body, statusCode } = await got(providerUrl, {
|
|
62
|
+
timeout: { request: PROXY_TIMEOUT },
|
|
63
|
+
...gotOpts?.(),
|
|
64
|
+
responseType: 'json',
|
|
65
|
+
throwHttpErrors: false,
|
|
66
|
+
retry: { limit: 0 },
|
|
67
|
+
...toTierGotOpts(tierGotOpts)
|
|
68
|
+
}).catch(error => {
|
|
69
|
+
if (error.code === BODY_PARSE_FAILURE) {
|
|
70
|
+
log.error({
|
|
71
|
+
statusCode: error.response?.statusCode,
|
|
72
|
+
status: 'blocked',
|
|
73
|
+
reason: 'non_json_body'
|
|
74
|
+
})
|
|
75
|
+
throw createProviderError({
|
|
76
|
+
provider: name,
|
|
77
|
+
statusCode: error.response?.statusCode,
|
|
78
|
+
miss: MISS.blocked
|
|
79
|
+
})
|
|
80
|
+
}
|
|
81
|
+
log.error({
|
|
82
|
+
status: 'request_failed',
|
|
83
|
+
code: error.code,
|
|
84
|
+
message: error.message
|
|
85
|
+
})
|
|
86
|
+
throw error
|
|
87
|
+
})
|
|
88
|
+
|
|
89
|
+
const response = { body, statusCode }
|
|
90
|
+
|
|
91
|
+
if (BLOCKED_STATUS_CODES.has(statusCode) || isBlocked?.(response)) {
|
|
92
|
+
log.error({ statusCode, status: 'blocked' })
|
|
93
|
+
throw createProviderError({
|
|
94
|
+
provider: name,
|
|
95
|
+
statusCode,
|
|
96
|
+
miss: MISS.blocked
|
|
97
|
+
})
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
if (statusCode === httpStatus.NOT_FOUND) {
|
|
101
|
+
log.error({ statusCode, status: 'not_found' })
|
|
102
|
+
return NOT_FOUND
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
if (statusCode >= httpStatus.BAD_REQUEST) {
|
|
106
|
+
log.error({ statusCode, status: 'upstream_error' })
|
|
107
|
+
throw createProviderError({
|
|
108
|
+
provider: name,
|
|
109
|
+
statusCode,
|
|
110
|
+
miss: MISS.upstreamError
|
|
111
|
+
})
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const result = getter(response)
|
|
115
|
+
|
|
116
|
+
if (result === NOT_FOUND) {
|
|
117
|
+
log.error({ statusCode, status: 'not_found' })
|
|
118
|
+
return NOT_FOUND
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
if (typeof result === 'string' && result !== '') {
|
|
122
|
+
log({ statusCode, status: 'success', result })
|
|
123
|
+
return result
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
log.error({ statusCode, status: 'empty' })
|
|
127
|
+
throw createProviderError({
|
|
128
|
+
provider: name,
|
|
129
|
+
statusCode,
|
|
130
|
+
miss: MISS.empty
|
|
131
|
+
})
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
const result =
|
|
135
|
+
typeof onFetch === 'function'
|
|
136
|
+
? await onFetch({
|
|
137
|
+
...context,
|
|
138
|
+
attempt,
|
|
139
|
+
provider: name,
|
|
140
|
+
providerUrl,
|
|
141
|
+
kind: 'api'
|
|
142
|
+
})
|
|
143
|
+
: await attempt().catch(swallowProviderMiss)
|
|
144
|
+
|
|
145
|
+
return result === NOT_FOUND ? undefined : result
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
provider.getUrl = url
|
|
149
|
+
return provider
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
return { createApiProvider }
|
|
153
|
+
}
|
|
@@ -36,7 +36,7 @@ const createErrorCause = ({ html, headers, statusCode }) => ({
|
|
|
36
36
|
module.exports = ({
|
|
37
37
|
PROXY_TIMEOUT,
|
|
38
38
|
getHTML,
|
|
39
|
-
|
|
39
|
+
onFetch,
|
|
40
40
|
userAgent = require('./crawler-agent')
|
|
41
41
|
}) => {
|
|
42
42
|
/**
|
|
@@ -177,11 +177,17 @@ module.exports = ({
|
|
|
177
177
|
throw error
|
|
178
178
|
}
|
|
179
179
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
180
|
+
const result =
|
|
181
|
+
typeof onFetch === 'function'
|
|
182
|
+
? await onFetch({
|
|
183
|
+
...context,
|
|
184
|
+
attempt,
|
|
185
|
+
provider: name,
|
|
186
|
+
providerUrl,
|
|
187
|
+
kind: 'html'
|
|
188
|
+
})
|
|
189
|
+
: await attempt()
|
|
183
190
|
|
|
184
|
-
const result = await attempt()
|
|
185
191
|
return result === NOT_FOUND ? undefined : result
|
|
186
192
|
}
|
|
187
193
|
|