@unavatar/core 3.71.12 → 3.72.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "name": "@unavatar/core",
3
3
  "description": "Get unified user avatar from social networks, including Instagram, SoundCloud, Telegram, Twitter, YouTube & more.",
4
4
  "homepage": "https://unavatar.io",
5
- "version": "3.71.12",
5
+ "version": "3.72.0",
6
6
  "main": "src/index.js",
7
7
  "exports": {
8
8
  ".": "./src/index.js",
package/src/index.js CHANGED
@@ -5,10 +5,12 @@ const DEFAULTS = require('./constant')
5
5
  module.exports = ({
6
6
  constants: userConstants,
7
7
  redis,
8
+ onFetch,
8
9
  onFetchHTML,
9
10
  userAgent
10
11
  } = {}) => {
11
12
  const constants = { ...DEFAULTS, ...userConstants }
13
+ const fetchHook = onFetch ?? onFetchHTML
12
14
 
13
15
  if (userConstants?.REQUEST_TIMEOUT && !userConstants?.PROXY_TIMEOUT) {
14
16
  constants.PROXY_TIMEOUT = Math.floor(constants.REQUEST_TIMEOUT * (1 / 3))
@@ -40,13 +42,19 @@ module.exports = ({
40
42
  require('./util/html-provider')({
41
43
  ...constants,
42
44
  getHTML,
43
- onFetchHTML,
45
+ onFetch: fetchHook,
44
46
  userAgent
45
47
  })
48
+ const { createApiProvider } = require('./util/api-provider')({
49
+ got,
50
+ PROXY_TIMEOUT: constants.PROXY_TIMEOUT,
51
+ onFetch: fetchHook
52
+ })
46
53
 
47
54
  const providerCtx = {
48
55
  constants,
49
56
  createHtmlProvider,
57
+ createApiProvider,
50
58
  getOgImage,
51
59
  NOT_FOUND,
52
60
  got,
@@ -2,26 +2,34 @@
2
2
 
3
3
  const API_URL = 'https://api.bilibili.com/x/web-interface/card'
4
4
 
5
+ const OK_CODE = 0
6
+ const MISSING_CODES = new Set([-400, -404])
7
+ const ANTIBOT_CODES = new Set([-352, -412, -509, -799])
8
+
5
9
  const getAvatarUrl = input => `${API_URL}?mid=${encodeURIComponent(input)}`
6
10
 
7
- // Users without a custom avatar get the generic `noface.jpg` placeholder.
8
- const getAvatar = body => {
9
- if (body?.code !== 0) return
11
+ const isPlaceholderFace = face => face.endsWith('/noface.jpg')
12
+
13
+ const isBlocked = ({ body }) => ANTIBOT_CODES.has(body?.code)
14
+
15
+ const getAvatar = ({ body, NOT_FOUND }) => {
16
+ if (MISSING_CODES.has(body?.code)) return NOT_FOUND
17
+ if (body?.code !== OK_CODE) return undefined
10
18
  const face = body.data?.card?.face
11
- return face && !face.endsWith('/noface.jpg') ? face : undefined
19
+ if (!face || isPlaceholderFace(face)) return NOT_FOUND
20
+ return face
12
21
  }
13
22
 
14
- module.exports = ({ got }) =>
15
- async function bilibili (input) {
16
- const { body, statusCode } = await got(getAvatarUrl(input), {
17
- responseType: 'json',
18
- throwHttpErrors: false
19
- })
20
-
21
- if (statusCode >= 400) return
23
+ const factory = ({ createApiProvider, NOT_FOUND }) =>
24
+ createApiProvider({
25
+ name: 'bilibili',
26
+ url: getAvatarUrl,
27
+ isBlocked,
28
+ getter: ({ body }) => getAvatar({ body, NOT_FOUND })
29
+ })
22
30
 
23
- return getAvatar(body)
24
- }
31
+ factory.getAvatarUrl = getAvatarUrl
32
+ factory.getAvatar = getAvatar
33
+ factory.isBlocked = isBlocked
25
34
 
26
- module.exports.getAvatarUrl = getAvatarUrl
27
- module.exports.getAvatar = getAvatar
35
+ module.exports = factory
@@ -2,6 +2,9 @@
2
2
 
3
3
  const API_URL = 'https://api.deezer.com'
4
4
 
5
+ const MISSING_ERROR_CODES = new Set([600, 800])
6
+ const QUOTA_ERROR_CODE = 4
7
+
5
8
  // Entities without artwork keep an empty image hash
6
9
  // (cdn-images.dzcdn.net/images/artist//500x500.jpg), which resolves to a
7
10
  // generic placeholder, so any empty hash segment is treated as a miss.
@@ -20,25 +23,37 @@ const getAvatarUrl = input => {
20
23
  return `${API_URL}/${encodeURIComponent(type)}/${encodeURIComponent(id)}`
21
24
  }
22
25
 
23
- const getAvatar = body => {
24
- if (body?.error) return
26
+ const isBlocked = ({ body }) => body?.error?.code === QUOTA_ERROR_CODE
27
+
28
+ const getAvatar = ({ body, NOT_FOUND }) => {
29
+ if (MISSING_ERROR_CODES.has(body?.error?.code)) return NOT_FOUND
30
+ if (body?.id === undefined) return undefined
25
31
 
26
- const pic = body?.picture_xl || body?.cover_xl || body?.album?.cover_xl
27
- return typeof pic === 'string' && pic && isArtwork(pic) ? pic : undefined
32
+ const pic = body.picture_xl || body.cover_xl || body.album?.cover_xl
33
+ if (typeof pic !== 'string' || !pic || !isArtwork(pic)) return NOT_FOUND
34
+ return pic
28
35
  }
29
36
 
30
- module.exports = ({ got }) =>
31
- async function deezer (input) {
32
- const { body, statusCode } = await got(getAvatarUrl(input), {
33
- responseType: 'json',
34
- throwHttpErrors: false
35
- })
37
+ const hasWhitespace = input => /\s/.test(input)
36
38
 
37
- if (statusCode >= 400) return
39
+ const factory = ({ createApiProvider, NOT_FOUND }) => {
40
+ const apiProvider = createApiProvider({
41
+ name: 'deezer',
42
+ url: getAvatarUrl,
43
+ isBlocked,
44
+ getter: ({ body }) => getAvatar({ body, NOT_FOUND })
45
+ })
38
46
 
39
- return getAvatar(body)
40
- }
47
+ const provider = (input, context) =>
48
+ hasWhitespace(input) ? undefined : apiProvider(input, context)
49
+
50
+ provider.getUrl = getAvatarUrl
51
+ return provider
52
+ }
53
+
54
+ factory.parseInput = parseInput
55
+ factory.getAvatarUrl = getAvatarUrl
56
+ factory.getAvatar = getAvatar
57
+ factory.isBlocked = isBlocked
41
58
 
42
- module.exports.parseInput = parseInput
43
- module.exports.getAvatarUrl = getAvatarUrl
44
- module.exports.getAvatar = getAvatar
59
+ module.exports = factory
@@ -11,17 +11,21 @@ const getAvatarUrl = ({ constants, slug }) =>
11
11
  constants.AVATAR_SIZE
12
12
  }.png`
13
13
 
14
- module.exports = ({ constants, got }) =>
15
- async function opencollective (input) {
16
- const { body, statusCode } = await got(getProfileUrl(input), {
17
- responseType: 'json',
18
- throwHttpErrors: false
19
- })
14
+ const getAvatar = ({ body, constants, NOT_FOUND }) => {
15
+ if (!body?.slug) return undefined
16
+ if (!body.image) return NOT_FOUND
17
+ return getAvatarUrl({ constants, slug: body.slug })
18
+ }
20
19
 
21
- if (statusCode >= 400 || !body?.image || !body.slug) return
20
+ const factory = ({ constants, createApiProvider, NOT_FOUND }) =>
21
+ createApiProvider({
22
+ name: 'opencollective',
23
+ url: getProfileUrl,
24
+ getter: ({ body }) => getAvatar({ body, constants, NOT_FOUND })
25
+ })
22
26
 
23
- return getAvatarUrl({ constants, slug: body.slug })
24
- }
27
+ factory.getProfileUrl = getProfileUrl
28
+ factory.getAvatarUrl = getAvatarUrl
29
+ factory.getAvatar = getAvatar
25
30
 
26
- module.exports.getProfileUrl = getProfileUrl
27
- module.exports.getAvatarUrl = getAvatarUrl
31
+ module.exports = factory
@@ -9,28 +9,32 @@ const randomDesktopUserAgent = uniqueRandomArray(
9
9
 
10
10
  const API_URL = 'https://www.zhihu.com/api/v4/members'
11
11
 
12
+ const ANTIBOT_ERROR_CODE = 40352
13
+ const MISSING_MEMBER_ERROR_CODE = 4041
14
+
12
15
  const getAvatarUrl = input =>
13
16
  `${API_URL}/${encodeURIComponent(input)}?include=avatar_url`
14
17
 
15
- // Misses (and bot challenges) come back as an `error` payload rather than
16
- // a plain 404, and default avatars are flagged via `use_default_avatar`.
17
- const getAvatar = body => {
18
- if (!body || body.error || body.use_default_avatar) return
19
- return body.avatar_url || undefined
20
- }
18
+ const isBlocked = ({ body }) => body?.error?.code === ANTIBOT_ERROR_CODE
21
19
 
22
- module.exports = ({ got }) =>
23
- async function zhihu (input) {
24
- const { body, statusCode } = await got(getAvatarUrl(input), {
25
- responseType: 'json',
26
- throwHttpErrors: false,
27
- headers: { 'user-agent': randomDesktopUserAgent() }
28
- })
20
+ const getAvatar = ({ body, NOT_FOUND }) => {
21
+ if (body?.error?.code === MISSING_MEMBER_ERROR_CODE) return NOT_FOUND
22
+ if (body?.error) return undefined
23
+ if (body?.use_default_avatar) return NOT_FOUND
24
+ return body?.avatar_url || undefined
25
+ }
29
26
 
30
- if (statusCode >= 400) return
27
+ const factory = ({ createApiProvider, NOT_FOUND }) =>
28
+ createApiProvider({
29
+ name: 'zhihu',
30
+ url: getAvatarUrl,
31
+ gotOpts: () => ({ headers: { 'user-agent': randomDesktopUserAgent() } }),
32
+ isBlocked,
33
+ getter: ({ body }) => getAvatar({ body, NOT_FOUND })
34
+ })
31
35
 
32
- return getAvatar(body)
33
- }
36
+ factory.getAvatarUrl = getAvatarUrl
37
+ factory.getAvatar = getAvatar
38
+ factory.isBlocked = isBlocked
34
39
 
35
- module.exports.getAvatarUrl = getAvatarUrl
36
- module.exports.getAvatar = getAvatar
40
+ module.exports = factory
@@ -0,0 +1,153 @@
1
+ 'use strict'
2
+
3
+ const debug = require('debug-logfmt')('api-provider')
4
+
5
+ const httpStatus = require('./http-status')
6
+ const ExtendableError = require('./error')
7
+ const { NOT_FOUND } = require('./html-provider')
8
+
9
+ const BLOCKED_STATUS_CODES = new Set([
10
+ httpStatus.FORBIDDEN,
11
+ httpStatus.TOO_MANY_REQUESTS
12
+ ])
13
+
14
+ const MISS = {
15
+ blocked: {
16
+ code: 'blocked',
17
+ blocked: true,
18
+ message: 'The provider blocked the request.'
19
+ },
20
+ upstreamError: {
21
+ code: 'upstream_error',
22
+ blocked: false,
23
+ message: 'The provider answered with an error.'
24
+ },
25
+ empty: {
26
+ code: 'empty_getter_result',
27
+ blocked: false,
28
+ message: 'Empty value returned by the provider.'
29
+ }
30
+ }
31
+
32
+ const createProviderError = ({ provider, statusCode, miss }) =>
33
+ new ExtendableError({ provider, statusCode, ...miss })
34
+
35
+ const BODY_PARSE_FAILURE = 'ERR_BODY_PARSE_FAILURE'
36
+
37
+ const isProviderMiss = error =>
38
+ error instanceof ExtendableError && typeof error.blocked === 'boolean'
39
+
40
+ const swallowProviderMiss = error => {
41
+ if (isProviderMiss(error)) return undefined
42
+ throw error
43
+ }
44
+
45
+ const toTierGotOpts = ({ timeout, agent } = {}) => ({
46
+ ...(agent && { agent }),
47
+ ...(timeout && { timeout: { request: timeout } })
48
+ })
49
+
50
+ module.exports = ({ got, PROXY_TIMEOUT, onFetch }) => {
51
+ const createApiProvider = ({ name, url, getter, isBlocked, gotOpts }) => {
52
+ async function provider (input, context) {
53
+ const providerUrl = await url(input)
54
+
55
+ const attempt = async ({
56
+ tier = 'origin',
57
+ gotOpts: tierGotOpts
58
+ } = {}) => {
59
+ const log = debug.duration({ provider: name, input, providerUrl, tier })
60
+
61
+ const { body, statusCode } = await got(providerUrl, {
62
+ timeout: { request: PROXY_TIMEOUT },
63
+ ...gotOpts?.(),
64
+ responseType: 'json',
65
+ throwHttpErrors: false,
66
+ retry: { limit: 0 },
67
+ ...toTierGotOpts(tierGotOpts)
68
+ }).catch(error => {
69
+ if (error.code === BODY_PARSE_FAILURE) {
70
+ log.error({
71
+ statusCode: error.response?.statusCode,
72
+ status: 'blocked',
73
+ reason: 'non_json_body'
74
+ })
75
+ throw createProviderError({
76
+ provider: name,
77
+ statusCode: error.response?.statusCode,
78
+ miss: MISS.blocked
79
+ })
80
+ }
81
+ log.error({
82
+ status: 'request_failed',
83
+ code: error.code,
84
+ message: error.message
85
+ })
86
+ throw error
87
+ })
88
+
89
+ const response = { body, statusCode }
90
+
91
+ if (BLOCKED_STATUS_CODES.has(statusCode) || isBlocked?.(response)) {
92
+ log.error({ statusCode, status: 'blocked' })
93
+ throw createProviderError({
94
+ provider: name,
95
+ statusCode,
96
+ miss: MISS.blocked
97
+ })
98
+ }
99
+
100
+ if (statusCode === httpStatus.NOT_FOUND) {
101
+ log.error({ statusCode, status: 'not_found' })
102
+ return NOT_FOUND
103
+ }
104
+
105
+ if (statusCode >= httpStatus.BAD_REQUEST) {
106
+ log.error({ statusCode, status: 'upstream_error' })
107
+ throw createProviderError({
108
+ provider: name,
109
+ statusCode,
110
+ miss: MISS.upstreamError
111
+ })
112
+ }
113
+
114
+ const result = getter(response)
115
+
116
+ if (result === NOT_FOUND) {
117
+ log.error({ statusCode, status: 'not_found' })
118
+ return NOT_FOUND
119
+ }
120
+
121
+ if (typeof result === 'string' && result !== '') {
122
+ log({ statusCode, status: 'success', result })
123
+ return result
124
+ }
125
+
126
+ log.error({ statusCode, status: 'empty' })
127
+ throw createProviderError({
128
+ provider: name,
129
+ statusCode,
130
+ miss: MISS.empty
131
+ })
132
+ }
133
+
134
+ const result =
135
+ typeof onFetch === 'function'
136
+ ? await onFetch({
137
+ ...context,
138
+ attempt,
139
+ provider: name,
140
+ providerUrl,
141
+ kind: 'api'
142
+ })
143
+ : await attempt().catch(swallowProviderMiss)
144
+
145
+ return result === NOT_FOUND ? undefined : result
146
+ }
147
+
148
+ provider.getUrl = url
149
+ return provider
150
+ }
151
+
152
+ return { createApiProvider }
153
+ }
@@ -36,7 +36,7 @@ const createErrorCause = ({ html, headers, statusCode }) => ({
36
36
  module.exports = ({
37
37
  PROXY_TIMEOUT,
38
38
  getHTML,
39
- onFetchHTML,
39
+ onFetch,
40
40
  userAgent = require('./crawler-agent')
41
41
  }) => {
42
42
  /**
@@ -177,11 +177,17 @@ module.exports = ({
177
177
  throw error
178
178
  }
179
179
 
180
- if (typeof onFetchHTML === 'function') {
181
- return onFetchHTML({ ...context, attempt, provider: name, providerUrl })
182
- }
180
+ const result =
181
+ typeof onFetch === 'function'
182
+ ? await onFetch({
183
+ ...context,
184
+ attempt,
185
+ provider: name,
186
+ providerUrl,
187
+ kind: 'html'
188
+ })
189
+ : await attempt()
183
190
 
184
- const result = await attempt()
185
191
  return result === NOT_FOUND ? undefined : result
186
192
  }
187
193