@sns-parse/core 0.3.0-alpha.1 → 0.4.0-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/config.d.ts +7 -0
- package/lib/config.js +28 -0
- package/lib/engine/config-io.d.ts +44 -0
- package/lib/engine/config-io.js +155 -0
- package/lib/engine/config.d.ts +9 -0
- package/lib/engine/config.js +265 -0
- package/lib/engine/fetcher.d.ts +21 -0
- package/lib/engine/fetcher.js +228 -0
- package/lib/engine/gateway.d.ts +21 -0
- package/lib/engine/gateway.js +19 -0
- package/lib/engine/parser.d.ts +2 -0
- package/lib/engine/parser.js +162 -0
- package/lib/engine/platform-config.d.ts +24 -0
- package/lib/engine/platform-config.js +76 -0
- package/lib/engine/runtime.d.ts +49 -0
- package/lib/engine/runtime.js +90 -0
- package/lib/engine/translate.d.ts +15 -0
- package/lib/engine/translate.js +107 -0
- package/lib/engine/twitter-user.d.ts +67 -0
- package/lib/engine/twitter-user.js +185 -0
- package/lib/engine/twitter.d.ts +102 -0
- package/lib/engine/twitter.js +791 -0
- package/lib/index.d.ts +21 -0
- package/lib/index.js +21 -0
- package/lib/registry.d.ts +20 -0
- package/lib/registry.js +77 -0
- package/lib/utils/cache.d.ts +9 -0
- package/lib/utils/cache.js +34 -0
- package/lib/utils/common.d.ts +7 -0
- package/lib/utils/common.js +58 -0
- package/lib/utils/concurrency.d.ts +8 -0
- package/lib/utils/concurrency.js +29 -0
- package/lib/utils/field-mapping.d.ts +2 -0
- package/lib/utils/field-mapping.js +29 -0
- package/lib/utils/format.d.ts +4 -0
- package/lib/utils/format.js +84 -0
- package/lib/utils/gif.d.ts +10 -0
- package/lib/utils/gif.js +104 -0
- package/lib/utils/image-size.d.ts +10 -0
- package/lib/utils/image-size.js +69 -0
- package/lib/utils/merge.d.ts +64 -0
- package/lib/utils/merge.js +402 -0
- package/lib/utils/tls-client.d.ts +30 -0
- package/lib/utils/tls-client.js +120 -0
- package/lib/utils/url.d.ts +10 -0
- package/lib/utils/url.js +89 -0
- package/package.json +16 -4
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.fetchApi = fetchApi;
|
|
7
|
+
exports.parseUrl = parseUrl;
|
|
8
|
+
exports.processSingleUrl = processSingleUrl;
|
|
9
|
+
const axios_1 = __importDefault(require("axios"));
|
|
10
|
+
const https_proxy_agent_1 = require("https-proxy-agent");
|
|
11
|
+
const logger_1 = require("../logger");
|
|
12
|
+
const common_1 = require("../utils/common");
|
|
13
|
+
const format_1 = require("../utils/format");
|
|
14
|
+
const parser_1 = require("./parser");
|
|
15
|
+
const platform_config_1 = require("./platform-config");
|
|
16
|
+
const twitter_1 = require("./twitter");
|
|
17
|
+
const translate_1 = require("./translate");
|
|
18
|
+
const tls_client_1 = require("../utils/tls-client");
|
|
19
|
+
const gateway_1 = require("./gateway");
|
|
20
|
+
async function fetchApi(rt, url, type, fieldMapping, platformConf) {
|
|
21
|
+
const { config, http, urlCacheLocal, proxyConfig, cacheTTL } = rt;
|
|
22
|
+
const cacheKey = url;
|
|
23
|
+
const cached = urlCacheLocal.get(cacheKey);
|
|
24
|
+
if (cached && cached.expire > Date.now()) {
|
|
25
|
+
(0, logger_1.debugLog)('缓存命中,直接返回:', url);
|
|
26
|
+
return cached.data;
|
|
27
|
+
}
|
|
28
|
+
const { apiUrl: dedicatedUrl, dedicatedFirst, apiKey, authHeaderType, customHeaderName, customProxy } = platformConf || (0, platform_config_1.getPlatformConfig)(rt, type);
|
|
29
|
+
// X / Twitter:统一网关均走原生 syndication 解析(除非用户自定义了 API)
|
|
30
|
+
if (type === 'twitter' && !dedicatedUrl) {
|
|
31
|
+
(0, logger_1.debugLog)('twitter 走原生 syndication 解析:', url);
|
|
32
|
+
const twCreds = (config.twitterAuthToken && config.twitterCt0)
|
|
33
|
+
? { authToken: String(config.twitterAuthToken), ct0: String(config.twitterCt0) }
|
|
34
|
+
: undefined;
|
|
35
|
+
// 代理串通:axios 走 runtime 实例;tlsget-rs(GraphQL/Grok 翻译)显式透传代理地址
|
|
36
|
+
const tlsProxy = (proxyConfig.enabled && proxyConfig.host)
|
|
37
|
+
? `${proxyConfig.protocol || 'http'}://${proxyConfig.auth?.username ? `${encodeURIComponent(proxyConfig.auth.username)}:${encodeURIComponent(proxyConfig.auth.password || '')}@` : ''}${proxyConfig.host}:${proxyConfig.port || 7890}`
|
|
38
|
+
: undefined;
|
|
39
|
+
const tlsGetWithProxy = (u, o) => (0, tls_client_1.tlsGet)(u, { ...o, proxy: tlsProxy });
|
|
40
|
+
const parsed = await (0, twitter_1.parseTwitter)(url, http, twCreds, tlsGetWithProxy);
|
|
41
|
+
// 推文翻译:目标语种与推文语种相同时跳过;Grok(需登录态)优先,通用翻译兜底
|
|
42
|
+
if (config.tweetTranslateEnabled && parsed.desc) {
|
|
43
|
+
const target = config.tweetTranslateLang || 'zh';
|
|
44
|
+
if (!(0, translate_1.shouldSkipTranslate)(parsed.lang, target)) {
|
|
45
|
+
let translated = null;
|
|
46
|
+
let provider = '';
|
|
47
|
+
if (twCreds) {
|
|
48
|
+
const grok = await (0, twitter_1.fetchGrokTranslation)(url, target, twCreds, tlsGetWithProxy).catch(() => null);
|
|
49
|
+
if (grok) {
|
|
50
|
+
translated = grok.text;
|
|
51
|
+
provider = 'Grok';
|
|
52
|
+
if (!parsed.lang && grok.sourceLang)
|
|
53
|
+
parsed.lang = grok.sourceLang;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
if (!translated) {
|
|
57
|
+
const generic = await (0, translate_1.translateText)(rt, parsed.desc, target, parsed.lang);
|
|
58
|
+
if (generic) {
|
|
59
|
+
translated = generic.text;
|
|
60
|
+
provider = generic.provider;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
if (translated) {
|
|
64
|
+
parsed.translation = translated;
|
|
65
|
+
parsed.translationProvider = provider;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
urlCacheLocal.set(cacheKey, { data: parsed, expire: Date.now() + cacheTTL });
|
|
70
|
+
return parsed;
|
|
71
|
+
}
|
|
72
|
+
// 网关选择(上游 issue #12):配置 apiKey → 新网关(无备用 API);否则 → 旧网关(主+备用)。
|
|
73
|
+
// 注意:primaryApiUrl 是旧网关覆盖字段,新网关不受其影响(避免旧配置劫持切换)
|
|
74
|
+
const useNewGateway = !!config.apiKey;
|
|
75
|
+
const primaryApi = useNewGateway
|
|
76
|
+
? gateway_1.NEW_GATEWAY_PRIMARY
|
|
77
|
+
: (config.primaryApiUrl || gateway_1.LEGACY_GATEWAY_PRIMARY);
|
|
78
|
+
const backupApi = config.backupApiUrl || gateway_1.LEGACY_GATEWAY_BACKUP;
|
|
79
|
+
const backupAllowed = !useNewGateway && new Set(['douyin', 'xiaohongshu', 'instagram', 'jimeng']).has(type);
|
|
80
|
+
const gwConf = useNewGateway
|
|
81
|
+
? { apiKey: config.apiKey, authHeaderType: 'X-API-Key', customHeaderName: 'X-API-Key' }
|
|
82
|
+
: { apiKey: '', authHeaderType: undefined, customHeaderName: undefined };
|
|
83
|
+
const apiList = [];
|
|
84
|
+
if (dedicatedFirst && dedicatedUrl) {
|
|
85
|
+
apiList.push({ url: dedicatedUrl, label: `专属API(${type})`, apiKey, authHeaderType, customHeaderName, fieldMapping });
|
|
86
|
+
apiList.push({ url: primaryApi, label: '默认主API', ...gwConf, fieldMapping });
|
|
87
|
+
if (backupAllowed)
|
|
88
|
+
apiList.push({ url: backupApi, label: '备用主API', ...gwConf, fieldMapping });
|
|
89
|
+
}
|
|
90
|
+
else {
|
|
91
|
+
apiList.push({ url: primaryApi, label: '默认主API', ...gwConf, fieldMapping });
|
|
92
|
+
if (backupAllowed)
|
|
93
|
+
apiList.push({ url: backupApi, label: '备用主API', ...gwConf, fieldMapping });
|
|
94
|
+
if (dedicatedUrl)
|
|
95
|
+
apiList.push({ url: dedicatedUrl, label: `专属API(${type})`, apiKey, authHeaderType, customHeaderName, fieldMapping });
|
|
96
|
+
}
|
|
97
|
+
if (type.startsWith('custom_') && apiList.length === 0 && dedicatedUrl) {
|
|
98
|
+
apiList.push({ url: dedicatedUrl, label: `自定义API(${type})`, apiKey, authHeaderType, customHeaderName, fieldMapping });
|
|
99
|
+
}
|
|
100
|
+
// 新网关模式下追加旧网关末位兜底:新网关无专属端点的平台(微博/小红书等)或主 API 故障
|
|
101
|
+
// (5xx/网关侧异常)时仍有机会解析;旧网关对语义问题会返回更明确的错误信息
|
|
102
|
+
if (useNewGateway && !type.startsWith('custom_')) {
|
|
103
|
+
const legacyDedicated = rt.dedicatedApis?.legacy?.[type];
|
|
104
|
+
const legacyMain = config.primaryApiUrl && config.primaryApiUrl !== gateway_1.LEGACY_GATEWAY_PRIMARY ? config.primaryApiUrl : gateway_1.LEGACY_GATEWAY_PRIMARY;
|
|
105
|
+
if (legacyDedicated && !apiList.some((a) => a.url === legacyDedicated)) {
|
|
106
|
+
apiList.push({ url: legacyDedicated, label: `旧网关专属API(${type})`, apiKey: '', authHeaderType: undefined, customHeaderName: undefined, fieldMapping });
|
|
107
|
+
}
|
|
108
|
+
if (!apiList.some((a) => a.url === legacyMain)) {
|
|
109
|
+
apiList.push({ url: legacyMain, label: '旧网关主API', apiKey: '', authHeaderType: undefined, customHeaderName: undefined, fieldMapping });
|
|
110
|
+
}
|
|
111
|
+
if (!apiList.some((a) => a.url === backupApi)) {
|
|
112
|
+
apiList.push({ url: backupApi, label: '旧网关备用API', apiKey: '', authHeaderType: undefined, customHeaderName: undefined, fieldMapping });
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
const customHeaders = config.customHeaders || [];
|
|
116
|
+
const errors = [];
|
|
117
|
+
for (const [apiIndex, api] of apiList.entries()) {
|
|
118
|
+
for (let attempt = 0; attempt <= config.retryTimes; attempt++) {
|
|
119
|
+
try {
|
|
120
|
+
const headers = {
|
|
121
|
+
'User-Agent': config.userAgent,
|
|
122
|
+
'Referer': 'https://www.baidu.com/',
|
|
123
|
+
'Content-Type': 'application/x-www-form-urlencoded'
|
|
124
|
+
};
|
|
125
|
+
for (const h of customHeaders) {
|
|
126
|
+
if (h.name && h.value)
|
|
127
|
+
headers[h.name] = h.value;
|
|
128
|
+
}
|
|
129
|
+
if (api.apiKey) {
|
|
130
|
+
const authHeaders = (0, platform_config_1.buildAuthHeaders)(api.apiKey, api.authHeaderType || 'Bearer', api.customHeaderName || 'X-API-Key');
|
|
131
|
+
Object.assign(headers, authHeaders);
|
|
132
|
+
}
|
|
133
|
+
const proxyToUse = customProxy && customProxy.enabled ? customProxy : (proxyConfig.enabled ? proxyConfig : undefined);
|
|
134
|
+
const axiosConfigLocal = {
|
|
135
|
+
params: { url },
|
|
136
|
+
timeout: config.timeout,
|
|
137
|
+
headers,
|
|
138
|
+
};
|
|
139
|
+
// 代理:runtime 实例已按全局代理配置好 https-proxy-agent;
|
|
140
|
+
// 仅当平台级 customProxy 覆盖时才按请求另建 agent(不用 axios 内置 proxy 选项——对 https 目标不做 CONNECT 隧道)
|
|
141
|
+
if (customProxy && customProxy.enabled && customProxy.host) {
|
|
142
|
+
const pUrl = `${customProxy.protocol || 'http'}://${customProxy.auth?.username ? `${encodeURIComponent(customProxy.auth.username)}:${encodeURIComponent(customProxy.auth.password || '')}@` : ''}${customProxy.host}:${customProxy.port || 7890}`;
|
|
143
|
+
const agent = new https_proxy_agent_1.HttpsProxyAgent(pUrl);
|
|
144
|
+
axiosConfigLocal.httpAgent = agent;
|
|
145
|
+
axiosConfigLocal.httpsAgent = agent;
|
|
146
|
+
axiosConfigLocal.proxy = false;
|
|
147
|
+
}
|
|
148
|
+
const res = await http.get(api.url, axiosConfigLocal);
|
|
149
|
+
// 兼容第三方 API:响应不含 code/status 状态字段时视为成功,直接交由解析器处理
|
|
150
|
+
const code = res.data?.code;
|
|
151
|
+
const ok = res.data && (code === 200 || code === 0 || (code === undefined && res.data.status === undefined));
|
|
152
|
+
if (ok) {
|
|
153
|
+
// 回退链路重要事件:非首个 API 成功说明上游发生了故障切换
|
|
154
|
+
if (apiIndex > 0)
|
|
155
|
+
logger_1.logger.info(`${api.label} 回退解析成功(此前 API 失败): ${url}`);
|
|
156
|
+
else
|
|
157
|
+
(0, logger_1.debugLog)(`${api.label} 解析成功: ${url}`);
|
|
158
|
+
const parsed = (0, parser_1.parseApiResponse)(res.data, config.maxDescLength, api.fieldMapping);
|
|
159
|
+
urlCacheLocal.set(cacheKey, { data: parsed, expire: Date.now() + cacheTTL });
|
|
160
|
+
return parsed;
|
|
161
|
+
}
|
|
162
|
+
throw new Error(res.data?.msg || res.data?.message || `API返回错误码: ${code ?? res.data?.status}`);
|
|
163
|
+
}
|
|
164
|
+
catch (error) {
|
|
165
|
+
const lastErr = error instanceof Error ? error : new Error(String(error));
|
|
166
|
+
errors.push(lastErr);
|
|
167
|
+
(0, logger_1.debugLog)(`${api.label} attempt ${attempt + 1} failed: ${lastErr.message}`);
|
|
168
|
+
if (axios_1.default.isAxiosError(error)) {
|
|
169
|
+
if (!error.response) {
|
|
170
|
+
if (attempt < config.retryTimes) {
|
|
171
|
+
await (0, common_1.delay)(config.retryInterval);
|
|
172
|
+
continue;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
const status = error.response?.status;
|
|
176
|
+
if (status && (status >= 500 || status === 429)) {
|
|
177
|
+
if (attempt < config.retryTimes) {
|
|
178
|
+
await (0, common_1.delay)(config.retryInterval);
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
break;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
logger_1.logger.warn(`${api.label} 全部重试失败`);
|
|
187
|
+
}
|
|
188
|
+
// 优先返回语义级错误(服务端明确说明原因),避免被后续兜底 API 的 5xx/网络/风控泛错覆盖;
|
|
189
|
+
// 「Request failed with status code NNN」这类 axios 泛错无信息量,同样降级
|
|
190
|
+
const semantic = errors.find((e) => !/status code \d{3}|socket|ECONNRESET|ETIMEDOUT|timeout|Bad Gateway|Service Unavailable/i.test(e.message));
|
|
191
|
+
throw semantic || errors[errors.length - 1] || new Error('所有API请求全部失败');
|
|
192
|
+
}
|
|
193
|
+
async function parseUrl(rt, url, type, fieldMapping, platformConf) {
|
|
194
|
+
try {
|
|
195
|
+
const info = await fetchApi(rt, url, type, fieldMapping, platformConf);
|
|
196
|
+
const hasMedia = !!(info.video || info.images.length > 0 || info.live_photo.length > 0);
|
|
197
|
+
if (hasMedia)
|
|
198
|
+
return { success: true, data: info };
|
|
199
|
+
// 纯文字内容(如 X 纯文字推文):有正文即视为成功,下游发送文字卡片
|
|
200
|
+
if (info.type === 'text' && (info.title || info.desc)) {
|
|
201
|
+
(0, logger_1.debugLog)(`纯文字内容: ${url}`);
|
|
202
|
+
return { success: true, data: info };
|
|
203
|
+
}
|
|
204
|
+
(0, logger_1.debugLog)(`解析成功但无内容: ${url}`);
|
|
205
|
+
return { success: false, msg: '解析接口返回空内容' };
|
|
206
|
+
}
|
|
207
|
+
catch (error) {
|
|
208
|
+
logger_1.logger.error(`解析失败: ${url}`, (0, common_1.getErrorMessage)(error));
|
|
209
|
+
let msg = (0, common_1.getErrorMessage)(error);
|
|
210
|
+
// 小红书风控:裸 /discovery/item 链接缺 xsec_token,服务端获取 gate 票据失败;提示用户改发完整分享链接
|
|
211
|
+
if (type === 'xiaohongshu' && /xsec_token|gate|票据/i.test(msg)) {
|
|
212
|
+
msg += '(小红书升级了风控:请发送完整分享链接,如 xhslink.com 短链、App 分享文案里的原链接,或带 xsec_token 参数的网页链接)';
|
|
213
|
+
}
|
|
214
|
+
// 微博:API 仅支持视频微博,图文微博/新版 base62 短链会报「无法从URL中提取视频ID」
|
|
215
|
+
if (type === 'weibo' && /无法从URL中提取视频ID|提取.*视频ID/.test(msg)) {
|
|
216
|
+
msg += '(微博解析仅支持视频微博链接,如 weibo.com/tv/show/… 或 video.weibo.com/show?fid=…;图文微博与新版短链 /用户ID/短ID 形式暂不支持)';
|
|
217
|
+
}
|
|
218
|
+
return { success: false, msg };
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
async function processSingleUrl(rt, url, type, fieldMapping, platformConf) {
|
|
222
|
+
const { config } = rt;
|
|
223
|
+
const result = await parseUrl(rt, url, type, fieldMapping, platformConf);
|
|
224
|
+
if (!result.success)
|
|
225
|
+
return { success: false, msg: result.msg, url };
|
|
226
|
+
const text = (0, format_1.generateFormattedText)(result.data, config.unifiedMessageFormat);
|
|
227
|
+
return { success: true, data: { text, parsed: result.data } };
|
|
228
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 双网关配置(上游 issue #12):
|
|
3
|
+
* - 旧网关 api.bugpk.com:无需 API Key,v1.5.8 及之前的行为
|
|
4
|
+
* - 新网关 api-new.ifphp.com:需 API Key(X-API-Key 头),上游唯一维护入口
|
|
5
|
+
* 选择规则:配置了 apiKey → 新网关;否则 → 旧网关
|
|
6
|
+
*
|
|
7
|
+
* 平台专属端点内联在各 PlatformDefinition(dedicated 字段),此处提供
|
|
8
|
+
* 由定义列表聚合出的映射函数(runtime 构建时计算一次挂在 rt 上)。
|
|
9
|
+
*/
|
|
10
|
+
import type { PlatformDefinition } from '../platform';
|
|
11
|
+
export declare const NEW_GATEWAY_PRIMARY = "https://api-new.ifphp.com/api/svparse";
|
|
12
|
+
export declare const LEGACY_GATEWAY_PRIMARY = "https://api.bugpk.com/api/short_videos";
|
|
13
|
+
export declare const LEGACY_GATEWAY_BACKUP = "https://api.bugpk.com/api/svparse";
|
|
14
|
+
export interface DedicatedApiMaps {
|
|
15
|
+
/** 旧网关平台专属端点(无需 Key) */
|
|
16
|
+
legacy: Record<string, string>;
|
|
17
|
+
/** 新网关平台专属端点(需 Key;未覆盖的平台走主 API 兜底) */
|
|
18
|
+
next: Record<string, string>;
|
|
19
|
+
}
|
|
20
|
+
/** 由平台定义列表聚合专属端点映射 */
|
|
21
|
+
export declare function dedicatedApisFrom(defs: PlatformDefinition[]): DedicatedApiMaps;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.LEGACY_GATEWAY_BACKUP = exports.LEGACY_GATEWAY_PRIMARY = exports.NEW_GATEWAY_PRIMARY = void 0;
|
|
4
|
+
exports.dedicatedApisFrom = dedicatedApisFrom;
|
|
5
|
+
exports.NEW_GATEWAY_PRIMARY = 'https://api-new.ifphp.com/api/svparse';
|
|
6
|
+
exports.LEGACY_GATEWAY_PRIMARY = 'https://api.bugpk.com/api/short_videos';
|
|
7
|
+
exports.LEGACY_GATEWAY_BACKUP = 'https://api.bugpk.com/api/svparse';
|
|
8
|
+
/** 由平台定义列表聚合专属端点映射 */
|
|
9
|
+
function dedicatedApisFrom(defs) {
|
|
10
|
+
const legacy = {};
|
|
11
|
+
const next = {};
|
|
12
|
+
for (const p of defs) {
|
|
13
|
+
if (p.dedicated?.legacy)
|
|
14
|
+
legacy[p.type] = p.dedicated.legacy;
|
|
15
|
+
if (p.dedicated?.next)
|
|
16
|
+
next[p.type] = p.dedicated.next;
|
|
17
|
+
}
|
|
18
|
+
return { legacy, next };
|
|
19
|
+
}
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.parseApiResponse = parseApiResponse;
|
|
4
|
+
const logger_1 = require("../logger");
|
|
5
|
+
const field_mapping_1 = require("../utils/field-mapping");
|
|
6
|
+
const common_1 = require("../utils/common");
|
|
7
|
+
function parseApiResponse(rawInput, maxDescLen, fieldMapping) {
|
|
8
|
+
// 兼容:响应本身为数组时取首个元素(部分第三方 API 行为)
|
|
9
|
+
const raw = Array.isArray(rawInput) ? rawInput[0] : rawInput;
|
|
10
|
+
(0, logger_1.debugLog)('API raw response', raw);
|
|
11
|
+
let data = raw?.data || {};
|
|
12
|
+
if (Array.isArray(data))
|
|
13
|
+
data = data[0] || {};
|
|
14
|
+
const extra = data.extra || {};
|
|
15
|
+
const mapField = (name, fallback) => {
|
|
16
|
+
if (fieldMapping && fieldMapping[name]) {
|
|
17
|
+
const value = (0, field_mapping_1.getNestedValue)(raw, fieldMapping[name]);
|
|
18
|
+
if (value !== undefined)
|
|
19
|
+
return value;
|
|
20
|
+
}
|
|
21
|
+
return fallback();
|
|
22
|
+
};
|
|
23
|
+
let type = mapField('type', () => {
|
|
24
|
+
let t = data.type || data.videoType || '';
|
|
25
|
+
if (!t) {
|
|
26
|
+
if (data.images?.length > 0 && !data.url)
|
|
27
|
+
t = 'image';
|
|
28
|
+
else if (data.live_photo?.length > 0)
|
|
29
|
+
t = 'live_photo';
|
|
30
|
+
else if (raw.msg === 'live' || data.live)
|
|
31
|
+
t = 'live';
|
|
32
|
+
else
|
|
33
|
+
t = 'video';
|
|
34
|
+
}
|
|
35
|
+
return t;
|
|
36
|
+
});
|
|
37
|
+
let authorObj = mapField('author', () => data.author || data.user);
|
|
38
|
+
let author = '', uid = '', avatar = '';
|
|
39
|
+
if (authorObj && typeof authorObj === 'object') {
|
|
40
|
+
author = authorObj.name || authorObj.author || '';
|
|
41
|
+
uid = String(authorObj.id || authorObj.userID || data.uid || data.userID || data.author_id || '');
|
|
42
|
+
avatar = authorObj.avatar || data.avatar || '';
|
|
43
|
+
}
|
|
44
|
+
else {
|
|
45
|
+
author = mapField('author', () => data.author || data.auther || '');
|
|
46
|
+
uid = String(mapField('uid', () => data.uid || data.userID || data.author_id || ''));
|
|
47
|
+
avatar = mapField('avatar', () => data.avatar || '');
|
|
48
|
+
}
|
|
49
|
+
let title = mapField('title', () => data.title || '');
|
|
50
|
+
let desc = mapField('desc', () => data.desc || data.description || '').slice(0, maxDescLen).trim();
|
|
51
|
+
const coverRaw = mapField('cover', () => data.cover || '');
|
|
52
|
+
const cover = coverRaw ? (String(coverRaw).startsWith('http') ? String(coverRaw) : 'https:' + String(coverRaw)) : '';
|
|
53
|
+
let video = '';
|
|
54
|
+
let videos = [];
|
|
55
|
+
const videoBackup = mapField('video_backup', () => data.video_backup);
|
|
56
|
+
if (Array.isArray(videoBackup) && videoBackup.length) {
|
|
57
|
+
const bestQ = (0, common_1.pickBestQuality)(videoBackup);
|
|
58
|
+
videos = bestQ;
|
|
59
|
+
video = bestQ[0]?.url || '';
|
|
60
|
+
}
|
|
61
|
+
if (!video) {
|
|
62
|
+
const rawVideos = mapField('videos', () => data.videos);
|
|
63
|
+
if (Array.isArray(rawVideos) && rawVideos.length) {
|
|
64
|
+
const validVideos = rawVideos.filter((v) => v && v.url);
|
|
65
|
+
if (validVideos.length) {
|
|
66
|
+
video = validVideos[0].url;
|
|
67
|
+
videos = validVideos.map((v) => ({ quality: v.accept?.[0] || 'unknown', url: v.url }));
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
if (!video && data.quality_urls && typeof data.quality_urls === 'object') {
|
|
72
|
+
const entries = Object.entries(data.quality_urls);
|
|
73
|
+
videos = entries.map(([label, url]) => ({ quality: label, url: String(url) }));
|
|
74
|
+
if (videos.length)
|
|
75
|
+
video = videos[0].url;
|
|
76
|
+
}
|
|
77
|
+
if (!video)
|
|
78
|
+
video = mapField('video', () => data.url || '');
|
|
79
|
+
if (video && !video.startsWith('http'))
|
|
80
|
+
video = 'https:' + video;
|
|
81
|
+
let images = [];
|
|
82
|
+
// 统一图片地址规范化:兼容大写协议、// 协议相对地址、{url} 对象与数组形式(上游 v1.6.7)
|
|
83
|
+
const normalizeImageUrl = (img) => {
|
|
84
|
+
if (typeof img === 'string') {
|
|
85
|
+
if (!img)
|
|
86
|
+
return null;
|
|
87
|
+
return /^https?:\/\//i.test(img) ? img : (/^\/\//.test(img) ? 'https:' + img : 'https:' + img);
|
|
88
|
+
}
|
|
89
|
+
if (Array.isArray(img) && img.length)
|
|
90
|
+
return normalizeImageUrl(img[0]);
|
|
91
|
+
if (img && typeof img === 'object' && img.url)
|
|
92
|
+
return normalizeImageUrl(String(img.url));
|
|
93
|
+
return null;
|
|
94
|
+
};
|
|
95
|
+
const directImages = mapField('images', () => data.images);
|
|
96
|
+
if (Array.isArray(directImages)) {
|
|
97
|
+
images = directImages.map(normalizeImageUrl).filter((u) => !!u);
|
|
98
|
+
}
|
|
99
|
+
else if (Array.isArray(data.imgurl)) {
|
|
100
|
+
images = data.imgurl.map(normalizeImageUrl).filter((u) => !!u);
|
|
101
|
+
}
|
|
102
|
+
else if (directImages && typeof directImages === 'object') {
|
|
103
|
+
const single = normalizeImageUrl(directImages);
|
|
104
|
+
if (single)
|
|
105
|
+
images = [single];
|
|
106
|
+
}
|
|
107
|
+
const live_photo = Array.isArray(data.live_photo) ? data.live_photo.filter((lp) => lp && lp.image).map((lp) => ({
|
|
108
|
+
image: lp.image.startsWith('http') ? lp.image : 'https:' + lp.image,
|
|
109
|
+
video: lp.video ? (lp.video.startsWith('http') ? lp.video : 'https:' + lp.video) : ''
|
|
110
|
+
})) : [];
|
|
111
|
+
if (type === 'live' && live_photo.length > 0 && !data.live) {
|
|
112
|
+
type = 'live_photo';
|
|
113
|
+
}
|
|
114
|
+
const musicCoverRaw = mapField('music_cover', () => data.music?.cover || data.music?.albumCover?.url || '');
|
|
115
|
+
const musicUrlRaw = mapField('music_url', () => data.music?.url || data.music?.playURL || '');
|
|
116
|
+
const musicUrlNorm = musicUrlRaw ? (String(musicUrlRaw).startsWith('http') ? String(musicUrlRaw) : 'https:' + String(musicUrlRaw)) : '';
|
|
117
|
+
const music = {
|
|
118
|
+
title: mapField('music_title', () => data.music?.title || data.music?.name || ''),
|
|
119
|
+
author: mapField('music_author', () => data.music?.author || data.music?.artist || ''),
|
|
120
|
+
cover: musicCoverRaw ? (String(musicCoverRaw).startsWith('http') ? String(musicCoverRaw) : 'https:' + String(musicCoverRaw)) : '',
|
|
121
|
+
// 过滤占位垃圾值(如即梦接口返回的 map[]),含空白/括号的地址不视为有效链接(上游 v1.6.7)
|
|
122
|
+
url: /[\s\[\]]/.test(musicUrlNorm) ? '' : musicUrlNorm,
|
|
123
|
+
};
|
|
124
|
+
const like = (0, common_1.parseCount)(mapField('like', () => data.like ?? data.statistics?.digg_count ?? data.statistics?.like_count ?? data.statistics?.likes ?? extra.statistics?.digg_count ?? extra.statistics?.like_count ?? extra.statistics?.likes ?? data.attitudes_count ?? 0));
|
|
125
|
+
const comment = (0, common_1.parseCount)(mapField('comment', () => data.comment ?? data.statistics?.comment_count ?? data.statistics?.comments ?? extra.statistics?.comment_count ?? extra.statistics?.comments ?? data.comments_count ?? 0));
|
|
126
|
+
const collect = (0, common_1.parseCount)(mapField('collect', () => data.collect ?? data.statistics?.collect_count ?? data.statistics?.favorite_count ?? data.statistics?.favorites ?? extra.statistics?.collect_count ?? extra.statistics?.favorite_count ?? extra.statistics?.favorites ?? data.favorites_count ?? 0));
|
|
127
|
+
const share = (0, common_1.parseCount)(mapField('share', () => data.share ?? data.statistics?.share_count ?? data.statistics?.forward_count ?? data.statistics?.shares ?? extra.statistics?.share_count ?? extra.statistics?.forward_count ?? extra.statistics?.shares ?? data.reposts_count ?? 0));
|
|
128
|
+
const play = (0, common_1.parseCount)(mapField('play', () => data.play ?? data.statistics?.play_count ?? data.statistics?.view_count ?? data.statistics?.plays ?? extra.statistics?.play_count ?? extra.statistics?.view_count ?? extra.statistics?.plays ?? data.play_count ?? data.view_count ?? 0));
|
|
129
|
+
let duration = 0;
|
|
130
|
+
if (extra.duration_ms) {
|
|
131
|
+
duration = Math.floor(Number(extra.duration_ms) / 1000);
|
|
132
|
+
}
|
|
133
|
+
else {
|
|
134
|
+
const durRaw = mapField('duration', () => data.duration);
|
|
135
|
+
if (durRaw) {
|
|
136
|
+
duration = typeof durRaw === 'string' ? parseInt(durRaw, 10) : Math.floor(Number(durRaw));
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
let publishTime = 0;
|
|
140
|
+
const timeRaw = mapField('publishTime', () => data.time);
|
|
141
|
+
if (timeRaw) {
|
|
142
|
+
publishTime = typeof timeRaw === 'number' ? timeRaw : parseInt(timeRaw, 10);
|
|
143
|
+
if (publishTime < 1000000000000)
|
|
144
|
+
publishTime *= 1000;
|
|
145
|
+
}
|
|
146
|
+
else if (extra.create_time) {
|
|
147
|
+
publishTime = Number(extra.create_time) * 1000;
|
|
148
|
+
}
|
|
149
|
+
const author_followers = (0, common_1.parseCount)(mapField('author_followers', () => extra.author_extra?.follower_count ?? data.author_extra?.follower_count ?? 0));
|
|
150
|
+
const author_signature = String(mapField('author_signature', () => extra.author_extra?.signature ?? data.author_extra?.signature ?? ''));
|
|
151
|
+
const admire = (0, common_1.parseCount)(mapField('admire', () => extra.statistics?.admire_count ?? data.statistics?.admire_count ?? 0));
|
|
152
|
+
title = title.replace(/\[话题\]/g, '');
|
|
153
|
+
desc = desc.replace(/\[话题\]/g, '');
|
|
154
|
+
if (title && desc && title.trim() === desc.trim()) {
|
|
155
|
+
desc = '';
|
|
156
|
+
}
|
|
157
|
+
if (title.trim().startsWith('#'))
|
|
158
|
+
title = '';
|
|
159
|
+
if (desc.trim().startsWith('#'))
|
|
160
|
+
desc = '';
|
|
161
|
+
return { type, title, desc, author, uid, avatar, cover, video, videos, images, live_photo, music, like, comment, collect, share, play, duration, publishTime, author_followers, author_signature, admire };
|
|
162
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 平台配置装配(自 custom.ts 下沉):
|
|
3
|
+
* - buildCustomLinkRules:customPlatforms 关键词 → 链接规则
|
|
4
|
+
* - buildAuthHeaders:API Key → 鉴权头
|
|
5
|
+
* - getPlatformConfig:平台专属端点/鉴权/字段映射的逐平台配置
|
|
6
|
+
*
|
|
7
|
+
* 专属端点映射来自 rt.dedicatedApis(由 createRuntime 从平台定义列表计算),
|
|
8
|
+
* 因此本模块不再依赖具体平台清单。
|
|
9
|
+
*/
|
|
10
|
+
import type { ParserRuntime } from './runtime';
|
|
11
|
+
export declare function buildCustomLinkRules(customPlatforms: any[]): {
|
|
12
|
+
pattern: RegExp;
|
|
13
|
+
type: string;
|
|
14
|
+
}[];
|
|
15
|
+
export declare function buildAuthHeaders(apiKey: string, authHeaderType: string, customHeaderName: string): Record<string, string>;
|
|
16
|
+
export declare function getPlatformConfig(rt: ParserRuntime, type: string): {
|
|
17
|
+
apiUrl: string | null;
|
|
18
|
+
dedicatedFirst: boolean;
|
|
19
|
+
apiKey: string;
|
|
20
|
+
authHeaderType: string;
|
|
21
|
+
customHeaderName: string;
|
|
22
|
+
fieldMapping?: Record<string, string>;
|
|
23
|
+
customProxy?: any;
|
|
24
|
+
};
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildCustomLinkRules = buildCustomLinkRules;
|
|
4
|
+
exports.buildAuthHeaders = buildAuthHeaders;
|
|
5
|
+
exports.getPlatformConfig = getPlatformConfig;
|
|
6
|
+
const field_mapping_1 = require("../utils/field-mapping");
|
|
7
|
+
function buildCustomLinkRules(customPlatforms) {
|
|
8
|
+
if (!Array.isArray(customPlatforms) || customPlatforms.length === 0)
|
|
9
|
+
return [];
|
|
10
|
+
return customPlatforms
|
|
11
|
+
.filter(p => p.keywords)
|
|
12
|
+
.map(p => {
|
|
13
|
+
const keywords = p.keywords.split(',').map((s) => s.trim()).filter(Boolean);
|
|
14
|
+
if (keywords.length === 0)
|
|
15
|
+
return null;
|
|
16
|
+
const escaped = keywords.map((k) => k.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'));
|
|
17
|
+
const pattern = new RegExp(`https?://[^/\\s"'“”‘’]*(${escaped.join('|')})[^\\s"'“”‘’]*`, 'gi');
|
|
18
|
+
return { pattern, type: `custom_${p.name}` };
|
|
19
|
+
})
|
|
20
|
+
.filter(Boolean);
|
|
21
|
+
}
|
|
22
|
+
function buildAuthHeaders(apiKey, authHeaderType, customHeaderName) {
|
|
23
|
+
if (!apiKey)
|
|
24
|
+
return {};
|
|
25
|
+
if (authHeaderType === 'Bearer')
|
|
26
|
+
return { 'Authorization': `Bearer ${apiKey}` };
|
|
27
|
+
if (authHeaderType === 'X-API-Key')
|
|
28
|
+
return { 'X-API-Key': apiKey };
|
|
29
|
+
if (authHeaderType === 'Custom' && customHeaderName)
|
|
30
|
+
return { [customHeaderName]: apiKey };
|
|
31
|
+
return {};
|
|
32
|
+
}
|
|
33
|
+
function getPlatformConfig(rt, type) {
|
|
34
|
+
const { config, customPlatforms } = rt;
|
|
35
|
+
if (type.startsWith('custom_')) {
|
|
36
|
+
const name = type.slice(7);
|
|
37
|
+
const custom = customPlatforms.find(p => p.name === name);
|
|
38
|
+
if (custom) {
|
|
39
|
+
return {
|
|
40
|
+
apiUrl: custom.apiUrl,
|
|
41
|
+
dedicatedFirst: true,
|
|
42
|
+
apiKey: custom.apiKey || '',
|
|
43
|
+
authHeaderType: custom.authHeaderType,
|
|
44
|
+
customHeaderName: custom.customHeaderName,
|
|
45
|
+
fieldMapping: custom.fieldMapping,
|
|
46
|
+
customProxy: custom.proxy
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
return { apiUrl: null, dedicatedFirst: false, apiKey: '', authHeaderType: 'Bearer', customHeaderName: 'X-API-Key' };
|
|
50
|
+
}
|
|
51
|
+
// 网关选择(上游 issue #12):配置 apiKey → 新网关;否则 → 旧网关
|
|
52
|
+
const useNewGateway = !!config.apiKey;
|
|
53
|
+
const maps = rt.dedicatedApis || {};
|
|
54
|
+
const custom = config.customApis?.find((item) => item.platform === type);
|
|
55
|
+
let apiUrl = (useNewGateway ? maps.next?.[type] : maps.legacy?.[type]) || null;
|
|
56
|
+
let apiKey = '';
|
|
57
|
+
let authHeaderType = 'X-API-Key';
|
|
58
|
+
let customHeaderName = 'X-API-Key';
|
|
59
|
+
let fieldMapping = undefined;
|
|
60
|
+
if (custom && custom.apiUrl) {
|
|
61
|
+
apiUrl = custom.apiUrl;
|
|
62
|
+
apiKey = custom.apiKey || '';
|
|
63
|
+
authHeaderType = custom.authHeaderType || 'Bearer';
|
|
64
|
+
customHeaderName = custom.customHeaderName || 'X-API-Key';
|
|
65
|
+
fieldMapping = (0, field_mapping_1.parseFieldMapping)(custom.fieldMapping);
|
|
66
|
+
}
|
|
67
|
+
else if (useNewGateway) {
|
|
68
|
+
apiKey = config.apiKey;
|
|
69
|
+
authHeaderType = 'X-API-Key';
|
|
70
|
+
}
|
|
71
|
+
const dedicatedFirst = config.platformDedicatedFirst?.[type] ?? false;
|
|
72
|
+
if (!fieldMapping) {
|
|
73
|
+
fieldMapping = (0, field_mapping_1.parseFieldMapping)(config.globalFieldMapping);
|
|
74
|
+
}
|
|
75
|
+
return { apiUrl, dedicatedFirst, apiKey, authHeaderType, customHeaderName, fieldMapping };
|
|
76
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 运行时构建(自宿主 runtime 下沉,平台无关)。
|
|
3
|
+
*
|
|
4
|
+
* 平台定义列表(defs)由调用方传入:
|
|
5
|
+
* - koishi 兼容层传内置 BUILTIN_PLATFORMS(本地 definitions)
|
|
6
|
+
* - CLI 传 collectPlatformDefinitions()(已安装的 @sns-parse/platform-*)
|
|
7
|
+
* 默认扩展实现同样注入:koishi 传含 NSFW 的完整默认实现,CLI 传 createCoreExtensions()。
|
|
8
|
+
*/
|
|
9
|
+
import { AxiosInstance } from 'axios';
|
|
10
|
+
import type { ParsedData, CustomPlatformConfig } from '../types';
|
|
11
|
+
import type { VideoParserHost } from '../host';
|
|
12
|
+
import type { VideoParserExtensions } from '../extensions';
|
|
13
|
+
import type { PlatformDefinition } from '../platform';
|
|
14
|
+
import { SimpleLRUCache } from '../utils/cache';
|
|
15
|
+
import { type DedicatedApiMaps } from './gateway';
|
|
16
|
+
export interface ParserRuntime {
|
|
17
|
+
/** 宿主抽象(core 统一入口) */
|
|
18
|
+
host: VideoParserHost;
|
|
19
|
+
/** 过渡期兼容字段:底层上下文(等同于 host.context) */
|
|
20
|
+
ctx: any;
|
|
21
|
+
config: any;
|
|
22
|
+
http: AxiosInstance;
|
|
23
|
+
proxyConfig: any;
|
|
24
|
+
cacheTTL: number;
|
|
25
|
+
dedupCache: SimpleLRUCache<number>;
|
|
26
|
+
urlCacheLocal: SimpleLRUCache<{
|
|
27
|
+
data: ParsedData;
|
|
28
|
+
expire: number;
|
|
29
|
+
}>;
|
|
30
|
+
contentDedupCache: SimpleLRUCache<number>;
|
|
31
|
+
customPlatforms: CustomPlatformConfig[];
|
|
32
|
+
allRules: {
|
|
33
|
+
pattern: RegExp;
|
|
34
|
+
type: string;
|
|
35
|
+
}[];
|
|
36
|
+
/** 平台专属端点映射(由 defs 计算;getPlatformConfig 使用) */
|
|
37
|
+
dedicatedApis: DedicatedApiMaps;
|
|
38
|
+
/** 扩展能力(默认实现 + 宿主覆盖) */
|
|
39
|
+
extensions: VideoParserExtensions;
|
|
40
|
+
}
|
|
41
|
+
export interface CreateRuntimeOptions {
|
|
42
|
+
/** 平台定义列表(内置或已安装包收集);缺省为空(仅自定义平台规则可用) */
|
|
43
|
+
defs?: PlatformDefinition[];
|
|
44
|
+
/** 默认扩展实现(koishi 层含 NSFW;CLI 可用 createCoreExtensions) */
|
|
45
|
+
defaultExtensions?: VideoParserExtensions;
|
|
46
|
+
}
|
|
47
|
+
export declare function createRuntime(source: any, config: any, opts?: CreateRuntimeOptions): ParserRuntime;
|
|
48
|
+
/** core 自带扩展实现(无 NSFW:CLI / 嵌入式宿主用;koishi 层在其上叠加 NSFW) */
|
|
49
|
+
export declare function createCoreExtensions(): VideoParserExtensions;
|