@sns-parse/core 0.3.0-alpha.1 → 0.4.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/config.d.ts +5 -0
- package/lib/config.js +28 -0
- package/lib/engine/config-io.d.ts +44 -0
- package/lib/engine/config-io.js +155 -0
- package/lib/engine/fetcher.d.ts +21 -0
- package/lib/engine/fetcher.js +228 -0
- package/lib/engine/gateway.d.ts +21 -0
- package/lib/engine/gateway.js +19 -0
- package/lib/engine/parser.d.ts +2 -0
- package/lib/engine/parser.js +162 -0
- package/lib/engine/platform-config.d.ts +24 -0
- package/lib/engine/platform-config.js +76 -0
- package/lib/engine/runtime.d.ts +49 -0
- package/lib/engine/runtime.js +90 -0
- package/lib/engine/translate.d.ts +15 -0
- package/lib/engine/translate.js +107 -0
- package/lib/engine/twitter.d.ts +26 -0
- package/lib/engine/twitter.js +398 -0
- package/lib/index.d.ts +19 -0
- package/lib/index.js +19 -0
- package/lib/registry.d.ts +20 -0
- package/lib/registry.js +77 -0
- package/lib/utils/cache.d.ts +9 -0
- package/lib/utils/cache.js +34 -0
- package/lib/utils/common.d.ts +7 -0
- package/lib/utils/common.js +58 -0
- package/lib/utils/concurrency.d.ts +8 -0
- package/lib/utils/concurrency.js +29 -0
- package/lib/utils/field-mapping.d.ts +2 -0
- package/lib/utils/field-mapping.js +29 -0
- package/lib/utils/format.d.ts +4 -0
- package/lib/utils/format.js +79 -0
- package/lib/utils/gif.d.ts +10 -0
- package/lib/utils/gif.js +104 -0
- package/lib/utils/image-size.d.ts +10 -0
- package/lib/utils/image-size.js +69 -0
- package/lib/utils/merge.d.ts +61 -0
- package/lib/utils/merge.js +388 -0
- package/lib/utils/tls-client.d.ts +30 -0
- package/lib/utils/tls-client.js +120 -0
- package/lib/utils/url.d.ts +10 -0
- package/lib/utils/url.js +89 -0
- package/package.json +16 -4
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.parseApiResponse = parseApiResponse;
|
|
4
|
+
const logger_1 = require("../logger");
|
|
5
|
+
const field_mapping_1 = require("../utils/field-mapping");
|
|
6
|
+
const common_1 = require("../utils/common");
|
|
7
|
+
function parseApiResponse(rawInput, maxDescLen, fieldMapping) {
|
|
8
|
+
// 兼容:响应本身为数组时取首个元素(部分第三方 API 行为)
|
|
9
|
+
const raw = Array.isArray(rawInput) ? rawInput[0] : rawInput;
|
|
10
|
+
(0, logger_1.debugLog)('API raw response', raw);
|
|
11
|
+
let data = raw?.data || {};
|
|
12
|
+
if (Array.isArray(data))
|
|
13
|
+
data = data[0] || {};
|
|
14
|
+
const extra = data.extra || {};
|
|
15
|
+
const mapField = (name, fallback) => {
|
|
16
|
+
if (fieldMapping && fieldMapping[name]) {
|
|
17
|
+
const value = (0, field_mapping_1.getNestedValue)(raw, fieldMapping[name]);
|
|
18
|
+
if (value !== undefined)
|
|
19
|
+
return value;
|
|
20
|
+
}
|
|
21
|
+
return fallback();
|
|
22
|
+
};
|
|
23
|
+
let type = mapField('type', () => {
|
|
24
|
+
let t = data.type || data.videoType || '';
|
|
25
|
+
if (!t) {
|
|
26
|
+
if (data.images?.length > 0 && !data.url)
|
|
27
|
+
t = 'image';
|
|
28
|
+
else if (data.live_photo?.length > 0)
|
|
29
|
+
t = 'live_photo';
|
|
30
|
+
else if (raw.msg === 'live' || data.live)
|
|
31
|
+
t = 'live';
|
|
32
|
+
else
|
|
33
|
+
t = 'video';
|
|
34
|
+
}
|
|
35
|
+
return t;
|
|
36
|
+
});
|
|
37
|
+
let authorObj = mapField('author', () => data.author || data.user);
|
|
38
|
+
let author = '', uid = '', avatar = '';
|
|
39
|
+
if (authorObj && typeof authorObj === 'object') {
|
|
40
|
+
author = authorObj.name || authorObj.author || '';
|
|
41
|
+
uid = String(authorObj.id || authorObj.userID || data.uid || data.userID || data.author_id || '');
|
|
42
|
+
avatar = authorObj.avatar || data.avatar || '';
|
|
43
|
+
}
|
|
44
|
+
else {
|
|
45
|
+
author = mapField('author', () => data.author || data.auther || '');
|
|
46
|
+
uid = String(mapField('uid', () => data.uid || data.userID || data.author_id || ''));
|
|
47
|
+
avatar = mapField('avatar', () => data.avatar || '');
|
|
48
|
+
}
|
|
49
|
+
let title = mapField('title', () => data.title || '');
|
|
50
|
+
let desc = mapField('desc', () => data.desc || data.description || '').slice(0, maxDescLen).trim();
|
|
51
|
+
const coverRaw = mapField('cover', () => data.cover || '');
|
|
52
|
+
const cover = coverRaw ? (String(coverRaw).startsWith('http') ? String(coverRaw) : 'https:' + String(coverRaw)) : '';
|
|
53
|
+
let video = '';
|
|
54
|
+
let videos = [];
|
|
55
|
+
const videoBackup = mapField('video_backup', () => data.video_backup);
|
|
56
|
+
if (Array.isArray(videoBackup) && videoBackup.length) {
|
|
57
|
+
const bestQ = (0, common_1.pickBestQuality)(videoBackup);
|
|
58
|
+
videos = bestQ;
|
|
59
|
+
video = bestQ[0]?.url || '';
|
|
60
|
+
}
|
|
61
|
+
if (!video) {
|
|
62
|
+
const rawVideos = mapField('videos', () => data.videos);
|
|
63
|
+
if (Array.isArray(rawVideos) && rawVideos.length) {
|
|
64
|
+
const validVideos = rawVideos.filter((v) => v && v.url);
|
|
65
|
+
if (validVideos.length) {
|
|
66
|
+
video = validVideos[0].url;
|
|
67
|
+
videos = validVideos.map((v) => ({ quality: v.accept?.[0] || 'unknown', url: v.url }));
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
if (!video && data.quality_urls && typeof data.quality_urls === 'object') {
|
|
72
|
+
const entries = Object.entries(data.quality_urls);
|
|
73
|
+
videos = entries.map(([label, url]) => ({ quality: label, url: String(url) }));
|
|
74
|
+
if (videos.length)
|
|
75
|
+
video = videos[0].url;
|
|
76
|
+
}
|
|
77
|
+
if (!video)
|
|
78
|
+
video = mapField('video', () => data.url || '');
|
|
79
|
+
if (video && !video.startsWith('http'))
|
|
80
|
+
video = 'https:' + video;
|
|
81
|
+
let images = [];
|
|
82
|
+
// 统一图片地址规范化:兼容大写协议、// 协议相对地址、{url} 对象与数组形式(上游 v1.6.7)
|
|
83
|
+
const normalizeImageUrl = (img) => {
|
|
84
|
+
if (typeof img === 'string') {
|
|
85
|
+
if (!img)
|
|
86
|
+
return null;
|
|
87
|
+
return /^https?:\/\//i.test(img) ? img : (/^\/\//.test(img) ? 'https:' + img : 'https:' + img);
|
|
88
|
+
}
|
|
89
|
+
if (Array.isArray(img) && img.length)
|
|
90
|
+
return normalizeImageUrl(img[0]);
|
|
91
|
+
if (img && typeof img === 'object' && img.url)
|
|
92
|
+
return normalizeImageUrl(String(img.url));
|
|
93
|
+
return null;
|
|
94
|
+
};
|
|
95
|
+
const directImages = mapField('images', () => data.images);
|
|
96
|
+
if (Array.isArray(directImages)) {
|
|
97
|
+
images = directImages.map(normalizeImageUrl).filter((u) => !!u);
|
|
98
|
+
}
|
|
99
|
+
else if (Array.isArray(data.imgurl)) {
|
|
100
|
+
images = data.imgurl.map(normalizeImageUrl).filter((u) => !!u);
|
|
101
|
+
}
|
|
102
|
+
else if (directImages && typeof directImages === 'object') {
|
|
103
|
+
const single = normalizeImageUrl(directImages);
|
|
104
|
+
if (single)
|
|
105
|
+
images = [single];
|
|
106
|
+
}
|
|
107
|
+
const live_photo = Array.isArray(data.live_photo) ? data.live_photo.filter((lp) => lp && lp.image).map((lp) => ({
|
|
108
|
+
image: lp.image.startsWith('http') ? lp.image : 'https:' + lp.image,
|
|
109
|
+
video: lp.video ? (lp.video.startsWith('http') ? lp.video : 'https:' + lp.video) : ''
|
|
110
|
+
})) : [];
|
|
111
|
+
if (type === 'live' && live_photo.length > 0 && !data.live) {
|
|
112
|
+
type = 'live_photo';
|
|
113
|
+
}
|
|
114
|
+
const musicCoverRaw = mapField('music_cover', () => data.music?.cover || data.music?.albumCover?.url || '');
|
|
115
|
+
const musicUrlRaw = mapField('music_url', () => data.music?.url || data.music?.playURL || '');
|
|
116
|
+
const musicUrlNorm = musicUrlRaw ? (String(musicUrlRaw).startsWith('http') ? String(musicUrlRaw) : 'https:' + String(musicUrlRaw)) : '';
|
|
117
|
+
const music = {
|
|
118
|
+
title: mapField('music_title', () => data.music?.title || data.music?.name || ''),
|
|
119
|
+
author: mapField('music_author', () => data.music?.author || data.music?.artist || ''),
|
|
120
|
+
cover: musicCoverRaw ? (String(musicCoverRaw).startsWith('http') ? String(musicCoverRaw) : 'https:' + String(musicCoverRaw)) : '',
|
|
121
|
+
// 过滤占位垃圾值(如即梦接口返回的 map[]),含空白/括号的地址不视为有效链接(上游 v1.6.7)
|
|
122
|
+
url: /[\s\[\]]/.test(musicUrlNorm) ? '' : musicUrlNorm,
|
|
123
|
+
};
|
|
124
|
+
const like = (0, common_1.parseCount)(mapField('like', () => data.like ?? data.statistics?.digg_count ?? data.statistics?.like_count ?? data.statistics?.likes ?? extra.statistics?.digg_count ?? extra.statistics?.like_count ?? extra.statistics?.likes ?? data.attitudes_count ?? 0));
|
|
125
|
+
const comment = (0, common_1.parseCount)(mapField('comment', () => data.comment ?? data.statistics?.comment_count ?? data.statistics?.comments ?? extra.statistics?.comment_count ?? extra.statistics?.comments ?? data.comments_count ?? 0));
|
|
126
|
+
const collect = (0, common_1.parseCount)(mapField('collect', () => data.collect ?? data.statistics?.collect_count ?? data.statistics?.favorite_count ?? data.statistics?.favorites ?? extra.statistics?.collect_count ?? extra.statistics?.favorite_count ?? extra.statistics?.favorites ?? data.favorites_count ?? 0));
|
|
127
|
+
const share = (0, common_1.parseCount)(mapField('share', () => data.share ?? data.statistics?.share_count ?? data.statistics?.forward_count ?? data.statistics?.shares ?? extra.statistics?.share_count ?? extra.statistics?.forward_count ?? extra.statistics?.shares ?? data.reposts_count ?? 0));
|
|
128
|
+
const play = (0, common_1.parseCount)(mapField('play', () => data.play ?? data.statistics?.play_count ?? data.statistics?.view_count ?? data.statistics?.plays ?? extra.statistics?.play_count ?? extra.statistics?.view_count ?? extra.statistics?.plays ?? data.play_count ?? data.view_count ?? 0));
|
|
129
|
+
let duration = 0;
|
|
130
|
+
if (extra.duration_ms) {
|
|
131
|
+
duration = Math.floor(Number(extra.duration_ms) / 1000);
|
|
132
|
+
}
|
|
133
|
+
else {
|
|
134
|
+
const durRaw = mapField('duration', () => data.duration);
|
|
135
|
+
if (durRaw) {
|
|
136
|
+
duration = typeof durRaw === 'string' ? parseInt(durRaw, 10) : Math.floor(Number(durRaw));
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
let publishTime = 0;
|
|
140
|
+
const timeRaw = mapField('publishTime', () => data.time);
|
|
141
|
+
if (timeRaw) {
|
|
142
|
+
publishTime = typeof timeRaw === 'number' ? timeRaw : parseInt(timeRaw, 10);
|
|
143
|
+
if (publishTime < 1000000000000)
|
|
144
|
+
publishTime *= 1000;
|
|
145
|
+
}
|
|
146
|
+
else if (extra.create_time) {
|
|
147
|
+
publishTime = Number(extra.create_time) * 1000;
|
|
148
|
+
}
|
|
149
|
+
const author_followers = (0, common_1.parseCount)(mapField('author_followers', () => extra.author_extra?.follower_count ?? data.author_extra?.follower_count ?? 0));
|
|
150
|
+
const author_signature = String(mapField('author_signature', () => extra.author_extra?.signature ?? data.author_extra?.signature ?? ''));
|
|
151
|
+
const admire = (0, common_1.parseCount)(mapField('admire', () => extra.statistics?.admire_count ?? data.statistics?.admire_count ?? 0));
|
|
152
|
+
title = title.replace(/\[话题\]/g, '');
|
|
153
|
+
desc = desc.replace(/\[话题\]/g, '');
|
|
154
|
+
if (title && desc && title.trim() === desc.trim()) {
|
|
155
|
+
desc = '';
|
|
156
|
+
}
|
|
157
|
+
if (title.trim().startsWith('#'))
|
|
158
|
+
title = '';
|
|
159
|
+
if (desc.trim().startsWith('#'))
|
|
160
|
+
desc = '';
|
|
161
|
+
return { type, title, desc, author, uid, avatar, cover, video, videos, images, live_photo, music, like, comment, collect, share, play, duration, publishTime, author_followers, author_signature, admire };
|
|
162
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 平台配置装配(自 custom.ts 下沉):
|
|
3
|
+
* - buildCustomLinkRules:customPlatforms 关键词 → 链接规则
|
|
4
|
+
* - buildAuthHeaders:API Key → 鉴权头
|
|
5
|
+
* - getPlatformConfig:平台专属端点/鉴权/字段映射的逐平台配置
|
|
6
|
+
*
|
|
7
|
+
* 专属端点映射来自 rt.dedicatedApis(由 createRuntime 从平台定义列表计算),
|
|
8
|
+
* 因此本模块不再依赖具体平台清单。
|
|
9
|
+
*/
|
|
10
|
+
import type { ParserRuntime } from './runtime';
|
|
11
|
+
export declare function buildCustomLinkRules(customPlatforms: any[]): {
|
|
12
|
+
pattern: RegExp;
|
|
13
|
+
type: string;
|
|
14
|
+
}[];
|
|
15
|
+
export declare function buildAuthHeaders(apiKey: string, authHeaderType: string, customHeaderName: string): Record<string, string>;
|
|
16
|
+
export declare function getPlatformConfig(rt: ParserRuntime, type: string): {
|
|
17
|
+
apiUrl: string | null;
|
|
18
|
+
dedicatedFirst: boolean;
|
|
19
|
+
apiKey: string;
|
|
20
|
+
authHeaderType: string;
|
|
21
|
+
customHeaderName: string;
|
|
22
|
+
fieldMapping?: Record<string, string>;
|
|
23
|
+
customProxy?: any;
|
|
24
|
+
};
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildCustomLinkRules = buildCustomLinkRules;
|
|
4
|
+
exports.buildAuthHeaders = buildAuthHeaders;
|
|
5
|
+
exports.getPlatformConfig = getPlatformConfig;
|
|
6
|
+
const field_mapping_1 = require("../utils/field-mapping");
|
|
7
|
+
function buildCustomLinkRules(customPlatforms) {
|
|
8
|
+
if (!Array.isArray(customPlatforms) || customPlatforms.length === 0)
|
|
9
|
+
return [];
|
|
10
|
+
return customPlatforms
|
|
11
|
+
.filter(p => p.keywords)
|
|
12
|
+
.map(p => {
|
|
13
|
+
const keywords = p.keywords.split(',').map((s) => s.trim()).filter(Boolean);
|
|
14
|
+
if (keywords.length === 0)
|
|
15
|
+
return null;
|
|
16
|
+
const escaped = keywords.map((k) => k.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'));
|
|
17
|
+
const pattern = new RegExp(`https?://[^/\\s"'“”‘’]*(${escaped.join('|')})[^\\s"'“”‘’]*`, 'gi');
|
|
18
|
+
return { pattern, type: `custom_${p.name}` };
|
|
19
|
+
})
|
|
20
|
+
.filter(Boolean);
|
|
21
|
+
}
|
|
22
|
+
function buildAuthHeaders(apiKey, authHeaderType, customHeaderName) {
|
|
23
|
+
if (!apiKey)
|
|
24
|
+
return {};
|
|
25
|
+
if (authHeaderType === 'Bearer')
|
|
26
|
+
return { 'Authorization': `Bearer ${apiKey}` };
|
|
27
|
+
if (authHeaderType === 'X-API-Key')
|
|
28
|
+
return { 'X-API-Key': apiKey };
|
|
29
|
+
if (authHeaderType === 'Custom' && customHeaderName)
|
|
30
|
+
return { [customHeaderName]: apiKey };
|
|
31
|
+
return {};
|
|
32
|
+
}
|
|
33
|
+
function getPlatformConfig(rt, type) {
|
|
34
|
+
const { config, customPlatforms } = rt;
|
|
35
|
+
if (type.startsWith('custom_')) {
|
|
36
|
+
const name = type.slice(7);
|
|
37
|
+
const custom = customPlatforms.find(p => p.name === name);
|
|
38
|
+
if (custom) {
|
|
39
|
+
return {
|
|
40
|
+
apiUrl: custom.apiUrl,
|
|
41
|
+
dedicatedFirst: true,
|
|
42
|
+
apiKey: custom.apiKey || '',
|
|
43
|
+
authHeaderType: custom.authHeaderType,
|
|
44
|
+
customHeaderName: custom.customHeaderName,
|
|
45
|
+
fieldMapping: custom.fieldMapping,
|
|
46
|
+
customProxy: custom.proxy
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
return { apiUrl: null, dedicatedFirst: false, apiKey: '', authHeaderType: 'Bearer', customHeaderName: 'X-API-Key' };
|
|
50
|
+
}
|
|
51
|
+
// 网关选择(上游 issue #12):配置 apiKey → 新网关;否则 → 旧网关
|
|
52
|
+
const useNewGateway = !!config.apiKey;
|
|
53
|
+
const maps = rt.dedicatedApis || {};
|
|
54
|
+
const custom = config.customApis?.find((item) => item.platform === type);
|
|
55
|
+
let apiUrl = (useNewGateway ? maps.next?.[type] : maps.legacy?.[type]) || null;
|
|
56
|
+
let apiKey = '';
|
|
57
|
+
let authHeaderType = 'X-API-Key';
|
|
58
|
+
let customHeaderName = 'X-API-Key';
|
|
59
|
+
let fieldMapping = undefined;
|
|
60
|
+
if (custom && custom.apiUrl) {
|
|
61
|
+
apiUrl = custom.apiUrl;
|
|
62
|
+
apiKey = custom.apiKey || '';
|
|
63
|
+
authHeaderType = custom.authHeaderType || 'Bearer';
|
|
64
|
+
customHeaderName = custom.customHeaderName || 'X-API-Key';
|
|
65
|
+
fieldMapping = (0, field_mapping_1.parseFieldMapping)(custom.fieldMapping);
|
|
66
|
+
}
|
|
67
|
+
else if (useNewGateway) {
|
|
68
|
+
apiKey = config.apiKey;
|
|
69
|
+
authHeaderType = 'X-API-Key';
|
|
70
|
+
}
|
|
71
|
+
const dedicatedFirst = config.platformDedicatedFirst?.[type] ?? false;
|
|
72
|
+
if (!fieldMapping) {
|
|
73
|
+
fieldMapping = (0, field_mapping_1.parseFieldMapping)(config.globalFieldMapping);
|
|
74
|
+
}
|
|
75
|
+
return { apiUrl, dedicatedFirst, apiKey, authHeaderType, customHeaderName, fieldMapping };
|
|
76
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 运行时构建(自宿主 runtime 下沉,平台无关)。
|
|
3
|
+
*
|
|
4
|
+
* 平台定义列表(defs)由调用方传入:
|
|
5
|
+
* - koishi 兼容层传内置 BUILTIN_PLATFORMS(本地 definitions)
|
|
6
|
+
* - CLI 传 collectPlatformDefinitions()(已安装的 @sns-parse/platform-*)
|
|
7
|
+
* 默认扩展实现同样注入:koishi 传含 NSFW 的完整默认实现,CLI 传 createCoreExtensions()。
|
|
8
|
+
*/
|
|
9
|
+
import { AxiosInstance } from 'axios';
|
|
10
|
+
import type { ParsedData, CustomPlatformConfig } from '../types';
|
|
11
|
+
import type { VideoParserHost } from '../host';
|
|
12
|
+
import type { VideoParserExtensions } from '../extensions';
|
|
13
|
+
import type { PlatformDefinition } from '../platform';
|
|
14
|
+
import { SimpleLRUCache } from '../utils/cache';
|
|
15
|
+
import { type DedicatedApiMaps } from './gateway';
|
|
16
|
+
export interface ParserRuntime {
|
|
17
|
+
/** 宿主抽象(core 统一入口) */
|
|
18
|
+
host: VideoParserHost;
|
|
19
|
+
/** 过渡期兼容字段:底层上下文(等同于 host.context) */
|
|
20
|
+
ctx: any;
|
|
21
|
+
config: any;
|
|
22
|
+
http: AxiosInstance;
|
|
23
|
+
proxyConfig: any;
|
|
24
|
+
cacheTTL: number;
|
|
25
|
+
dedupCache: SimpleLRUCache<number>;
|
|
26
|
+
urlCacheLocal: SimpleLRUCache<{
|
|
27
|
+
data: ParsedData;
|
|
28
|
+
expire: number;
|
|
29
|
+
}>;
|
|
30
|
+
contentDedupCache: SimpleLRUCache<number>;
|
|
31
|
+
customPlatforms: CustomPlatformConfig[];
|
|
32
|
+
allRules: {
|
|
33
|
+
pattern: RegExp;
|
|
34
|
+
type: string;
|
|
35
|
+
}[];
|
|
36
|
+
/** 平台专属端点映射(由 defs 计算;getPlatformConfig 使用) */
|
|
37
|
+
dedicatedApis: DedicatedApiMaps;
|
|
38
|
+
/** 扩展能力(默认实现 + 宿主覆盖) */
|
|
39
|
+
extensions: VideoParserExtensions;
|
|
40
|
+
}
|
|
41
|
+
export interface CreateRuntimeOptions {
|
|
42
|
+
/** 平台定义列表(内置或已安装包收集);缺省为空(仅自定义平台规则可用) */
|
|
43
|
+
defs?: PlatformDefinition[];
|
|
44
|
+
/** 默认扩展实现(koishi 层含 NSFW;CLI 可用 createCoreExtensions) */
|
|
45
|
+
defaultExtensions?: VideoParserExtensions;
|
|
46
|
+
}
|
|
47
|
+
export declare function createRuntime(source: any, config: any, opts?: CreateRuntimeOptions): ParserRuntime;
|
|
48
|
+
/** core 自带扩展实现(无 NSFW:CLI / 嵌入式宿主用;koishi 层在其上叠加 NSFW) */
|
|
49
|
+
export declare function createCoreExtensions(): VideoParserExtensions;
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.createRuntime = createRuntime;
|
|
7
|
+
exports.createCoreExtensions = createCoreExtensions;
|
|
8
|
+
/**
|
|
9
|
+
* 运行时构建(自宿主 runtime 下沉,平台无关)。
|
|
10
|
+
*
|
|
11
|
+
* 平台定义列表(defs)由调用方传入:
|
|
12
|
+
* - koishi 兼容层传内置 BUILTIN_PLATFORMS(本地 definitions)
|
|
13
|
+
* - CLI 传 collectPlatformDefinitions()(已安装的 @sns-parse/platform-*)
|
|
14
|
+
* 默认扩展实现同样注入:koishi 传含 NSFW 的完整默认实现,CLI 传 createCoreExtensions()。
|
|
15
|
+
*/
|
|
16
|
+
const axios_1 = __importDefault(require("axios"));
|
|
17
|
+
const https_proxy_agent_1 = require("https-proxy-agent");
|
|
18
|
+
const host_1 = require("../host");
|
|
19
|
+
const cache_1 = require("../utils/cache");
|
|
20
|
+
const field_mapping_1 = require("../utils/field-mapping");
|
|
21
|
+
const platform_config_1 = require("./platform-config");
|
|
22
|
+
const gateway_1 = require("./gateway");
|
|
23
|
+
const merge_1 = require("../utils/merge");
|
|
24
|
+
const gif_1 = require("../utils/gif");
|
|
25
|
+
const translate_1 = require("./translate");
|
|
26
|
+
function createRuntime(source, config, opts = {}) {
|
|
27
|
+
const host = (0, host_1.createHost)(source);
|
|
28
|
+
const ctx = host.context;
|
|
29
|
+
const extensions = { ...(opts.defaultExtensions || {}), ...(host.extensions || {}) };
|
|
30
|
+
const dedupCache = new cache_1.SimpleLRUCache(1000, config.deduplicationInterval * 1000);
|
|
31
|
+
const cacheTTL = (config.cacheTTL || 600) * 1000;
|
|
32
|
+
const urlCacheLocal = new cache_1.SimpleLRUCache(500, cacheTTL);
|
|
33
|
+
const contentDedupCache = new cache_1.SimpleLRUCache(1000, config.deduplicationInterval * 1000);
|
|
34
|
+
const proxyConfig = config.proxy || {};
|
|
35
|
+
const customPlatforms = (config.customPlatforms || []).map((p) => ({
|
|
36
|
+
name: p.name,
|
|
37
|
+
apiUrl: p.apiUrl,
|
|
38
|
+
apiKey: p.apiKey || '',
|
|
39
|
+
authHeaderType: p.authHeaderType || 'Bearer',
|
|
40
|
+
customHeaderName: p.customHeaderName || 'X-API-Key',
|
|
41
|
+
fieldMapping: (0, field_mapping_1.parseFieldMapping)(p.fieldMapping),
|
|
42
|
+
proxy: p.proxy || null
|
|
43
|
+
}));
|
|
44
|
+
const defs = opts.defs || [];
|
|
45
|
+
const builtinRules = defs.flatMap(d => d.rules.map(pattern => ({ pattern, type: d.type })));
|
|
46
|
+
const customRules = (0, platform_config_1.buildCustomLinkRules)(config.customPlatforms || []);
|
|
47
|
+
const allRules = [...builtinRules, ...customRules];
|
|
48
|
+
const dedicatedApis = (0, gateway_1.dedicatedApisFrom)(defs);
|
|
49
|
+
const axiosConfig = {
|
|
50
|
+
timeout: config.timeout,
|
|
51
|
+
headers: {
|
|
52
|
+
'User-Agent': config.userAgent,
|
|
53
|
+
'Referer': 'https://www.baidu.com/',
|
|
54
|
+
'Content-Type': 'application/x-www-form-urlencoded'
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
if (proxyConfig.enabled && proxyConfig.host) {
|
|
58
|
+
// axios 内置 proxy 选项对 https 目标存在不做 CONNECT 隧道的经典缺陷,
|
|
59
|
+
// 改用 https-proxy-agent 显式代理(按请求生效,不污染全局 env)
|
|
60
|
+
const proxyUrl = `${proxyConfig.protocol || 'http'}://${proxyConfig.auth?.username ? `${encodeURIComponent(proxyConfig.auth.username)}:${encodeURIComponent(proxyConfig.auth.password || '')}@` : ''}${proxyConfig.host}:${proxyConfig.port || 7890}`;
|
|
61
|
+
const agent = new https_proxy_agent_1.HttpsProxyAgent(proxyUrl);
|
|
62
|
+
axiosConfig.httpAgent = agent;
|
|
63
|
+
axiosConfig.httpsAgent = agent;
|
|
64
|
+
axiosConfig.proxy = false;
|
|
65
|
+
}
|
|
66
|
+
const http = axios_1.default.create(axiosConfig);
|
|
67
|
+
return {
|
|
68
|
+
host,
|
|
69
|
+
ctx,
|
|
70
|
+
config,
|
|
71
|
+
http,
|
|
72
|
+
proxyConfig,
|
|
73
|
+
cacheTTL,
|
|
74
|
+
dedupCache,
|
|
75
|
+
urlCacheLocal,
|
|
76
|
+
contentDedupCache,
|
|
77
|
+
customPlatforms,
|
|
78
|
+
allRules,
|
|
79
|
+
dedicatedApis,
|
|
80
|
+
extensions,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
/** core 自带扩展实现(无 NSFW:CLI / 嵌入式宿主用;koishi 层在其上叠加 NSFW) */
|
|
84
|
+
function createCoreExtensions() {
|
|
85
|
+
return {
|
|
86
|
+
mergeImages: merge_1.mergeImages,
|
|
87
|
+
mp4ToGif: gif_1.mp4ToGif,
|
|
88
|
+
translate: translate_1.translateText,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 推文翻译:Google gtx 免费端点(无 Key、GET-only)。
|
|
3
|
+
*
|
|
4
|
+
* X 网页端的 Grok 翻译走私有通道(translations/show.json 已 404,GraphQL
|
|
5
|
+
* 仅返回 is_translatable 标志),无公开端点可调,故用通用翻译源兜底。
|
|
6
|
+
* 译文存入 ParsedData.translation,经 ${翻译} 模板变量展示,空值行自动隐藏。
|
|
7
|
+
*/
|
|
8
|
+
import type { ParserRuntime } from './runtime';
|
|
9
|
+
import type { TranslateResult } from '../extensions';
|
|
10
|
+
export type { TranslateResult };
|
|
11
|
+
/** 判断是否需要跳过翻译(目标语系与推文语种相同 / 无语言内容) */
|
|
12
|
+
export declare function shouldSkipTranslate(tweetLang: string | undefined, target: string): boolean;
|
|
13
|
+
export declare function langName(code?: string): string;
|
|
14
|
+
/** 翻译文本;失败返回 null(调用方保持无译文继续发送)。gtx 优先,MyMemory 兜底 */
|
|
15
|
+
export declare function translateText(rt: ParserRuntime, text: string, target: string, sourceLang?: string): Promise<TranslateResult | null>;
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.shouldSkipTranslate = shouldSkipTranslate;
|
|
4
|
+
exports.langName = langName;
|
|
5
|
+
exports.translateText = translateText;
|
|
6
|
+
const logger_1 = require("../logger");
|
|
7
|
+
/** 判断是否需要跳过翻译(目标语系与推文语种相同 / 无语言内容) */
|
|
8
|
+
function shouldSkipTranslate(tweetLang, target) {
|
|
9
|
+
if (!tweetLang)
|
|
10
|
+
return false;
|
|
11
|
+
const l = tweetLang.toLowerCase();
|
|
12
|
+
if (l === 'zxx' || l === 'und')
|
|
13
|
+
return l === 'zxx'; // zxx=无语言内容;und=未定则仍尝试
|
|
14
|
+
const t = target.toLowerCase();
|
|
15
|
+
if (t === 'zh')
|
|
16
|
+
return l === 'zh' || l.startsWith('zh-');
|
|
17
|
+
if (t === 'zh-tw')
|
|
18
|
+
return l === 'zh-tw' || l === 'zh-hant' || (l.startsWith('zh') && l !== 'zh-cn');
|
|
19
|
+
if (t === 'en')
|
|
20
|
+
return l === 'en';
|
|
21
|
+
return l === t;
|
|
22
|
+
}
|
|
23
|
+
/** 常见语种代码 → 中文名(未知代码原样展示) */
|
|
24
|
+
const LANG_NAMES = {
|
|
25
|
+
fr: '法语', ja: '日语', ko: '韩语', en: '英语', zh: '中文', es: '西班牙语', pt: '葡萄牙语',
|
|
26
|
+
de: '德语', ru: '俄语', ar: '阿拉伯语', it: '意大利语', th: '泰语', vi: '越南语',
|
|
27
|
+
id: '印尼语', tr: '土耳其语', nl: '荷兰语', pl: '波兰语', hi: '印地语', uk: '乌克兰语',
|
|
28
|
+
sv: '瑞典语', no: '挪威语', fi: '芬兰语', da: '丹麦语', cs: '捷克语', ro: '罗马尼亚语',
|
|
29
|
+
hu: '匈牙利语', el: '希腊语', he: '希伯来语', fa: '波斯语', ur: '乌尔都语', ms: '马来语',
|
|
30
|
+
tl: '菲律宾语', ca: '加泰罗尼亚语', bg: '保加利亚语', hr: '克罗地亚语', sr: '塞尔维亚语',
|
|
31
|
+
};
|
|
32
|
+
function langName(code) {
|
|
33
|
+
if (!code)
|
|
34
|
+
return '';
|
|
35
|
+
const base = code.toLowerCase().split('-')[0];
|
|
36
|
+
if (base === 'und')
|
|
37
|
+
return '';
|
|
38
|
+
return LANG_NAMES[base] || code;
|
|
39
|
+
}
|
|
40
|
+
/** 翻译文本;失败返回 null(调用方保持无译文继续发送)。gtx 优先,MyMemory 兜底 */
|
|
41
|
+
async function translateText(rt, text, target, sourceLang) {
|
|
42
|
+
if (!text)
|
|
43
|
+
return null;
|
|
44
|
+
const out = await viaGtx(rt, text, target);
|
|
45
|
+
if (out)
|
|
46
|
+
return { text: out, provider: 'Google' };
|
|
47
|
+
const backup = await viaMyMemory(rt, text, target, sourceLang);
|
|
48
|
+
if (backup)
|
|
49
|
+
return { text: backup, provider: 'MyMemory' };
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
/** Google gtx 免费端点(无 Key;sl=auto 自动检测源语种) */
|
|
53
|
+
async function viaGtx(rt, text, target) {
|
|
54
|
+
const url = 'https://translate.googleapis.com/translate_a/single' +
|
|
55
|
+
`?client=gtx&sl=auto&dt=t&tl=${encodeURIComponent(target)}` +
|
|
56
|
+
`&q=${encodeURIComponent(text.slice(0, 2000))}`;
|
|
57
|
+
try {
|
|
58
|
+
const res = await rt.http.get(url, {
|
|
59
|
+
timeout: 15000,
|
|
60
|
+
headers: { 'User-Agent': rt.config.userAgent },
|
|
61
|
+
});
|
|
62
|
+
const data = res.data;
|
|
63
|
+
// gtx 响应形如 [[["译文","原文",null,null,10]],null,"fr",...]
|
|
64
|
+
if (Array.isArray(data?.[0])) {
|
|
65
|
+
const out = data[0].map((seg) => (typeof seg?.[0] === 'string' ? seg[0] : '')).join('').trim();
|
|
66
|
+
if (out)
|
|
67
|
+
return out;
|
|
68
|
+
}
|
|
69
|
+
(0, logger_1.debugLog)('gtx 翻译响应结构异常,尝试备用通道');
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
catch (e) {
|
|
73
|
+
(0, logger_1.debugLog)(`gtx 翻译失败:${e?.message || e}`);
|
|
74
|
+
return null;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
/** MyMemory 备用(免费无 Key;需已知源语种,不支持 auto) */
|
|
78
|
+
async function viaMyMemory(rt, text, target, sourceLang) {
|
|
79
|
+
const sl = normalizeLang(sourceLang);
|
|
80
|
+
if (!sl)
|
|
81
|
+
return null;
|
|
82
|
+
const tl = target.toLowerCase();
|
|
83
|
+
const url = 'https://api.mymemory.translated.net/get' +
|
|
84
|
+
`?q=${encodeURIComponent(text.slice(0, 500))}&langpair=${encodeURIComponent(sl + '|' + tl)}`;
|
|
85
|
+
try {
|
|
86
|
+
const res = await rt.http.get(url, { timeout: 15000, headers: { 'User-Agent': rt.config.userAgent } });
|
|
87
|
+
const out = res.data?.responseData?.translatedText;
|
|
88
|
+
if (typeof out === 'string' && out.trim() && !/^MYMEMORY WARNING/i.test(out))
|
|
89
|
+
return out.trim();
|
|
90
|
+
(0, logger_1.debugLog)('MyMemory 翻译响应异常,放弃译文');
|
|
91
|
+
return null;
|
|
92
|
+
}
|
|
93
|
+
catch (e) {
|
|
94
|
+
logger_1.logger.info(`推文翻译失败(跳过译文继续发送):${e?.message || e}`);
|
|
95
|
+
return null;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
function normalizeLang(lang) {
|
|
99
|
+
if (!lang)
|
|
100
|
+
return null;
|
|
101
|
+
const l = lang.toLowerCase();
|
|
102
|
+
if (l === 'und' || l === 'zxx')
|
|
103
|
+
return null;
|
|
104
|
+
if (l.startsWith('zh'))
|
|
105
|
+
return 'zh-CN';
|
|
106
|
+
return l.split('-')[0];
|
|
107
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { AxiosInstance } from 'axios';
|
|
2
|
+
import type { ParsedData } from '../types';
|
|
3
|
+
/** GraphQL GET 器:可注入(生产用 tlsGet,测试用 mock) */
|
|
4
|
+
export type GraphqlGetter = (url: string, opts: {
|
|
5
|
+
headers?: Record<string, string>;
|
|
6
|
+
cookies?: Record<string, string>;
|
|
7
|
+
timeout?: number;
|
|
8
|
+
}) => Promise<{
|
|
9
|
+
status: number;
|
|
10
|
+
data: any;
|
|
11
|
+
}>;
|
|
12
|
+
export interface TwitterCreds {
|
|
13
|
+
authToken: string;
|
|
14
|
+
ct0: string;
|
|
15
|
+
}
|
|
16
|
+
export interface GrokTranslation {
|
|
17
|
+
text: string;
|
|
18
|
+
sourceLang?: string;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* X 原生 Grok 翻译(与网页"翻译推文"同源)。
|
|
22
|
+
* 需登录态;网页抓包实证:grok 特性开关 + client-language 头即可,无需
|
|
23
|
+
* x-client-transaction-id / cf_clearance。任何失败返回 null 由调用方回落通用翻译。
|
|
24
|
+
*/
|
|
25
|
+
export declare function fetchGrokTranslation(url: string, targetLang: string, creds: TwitterCreds, get?: GraphqlGetter): Promise<GrokTranslation | null>;
|
|
26
|
+
export declare function parseTwitter(url: string, http: AxiosInstance, creds?: TwitterCreds, getGraphql?: GraphqlGetter): Promise<ParsedData>;
|