@sns-parse/core 0.3.0-alpha.1 → 0.4.0-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/config.d.ts +7 -0
- package/lib/config.js +28 -0
- package/lib/engine/config-io.d.ts +44 -0
- package/lib/engine/config-io.js +155 -0
- package/lib/engine/config.d.ts +9 -0
- package/lib/engine/config.js +265 -0
- package/lib/engine/fetcher.d.ts +21 -0
- package/lib/engine/fetcher.js +228 -0
- package/lib/engine/gateway.d.ts +21 -0
- package/lib/engine/gateway.js +19 -0
- package/lib/engine/parser.d.ts +2 -0
- package/lib/engine/parser.js +162 -0
- package/lib/engine/platform-config.d.ts +24 -0
- package/lib/engine/platform-config.js +76 -0
- package/lib/engine/runtime.d.ts +49 -0
- package/lib/engine/runtime.js +90 -0
- package/lib/engine/translate.d.ts +15 -0
- package/lib/engine/translate.js +107 -0
- package/lib/engine/twitter-user.d.ts +67 -0
- package/lib/engine/twitter-user.js +185 -0
- package/lib/engine/twitter.d.ts +102 -0
- package/lib/engine/twitter.js +791 -0
- package/lib/index.d.ts +21 -0
- package/lib/index.js +21 -0
- package/lib/registry.d.ts +20 -0
- package/lib/registry.js +77 -0
- package/lib/utils/cache.d.ts +9 -0
- package/lib/utils/cache.js +34 -0
- package/lib/utils/common.d.ts +7 -0
- package/lib/utils/common.js +58 -0
- package/lib/utils/concurrency.d.ts +8 -0
- package/lib/utils/concurrency.js +29 -0
- package/lib/utils/field-mapping.d.ts +2 -0
- package/lib/utils/field-mapping.js +29 -0
- package/lib/utils/format.d.ts +4 -0
- package/lib/utils/format.js +84 -0
- package/lib/utils/gif.d.ts +10 -0
- package/lib/utils/gif.js +104 -0
- package/lib/utils/image-size.d.ts +10 -0
- package/lib/utils/image-size.js +69 -0
- package/lib/utils/merge.d.ts +64 -0
- package/lib/utils/merge.js +402 -0
- package/lib/utils/tls-client.d.ts +30 -0
- package/lib/utils/tls-client.js +120 -0
- package/lib/utils/url.d.ts +10 -0
- package/lib/utils/url.js +89 -0
- package/package.json +16 -4
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.createRuntime = createRuntime;
|
|
7
|
+
exports.createCoreExtensions = createCoreExtensions;
|
|
8
|
+
/**
|
|
9
|
+
* 运行时构建(自宿主 runtime 下沉,平台无关)。
|
|
10
|
+
*
|
|
11
|
+
* 平台定义列表(defs)由调用方传入:
|
|
12
|
+
* - koishi 兼容层传内置 BUILTIN_PLATFORMS(本地 definitions)
|
|
13
|
+
* - CLI 传 collectPlatformDefinitions()(已安装的 @sns-parse/platform-*)
|
|
14
|
+
* 默认扩展实现同样注入:koishi 传含 NSFW 的完整默认实现,CLI 传 createCoreExtensions()。
|
|
15
|
+
*/
|
|
16
|
+
const axios_1 = __importDefault(require("axios"));
|
|
17
|
+
const https_proxy_agent_1 = require("https-proxy-agent");
|
|
18
|
+
const host_1 = require("../host");
|
|
19
|
+
const cache_1 = require("../utils/cache");
|
|
20
|
+
const field_mapping_1 = require("../utils/field-mapping");
|
|
21
|
+
const platform_config_1 = require("./platform-config");
|
|
22
|
+
const gateway_1 = require("./gateway");
|
|
23
|
+
const merge_1 = require("../utils/merge");
|
|
24
|
+
const gif_1 = require("../utils/gif");
|
|
25
|
+
const translate_1 = require("./translate");
|
|
26
|
+
function createRuntime(source, config, opts = {}) {
|
|
27
|
+
const host = (0, host_1.createHost)(source);
|
|
28
|
+
const ctx = host.context;
|
|
29
|
+
const extensions = { ...(opts.defaultExtensions || {}), ...(host.extensions || {}) };
|
|
30
|
+
const dedupCache = new cache_1.SimpleLRUCache(1000, config.deduplicationInterval * 1000);
|
|
31
|
+
const cacheTTL = (config.cacheTTL || 600) * 1000;
|
|
32
|
+
const urlCacheLocal = new cache_1.SimpleLRUCache(500, cacheTTL);
|
|
33
|
+
const contentDedupCache = new cache_1.SimpleLRUCache(1000, config.deduplicationInterval * 1000);
|
|
34
|
+
const proxyConfig = config.proxy || {};
|
|
35
|
+
const customPlatforms = (config.customPlatforms || []).map((p) => ({
|
|
36
|
+
name: p.name,
|
|
37
|
+
apiUrl: p.apiUrl,
|
|
38
|
+
apiKey: p.apiKey || '',
|
|
39
|
+
authHeaderType: p.authHeaderType || 'Bearer',
|
|
40
|
+
customHeaderName: p.customHeaderName || 'X-API-Key',
|
|
41
|
+
fieldMapping: (0, field_mapping_1.parseFieldMapping)(p.fieldMapping),
|
|
42
|
+
proxy: p.proxy || null
|
|
43
|
+
}));
|
|
44
|
+
const defs = opts.defs || [];
|
|
45
|
+
const builtinRules = defs.flatMap(d => d.rules.map(pattern => ({ pattern, type: d.type })));
|
|
46
|
+
const customRules = (0, platform_config_1.buildCustomLinkRules)(config.customPlatforms || []);
|
|
47
|
+
const allRules = [...builtinRules, ...customRules];
|
|
48
|
+
const dedicatedApis = (0, gateway_1.dedicatedApisFrom)(defs);
|
|
49
|
+
const axiosConfig = {
|
|
50
|
+
timeout: config.timeout,
|
|
51
|
+
headers: {
|
|
52
|
+
'User-Agent': config.userAgent,
|
|
53
|
+
'Referer': 'https://www.baidu.com/',
|
|
54
|
+
'Content-Type': 'application/x-www-form-urlencoded'
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
if (proxyConfig.enabled && proxyConfig.host) {
|
|
58
|
+
// axios 内置 proxy 选项对 https 目标存在不做 CONNECT 隧道的经典缺陷,
|
|
59
|
+
// 改用 https-proxy-agent 显式代理(按请求生效,不污染全局 env)
|
|
60
|
+
const proxyUrl = `${proxyConfig.protocol || 'http'}://${proxyConfig.auth?.username ? `${encodeURIComponent(proxyConfig.auth.username)}:${encodeURIComponent(proxyConfig.auth.password || '')}@` : ''}${proxyConfig.host}:${proxyConfig.port || 7890}`;
|
|
61
|
+
const agent = new https_proxy_agent_1.HttpsProxyAgent(proxyUrl);
|
|
62
|
+
axiosConfig.httpAgent = agent;
|
|
63
|
+
axiosConfig.httpsAgent = agent;
|
|
64
|
+
axiosConfig.proxy = false;
|
|
65
|
+
}
|
|
66
|
+
const http = axios_1.default.create(axiosConfig);
|
|
67
|
+
return {
|
|
68
|
+
host,
|
|
69
|
+
ctx,
|
|
70
|
+
config,
|
|
71
|
+
http,
|
|
72
|
+
proxyConfig,
|
|
73
|
+
cacheTTL,
|
|
74
|
+
dedupCache,
|
|
75
|
+
urlCacheLocal,
|
|
76
|
+
contentDedupCache,
|
|
77
|
+
customPlatforms,
|
|
78
|
+
allRules,
|
|
79
|
+
dedicatedApis,
|
|
80
|
+
extensions,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
/** core 自带扩展实现(无 NSFW:CLI / 嵌入式宿主用;koishi 层在其上叠加 NSFW) */
|
|
84
|
+
function createCoreExtensions() {
|
|
85
|
+
return {
|
|
86
|
+
mergeImages: merge_1.mergeImages,
|
|
87
|
+
mp4ToGif: gif_1.mp4ToGif,
|
|
88
|
+
translate: translate_1.translateText,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 推文翻译:Google gtx 免费端点(无 Key、GET-only)。
|
|
3
|
+
*
|
|
4
|
+
* X 网页端的 Grok 翻译走私有通道(translations/show.json 已 404,GraphQL
|
|
5
|
+
* 仅返回 is_translatable 标志),无公开端点可调,故用通用翻译源兜底。
|
|
6
|
+
* 译文存入 ParsedData.translation,经 ${翻译} 模板变量展示,空值行自动隐藏。
|
|
7
|
+
*/
|
|
8
|
+
import type { ParserRuntime } from './runtime';
|
|
9
|
+
import type { TranslateResult } from '../extensions';
|
|
10
|
+
export type { TranslateResult };
|
|
11
|
+
/** 判断是否需要跳过翻译(目标语系与推文语种相同 / 无语言内容) */
|
|
12
|
+
export declare function shouldSkipTranslate(tweetLang: string | undefined, target: string): boolean;
|
|
13
|
+
export declare function langName(code?: string): string;
|
|
14
|
+
/** 翻译文本;失败返回 null(调用方保持无译文继续发送)。gtx 优先,MyMemory 兜底 */
|
|
15
|
+
export declare function translateText(rt: ParserRuntime, text: string, target: string, sourceLang?: string): Promise<TranslateResult | null>;
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.shouldSkipTranslate = shouldSkipTranslate;
|
|
4
|
+
exports.langName = langName;
|
|
5
|
+
exports.translateText = translateText;
|
|
6
|
+
const logger_1 = require("../logger");
|
|
7
|
+
/** 判断是否需要跳过翻译(目标语系与推文语种相同 / 无语言内容) */
|
|
8
|
+
function shouldSkipTranslate(tweetLang, target) {
|
|
9
|
+
if (!tweetLang)
|
|
10
|
+
return false;
|
|
11
|
+
const l = tweetLang.toLowerCase();
|
|
12
|
+
if (l === 'zxx' || l === 'und')
|
|
13
|
+
return l === 'zxx'; // zxx=无语言内容;und=未定则仍尝试
|
|
14
|
+
const t = target.toLowerCase();
|
|
15
|
+
if (t === 'zh')
|
|
16
|
+
return l === 'zh' || l.startsWith('zh-');
|
|
17
|
+
if (t === 'zh-tw')
|
|
18
|
+
return l === 'zh-tw' || l === 'zh-hant' || (l.startsWith('zh') && l !== 'zh-cn');
|
|
19
|
+
if (t === 'en')
|
|
20
|
+
return l === 'en';
|
|
21
|
+
return l === t;
|
|
22
|
+
}
|
|
23
|
+
/** 常见语种代码 → 中文名(未知代码原样展示) */
|
|
24
|
+
const LANG_NAMES = {
|
|
25
|
+
fr: '法语', ja: '日语', ko: '韩语', en: '英语', zh: '中文', es: '西班牙语', pt: '葡萄牙语',
|
|
26
|
+
de: '德语', ru: '俄语', ar: '阿拉伯语', it: '意大利语', th: '泰语', vi: '越南语',
|
|
27
|
+
id: '印尼语', tr: '土耳其语', nl: '荷兰语', pl: '波兰语', hi: '印地语', uk: '乌克兰语',
|
|
28
|
+
sv: '瑞典语', no: '挪威语', fi: '芬兰语', da: '丹麦语', cs: '捷克语', ro: '罗马尼亚语',
|
|
29
|
+
hu: '匈牙利语', el: '希腊语', he: '希伯来语', fa: '波斯语', ur: '乌尔都语', ms: '马来语',
|
|
30
|
+
tl: '菲律宾语', ca: '加泰罗尼亚语', bg: '保加利亚语', hr: '克罗地亚语', sr: '塞尔维亚语',
|
|
31
|
+
};
|
|
32
|
+
function langName(code) {
|
|
33
|
+
if (!code)
|
|
34
|
+
return '';
|
|
35
|
+
const base = code.toLowerCase().split('-')[0];
|
|
36
|
+
if (base === 'und')
|
|
37
|
+
return '';
|
|
38
|
+
return LANG_NAMES[base] || code;
|
|
39
|
+
}
|
|
40
|
+
/** 翻译文本;失败返回 null(调用方保持无译文继续发送)。gtx 优先,MyMemory 兜底 */
|
|
41
|
+
async function translateText(rt, text, target, sourceLang) {
|
|
42
|
+
if (!text)
|
|
43
|
+
return null;
|
|
44
|
+
const out = await viaGtx(rt, text, target);
|
|
45
|
+
if (out)
|
|
46
|
+
return { text: out, provider: 'Google' };
|
|
47
|
+
const backup = await viaMyMemory(rt, text, target, sourceLang);
|
|
48
|
+
if (backup)
|
|
49
|
+
return { text: backup, provider: 'MyMemory' };
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
/** Google gtx 免费端点(无 Key;sl=auto 自动检测源语种) */
|
|
53
|
+
async function viaGtx(rt, text, target) {
|
|
54
|
+
const url = 'https://translate.googleapis.com/translate_a/single' +
|
|
55
|
+
`?client=gtx&sl=auto&dt=t&tl=${encodeURIComponent(target)}` +
|
|
56
|
+
`&q=${encodeURIComponent(text.slice(0, 2000))}`;
|
|
57
|
+
try {
|
|
58
|
+
const res = await rt.http.get(url, {
|
|
59
|
+
timeout: 15000,
|
|
60
|
+
headers: { 'User-Agent': rt.config.userAgent },
|
|
61
|
+
});
|
|
62
|
+
const data = res.data;
|
|
63
|
+
// gtx 响应形如 [[["译文","原文",null,null,10]],null,"fr",...]
|
|
64
|
+
if (Array.isArray(data?.[0])) {
|
|
65
|
+
const out = data[0].map((seg) => (typeof seg?.[0] === 'string' ? seg[0] : '')).join('').trim();
|
|
66
|
+
if (out)
|
|
67
|
+
return out;
|
|
68
|
+
}
|
|
69
|
+
(0, logger_1.debugLog)('gtx 翻译响应结构异常,尝试备用通道');
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
catch (e) {
|
|
73
|
+
(0, logger_1.debugLog)(`gtx 翻译失败:${e?.message || e}`);
|
|
74
|
+
return null;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
/** MyMemory 备用(免费无 Key;需已知源语种,不支持 auto) */
|
|
78
|
+
async function viaMyMemory(rt, text, target, sourceLang) {
|
|
79
|
+
const sl = normalizeLang(sourceLang);
|
|
80
|
+
if (!sl)
|
|
81
|
+
return null;
|
|
82
|
+
const tl = target.toLowerCase();
|
|
83
|
+
const url = 'https://api.mymemory.translated.net/get' +
|
|
84
|
+
`?q=${encodeURIComponent(text.slice(0, 500))}&langpair=${encodeURIComponent(sl + '|' + tl)}`;
|
|
85
|
+
try {
|
|
86
|
+
const res = await rt.http.get(url, { timeout: 15000, headers: { 'User-Agent': rt.config.userAgent } });
|
|
87
|
+
const out = res.data?.responseData?.translatedText;
|
|
88
|
+
if (typeof out === 'string' && out.trim() && !/^MYMEMORY WARNING/i.test(out))
|
|
89
|
+
return out.trim();
|
|
90
|
+
(0, logger_1.debugLog)('MyMemory 翻译响应异常,放弃译文');
|
|
91
|
+
return null;
|
|
92
|
+
}
|
|
93
|
+
catch (e) {
|
|
94
|
+
logger_1.logger.info(`推文翻译失败(跳过译文继续发送):${e?.message || e}`);
|
|
95
|
+
return null;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
function normalizeLang(lang) {
|
|
99
|
+
if (!lang)
|
|
100
|
+
return null;
|
|
101
|
+
const l = lang.toLowerCase();
|
|
102
|
+
if (l === 'und' || l === 'zxx')
|
|
103
|
+
return null;
|
|
104
|
+
if (l.startsWith('zh'))
|
|
105
|
+
return 'zh-CN';
|
|
106
|
+
return l.split('-')[0];
|
|
107
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* X 用户维度查询:时间线(推文/回复/点赞)与关注/粉丝列表。
|
|
3
|
+
*
|
|
4
|
+
* 全部走鉴权 GraphQL(auth_token + ct0,TLS 指纹由 tlsget-rs 提供)。
|
|
5
|
+
* queryId 与特性集等公共件见 ./twitter(DEFAULT_QUERY_IDS 可被调用方覆盖)。
|
|
6
|
+
*/
|
|
7
|
+
import type { ParsedData } from '../types';
|
|
8
|
+
import { categorizeTweetResult, DEFAULT_QUERY_IDS, TIMELINE_FEATURES, type GraphqlGetter, type TwitterCreds, type TwitterQueryIds } from './twitter';
|
|
9
|
+
export { categorizeTweetResult, DEFAULT_QUERY_IDS, TIMELINE_FEATURES };
|
|
10
|
+
export type { TwitterQueryIds };
|
|
11
|
+
export interface TwitterUserInfo {
|
|
12
|
+
userId: string;
|
|
13
|
+
name: string;
|
|
14
|
+
screenName: string;
|
|
15
|
+
description: string;
|
|
16
|
+
followers: number;
|
|
17
|
+
friends: number;
|
|
18
|
+
}
|
|
19
|
+
export declare function resolveTwitterUser(screenName: string, creds: TwitterCreds, get?: GraphqlGetter, queryIds?: TwitterQueryIds): Promise<TwitterUserInfo>;
|
|
20
|
+
export type TimelineTab = 'tweets' | 'replies' | 'likes';
|
|
21
|
+
export interface TimelineEntry {
|
|
22
|
+
id: string;
|
|
23
|
+
tweet: ParsedData;
|
|
24
|
+
/** 文字/视频/图片/转推/回复 中的若干类 */
|
|
25
|
+
kinds: ('text' | 'video' | 'image' | 'retweet' | 'reply')[];
|
|
26
|
+
isRetweet: boolean;
|
|
27
|
+
isReply: boolean;
|
|
28
|
+
replyToId?: string;
|
|
29
|
+
url: string;
|
|
30
|
+
}
|
|
31
|
+
/** 兼容别名:timeline instructions → 推文结果 + 底部游标 */
|
|
32
|
+
export declare function parseTimeline(instructions: any[]): {
|
|
33
|
+
results: any[];
|
|
34
|
+
bottomCursor?: string;
|
|
35
|
+
};
|
|
36
|
+
/** 用户时间线(tab:tweets=推文/转推;replies=含回复;likes=当前登录用户的点赞) */
|
|
37
|
+
export declare function fetchUserTimeline(opts: {
|
|
38
|
+
screenName?: string;
|
|
39
|
+
userId?: string;
|
|
40
|
+
tab?: TimelineTab;
|
|
41
|
+
limit?: number;
|
|
42
|
+
creds: TwitterCreds;
|
|
43
|
+
get?: GraphqlGetter;
|
|
44
|
+
queryIds?: TwitterQueryIds;
|
|
45
|
+
}): Promise<TimelineEntry[]>;
|
|
46
|
+
export interface TwitterConnectionUser {
|
|
47
|
+
name: string;
|
|
48
|
+
screenName: string;
|
|
49
|
+
description: string;
|
|
50
|
+
followers: number;
|
|
51
|
+
verified: boolean;
|
|
52
|
+
followedBy: boolean;
|
|
53
|
+
}
|
|
54
|
+
/** 纯函数:从 Followers/Following instructions 抽取用户与底部游标 */
|
|
55
|
+
export declare function parseConnections(instructions: any[]): {
|
|
56
|
+
users: TwitterConnectionUser[];
|
|
57
|
+
bottomCursor?: string;
|
|
58
|
+
};
|
|
59
|
+
/** 关注列表(following)或粉丝列表(followers) */
|
|
60
|
+
export declare function fetchUserConnections(opts: {
|
|
61
|
+
screenName: string;
|
|
62
|
+
type: 'followers' | 'following';
|
|
63
|
+
limit?: number;
|
|
64
|
+
creds: TwitterCreds;
|
|
65
|
+
get?: GraphqlGetter;
|
|
66
|
+
queryIds?: TwitterQueryIds;
|
|
67
|
+
}): Promise<TwitterConnectionUser[]>;
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.TIMELINE_FEATURES = exports.DEFAULT_QUERY_IDS = exports.categorizeTweetResult = void 0;
|
|
4
|
+
exports.resolveTwitterUser = resolveTwitterUser;
|
|
5
|
+
exports.parseTimeline = parseTimeline;
|
|
6
|
+
exports.fetchUserTimeline = fetchUserTimeline;
|
|
7
|
+
exports.parseConnections = parseConnections;
|
|
8
|
+
exports.fetchUserConnections = fetchUserConnections;
|
|
9
|
+
const tls_client_1 = require("../utils/tls-client");
|
|
10
|
+
const twitter_1 = require("./twitter");
|
|
11
|
+
Object.defineProperty(exports, "categorizeTweetResult", { enumerable: true, get: function () { return twitter_1.categorizeTweetResult; } });
|
|
12
|
+
Object.defineProperty(exports, "DEFAULT_QUERY_IDS", { enumerable: true, get: function () { return twitter_1.DEFAULT_QUERY_IDS; } });
|
|
13
|
+
Object.defineProperty(exports, "TIMELINE_FEATURES", { enumerable: true, get: function () { return twitter_1.TIMELINE_FEATURES; } });
|
|
14
|
+
async function resolveTwitterUser(screenName, creds, get = tls_client_1.tlsGet, queryIds = {}) {
|
|
15
|
+
const ids = { ...twitter_1.DEFAULT_QUERY_IDS, ...queryIds };
|
|
16
|
+
const variables = {
|
|
17
|
+
screen_name: screenName.replace(/^@/, ''),
|
|
18
|
+
withSafetyModeUserFields: true,
|
|
19
|
+
withSuperFollowsUserFields: true,
|
|
20
|
+
};
|
|
21
|
+
const data = await (0, twitter_1.twitterGraphql)('UserByScreenName', ids.UserByScreenName, variables, creds, get);
|
|
22
|
+
const user = data?.data?.user?.result;
|
|
23
|
+
if (!user || user.__typename === 'UserUnavailable') {
|
|
24
|
+
throw new Error(`用户 @${screenName} 不可访问(不存在、被封或需登录)`);
|
|
25
|
+
}
|
|
26
|
+
const legacy = user.legacy || user.core || {};
|
|
27
|
+
return {
|
|
28
|
+
userId: String(user.rest_id ?? legacy.id_str ?? ''),
|
|
29
|
+
name: String(legacy.name ?? ''),
|
|
30
|
+
screenName: String(legacy.screen_name ?? screenName.replace(/^@/, '')),
|
|
31
|
+
description: String(user.profile_bio?.description ?? legacy.description ?? ''),
|
|
32
|
+
followers: Number(legacy.followers_count ?? 0) || 0,
|
|
33
|
+
friends: Number(legacy.friends_count ?? 0) || 0,
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
/** 兼容别名:timeline instructions → 推文结果 + 底部游标 */
|
|
37
|
+
function parseTimeline(instructions) {
|
|
38
|
+
return (0, twitter_1.walkTimelineResults)(instructions);
|
|
39
|
+
}
|
|
40
|
+
/** 用户时间线(tab:tweets=推文/转推;replies=含回复;likes=当前登录用户的点赞) */
|
|
41
|
+
async function fetchUserTimeline(opts) {
|
|
42
|
+
const tab = opts.tab || 'tweets';
|
|
43
|
+
const limit = Math.max(1, Math.min(200, opts.limit ?? 20));
|
|
44
|
+
const get = opts.get || tls_client_1.tlsGet;
|
|
45
|
+
const ids = { ...twitter_1.DEFAULT_QUERY_IDS, ...opts.queryIds };
|
|
46
|
+
let userId = opts.userId;
|
|
47
|
+
if (!userId) {
|
|
48
|
+
const u = await resolveTwitterUser(opts.screenName, opts.creds, get, opts.queryIds);
|
|
49
|
+
userId = u.userId;
|
|
50
|
+
}
|
|
51
|
+
const [opName, queryId] = tab === 'likes'
|
|
52
|
+
? ['Likes', ids.Likes]
|
|
53
|
+
: tab === 'replies'
|
|
54
|
+
? ['UserTweetsAndReplies', ids.UserTweetsAndReplies]
|
|
55
|
+
: ['UserTweets', ids.UserTweets];
|
|
56
|
+
const out = [];
|
|
57
|
+
const seen = new Set();
|
|
58
|
+
let cursor;
|
|
59
|
+
let pages = 0;
|
|
60
|
+
while (out.length < limit && pages < 20) {
|
|
61
|
+
pages++;
|
|
62
|
+
const variables = {
|
|
63
|
+
userId,
|
|
64
|
+
count: 20,
|
|
65
|
+
includePromotedContent: false,
|
|
66
|
+
withQuickPromoteEligibilityTweetQuery: false,
|
|
67
|
+
withVoice: true,
|
|
68
|
+
withV2Timeline: true,
|
|
69
|
+
};
|
|
70
|
+
if (cursor)
|
|
71
|
+
variables.cursor = cursor;
|
|
72
|
+
const data = await (0, twitter_1.twitterGraphql)(opName, queryId, variables, opts.creds, get);
|
|
73
|
+
const instructions = (() => {
|
|
74
|
+
const user = data?.data?.user?.result;
|
|
75
|
+
return (user?.timeline_v2?.timeline || user?.timeline?.timeline)?.instructions || [];
|
|
76
|
+
})();
|
|
77
|
+
const { results, bottomCursor } = (0, twitter_1.walkTimelineResults)(instructions);
|
|
78
|
+
for (const r of results) {
|
|
79
|
+
const cat = (0, twitter_1.categorizeTweetResult)(r);
|
|
80
|
+
if (!cat.id || seen.has(cat.id))
|
|
81
|
+
continue;
|
|
82
|
+
seen.add(cat.id);
|
|
83
|
+
const kinds = [];
|
|
84
|
+
if (cat.isRetweet)
|
|
85
|
+
kinds.push('retweet');
|
|
86
|
+
if (cat.isReply)
|
|
87
|
+
kinds.push('reply');
|
|
88
|
+
if (cat.hasVideo)
|
|
89
|
+
kinds.push('video');
|
|
90
|
+
else if (cat.hasImage)
|
|
91
|
+
kinds.push('image');
|
|
92
|
+
else if (!cat.isRetweet)
|
|
93
|
+
kinds.push('text');
|
|
94
|
+
out.push({
|
|
95
|
+
id: cat.id,
|
|
96
|
+
tweet: (0, twitter_1.mapGraphql)(cat.inner),
|
|
97
|
+
kinds,
|
|
98
|
+
isRetweet: cat.isRetweet,
|
|
99
|
+
isReply: cat.isReply,
|
|
100
|
+
replyToId: cat.replyToId,
|
|
101
|
+
url: `https://x.com/i/web/status/${cat.id}`,
|
|
102
|
+
});
|
|
103
|
+
if (out.length >= limit)
|
|
104
|
+
break;
|
|
105
|
+
}
|
|
106
|
+
if (!bottomCursor || !results.length)
|
|
107
|
+
break;
|
|
108
|
+
cursor = bottomCursor;
|
|
109
|
+
}
|
|
110
|
+
return out;
|
|
111
|
+
}
|
|
112
|
+
function userOfEntry(itemContent) {
|
|
113
|
+
const user = itemContent?.user_results?.result;
|
|
114
|
+
if (!user || user.__typename === 'UserUnavailable')
|
|
115
|
+
return null;
|
|
116
|
+
const legacy = user.legacy || {};
|
|
117
|
+
return {
|
|
118
|
+
name: String(legacy.name ?? user.core?.name ?? ''),
|
|
119
|
+
screenName: String(legacy.screen_name ?? user.core?.screen_name ?? ''),
|
|
120
|
+
description: String(user.profile_bio?.description ?? legacy.description ?? ''),
|
|
121
|
+
followers: Number(legacy.followers_count ?? 0) || 0,
|
|
122
|
+
verified: !!user.is_blue_verified || !!legacy.verified,
|
|
123
|
+
followedBy: !!legacy.followed_by,
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
/** 纯函数:从 Followers/Following instructions 抽取用户与底部游标 */
|
|
127
|
+
function parseConnections(instructions) {
|
|
128
|
+
const users = [];
|
|
129
|
+
let bottomCursor;
|
|
130
|
+
for (const ins of instructions || []) {
|
|
131
|
+
for (const entry of ins?.entries || []) {
|
|
132
|
+
const content = entry?.content;
|
|
133
|
+
if (!content)
|
|
134
|
+
continue;
|
|
135
|
+
if (content.entryType === 'TimelineTimelineCursor' && content.contentType === 'Bottom') {
|
|
136
|
+
bottomCursor = String(content.value || bottomCursor || '');
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
if (content.entryType === 'TimelineTimelineItem' && content.itemContent) {
|
|
140
|
+
const u = userOfEntry(content.itemContent);
|
|
141
|
+
if (u && u.screenName)
|
|
142
|
+
users.push(u);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return { users, bottomCursor };
|
|
147
|
+
}
|
|
148
|
+
/** 关注列表(following)或粉丝列表(followers) */
|
|
149
|
+
async function fetchUserConnections(opts) {
|
|
150
|
+
const limit = Math.max(1, Math.min(1000, opts.limit ?? 50));
|
|
151
|
+
const get = opts.get || tls_client_1.tlsGet;
|
|
152
|
+
const ids = { ...twitter_1.DEFAULT_QUERY_IDS, ...opts.queryIds };
|
|
153
|
+
const u = await resolveTwitterUser(opts.screenName, opts.creds, get, opts.queryIds);
|
|
154
|
+
const [opName, queryId] = opts.type === 'following'
|
|
155
|
+
? ['Following', ids.Following]
|
|
156
|
+
: ['Followers', ids.Followers];
|
|
157
|
+
const out = [];
|
|
158
|
+
const seen = new Set();
|
|
159
|
+
let cursor;
|
|
160
|
+
let pages = 0;
|
|
161
|
+
while (out.length < limit && pages < 50) {
|
|
162
|
+
pages++;
|
|
163
|
+
const variables = { userId: u.userId, count: 50, includePromotedContent: false };
|
|
164
|
+
if (cursor)
|
|
165
|
+
variables.cursor = cursor;
|
|
166
|
+
const data = await (0, twitter_1.twitterGraphql)(opName, queryId, variables, opts.creds, get);
|
|
167
|
+
const instructions = (() => {
|
|
168
|
+
const user = data?.data?.user?.result;
|
|
169
|
+
return (user?.timeline_v2?.timeline || user?.timeline?.timeline)?.instructions || [];
|
|
170
|
+
})();
|
|
171
|
+
const { users, bottomCursor } = parseConnections(instructions);
|
|
172
|
+
for (const user of users) {
|
|
173
|
+
if (seen.has(user.screenName))
|
|
174
|
+
continue;
|
|
175
|
+
seen.add(user.screenName);
|
|
176
|
+
out.push(user);
|
|
177
|
+
if (out.length >= limit)
|
|
178
|
+
break;
|
|
179
|
+
}
|
|
180
|
+
if (!bottomCursor || !users.length)
|
|
181
|
+
break;
|
|
182
|
+
cursor = bottomCursor;
|
|
183
|
+
}
|
|
184
|
+
return out;
|
|
185
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import type { AxiosInstance } from 'axios';
|
|
2
|
+
import type { ParsedData } from '../types';
|
|
3
|
+
/** GraphQL GET 器:可注入(生产用 tlsGet,测试用 mock) */
|
|
4
|
+
export type GraphqlGetter = (url: string, opts: {
|
|
5
|
+
headers?: Record<string, string>;
|
|
6
|
+
cookies?: Record<string, string>;
|
|
7
|
+
timeout?: number;
|
|
8
|
+
}) => Promise<{
|
|
9
|
+
status: number;
|
|
10
|
+
data: any;
|
|
11
|
+
}>;
|
|
12
|
+
export interface TwitterCreds {
|
|
13
|
+
authToken: string;
|
|
14
|
+
ct0: string;
|
|
15
|
+
}
|
|
16
|
+
export interface TwitterQueryIds {
|
|
17
|
+
UserByScreenName?: string;
|
|
18
|
+
UserTweets?: string;
|
|
19
|
+
UserTweetsAndReplies?: string;
|
|
20
|
+
Likes?: string;
|
|
21
|
+
Followers?: string;
|
|
22
|
+
Following?: string;
|
|
23
|
+
TweetDetail?: string;
|
|
24
|
+
}
|
|
25
|
+
/** 默认 queryId 清单(随 x.com web 版本漂移;调用方可整体覆盖) */
|
|
26
|
+
export declare const DEFAULT_QUERY_IDS: Required<TwitterQueryIds>;
|
|
27
|
+
/** 时间线类查询的特性集(UserTweets/Likes/Followers/TweetDetail 共用) */
|
|
28
|
+
export declare const TIMELINE_FEATURES: Record<string, boolean>;
|
|
29
|
+
/** 鉴权 GraphQL 通用请求(headers/cookie 与 TweetResultByRestId 同源) */
|
|
30
|
+
export declare function twitterGraphql(opName: string, queryId: string, variables: Record<string, any>, creds: TwitterCreds, get: GraphqlGetter, features?: Record<string, boolean>): Promise<any>;
|
|
31
|
+
/** 纯函数:把单条 tweet result 归类(含转推解包) */
|
|
32
|
+
export declare function categorizeTweetResult(result: any): {
|
|
33
|
+
inner: any;
|
|
34
|
+
isRetweet: boolean;
|
|
35
|
+
isReply: boolean;
|
|
36
|
+
hasVideo: boolean;
|
|
37
|
+
hasImage: boolean;
|
|
38
|
+
isText: boolean;
|
|
39
|
+
id: string;
|
|
40
|
+
replyToId?: string;
|
|
41
|
+
};
|
|
42
|
+
/** 纯函数:从 timeline instructions 抽取推文结果与底部游标(Item/Module 均收) */
|
|
43
|
+
export declare function walkTimelineResults(instructions: any[]): {
|
|
44
|
+
results: any[];
|
|
45
|
+
bottomCursor?: string;
|
|
46
|
+
};
|
|
47
|
+
export interface GrokTranslation {
|
|
48
|
+
text: string;
|
|
49
|
+
sourceLang?: string;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* X 原生 Grok 翻译(与网页"翻译推文"同源)。
|
|
53
|
+
* 需登录态;网页抓包实证:grok 特性开关 + client-language 头即可,无需
|
|
54
|
+
* x-client-transaction-id / cf_clearance。任何失败返回 null 由调用方回落通用翻译。
|
|
55
|
+
*/
|
|
56
|
+
export declare function fetchGrokTranslation(url: string, targetLang: string, creds: TwitterCreds, get?: GraphqlGetter): Promise<GrokTranslation | null>;
|
|
57
|
+
export declare function extractTweetId(url: string): string | null;
|
|
58
|
+
/** 把 GraphQL TweetResultByRestId 响应映射为 ParsedData */
|
|
59
|
+
export declare function mapGraphql(rawResult: any): ParsedData;
|
|
60
|
+
/** 解除 TweetWithVisibilityResults 包裹(NSFW/受限推文) */
|
|
61
|
+
export declare function unwrapTweetResult(result: any): any;
|
|
62
|
+
/** 鉴权 GraphQL 原始获取:返回未映射的 tweet result 节点(树构建复用) */
|
|
63
|
+
export declare function fetchGraphqlRaw(id: string, creds: TwitterCreds, get: GraphqlGetter): Promise<any>;
|
|
64
|
+
export declare function parseTwitter(url: string, http: AxiosInstance, creds?: TwitterCreds, getGraphql?: GraphqlGetter): Promise<ParsedData>;
|
|
65
|
+
/**
|
|
66
|
+
* 推文树节点:
|
|
67
|
+
* - quoted = 本推引用的推文(递归)
|
|
68
|
+
* - replyTo = 本推回复的目标(向根方向递归)
|
|
69
|
+
* - replies = 本推收到的回复(评论,向下递归;withReplies 时填充)
|
|
70
|
+
*/
|
|
71
|
+
export interface TweetTree {
|
|
72
|
+
id: string;
|
|
73
|
+
tweet: ParsedData;
|
|
74
|
+
quoted?: TweetTree;
|
|
75
|
+
replyTo?: TweetTree;
|
|
76
|
+
replies?: TweetTree[];
|
|
77
|
+
}
|
|
78
|
+
/** 游客态(syndication)取被引用推文 ID:显式字段优先;否则启发式——引用链的
|
|
79
|
+
* t.co 短链由 X 追加在正文末尾,expanded_url 为推文永久链。 */
|
|
80
|
+
export declare function guestQuoteId(tw: any): string | undefined;
|
|
81
|
+
/** 游客态(syndication)取回复目标推文 ID */
|
|
82
|
+
export declare function guestReplyToId(tw: any): string | undefined;
|
|
83
|
+
export interface FetchTweetTreeOptions {
|
|
84
|
+
/** 拉取会话回复树(TweetDetail;需登录态) */
|
|
85
|
+
withReplies?: boolean;
|
|
86
|
+
/** 回复条数上限(默认 100) */
|
|
87
|
+
repliesLimit?: number;
|
|
88
|
+
/** queryId 覆盖 */
|
|
89
|
+
queryIds?: TwitterQueryIds;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* 纯函数:由会话内的全部 tweet result 装配回复树(挂在 focal 节点下)。
|
|
93
|
+
* 依据 in_reply_to_status_id_str 建父子边;父不在集合内的孤儿回复跳过。
|
|
94
|
+
*/
|
|
95
|
+
export declare function assembleReplies(focalId: string, results: any[], limit: number, makeNode: (raw: any) => TweetTree): TweetTree[];
|
|
96
|
+
/**
|
|
97
|
+
* 拉取推文树:引用链(quoted,向下递归)+ 回复链(replyTo,向根递归)
|
|
98
|
+
* + 可选会话回复树(replies,向下递归,TweetDetail 装配)。
|
|
99
|
+
* - 有登录态:GraphQL 单次响应自带嵌套 quoted_status_result;父级逐次拉取
|
|
100
|
+
* - 无登录态:syndication 逐节点拉取(引用 ID 走启发式);不可达分支静默截断
|
|
101
|
+
*/
|
|
102
|
+
export declare function fetchTweetTree(url: string, http: AxiosInstance, creds?: TwitterCreds, getGraphql?: GraphqlGetter, opts?: FetchTweetTreeOptions): Promise<TweetTree>;
|