@sns-parse/core 0.4.0-alpha.6 → 0.4.0-alpha.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/engine/twitter-user.d.ts +87 -0
- package/lib/engine/twitter-user.js +307 -0
- package/lib/engine/twitter.d.ts +25 -0
- package/lib/engine/twitter.js +128 -3
- package/lib/index.d.ts +1 -0
- package/lib/index.js +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* X 用户维度查询:时间线(推文/回复/点赞)与关注/粉丝列表。
|
|
3
|
+
*
|
|
4
|
+
* 全部走鉴权 GraphQL(auth_token + ct0,TLS 指纹由 tlsget-rs 提供)。
|
|
5
|
+
* queryId 随 x.com web 版本漂移:默认值取自活跃维护的开源清单
|
|
6
|
+
* (twitter-api-client),并支持调用方整体覆盖(queryIds 参数)。
|
|
7
|
+
*/
|
|
8
|
+
import type { ParsedData } from '../types';
|
|
9
|
+
import { type GraphqlGetter, type TwitterCreds } from './twitter';
|
|
10
|
+
export interface TwitterQueryIds {
|
|
11
|
+
UserByScreenName?: string;
|
|
12
|
+
UserTweets?: string;
|
|
13
|
+
UserTweetsAndReplies?: string;
|
|
14
|
+
Likes?: string;
|
|
15
|
+
Followers?: string;
|
|
16
|
+
Following?: string;
|
|
17
|
+
}
|
|
18
|
+
/** 默认 queryId 清单(漂移时可用 queryIds 参数整体覆盖) */
|
|
19
|
+
export declare const DEFAULT_QUERY_IDS: Required<TwitterQueryIds>;
|
|
20
|
+
export interface TwitterUserInfo {
|
|
21
|
+
userId: string;
|
|
22
|
+
name: string;
|
|
23
|
+
screenName: string;
|
|
24
|
+
description: string;
|
|
25
|
+
followers: number;
|
|
26
|
+
friends: number;
|
|
27
|
+
}
|
|
28
|
+
export declare function resolveTwitterUser(screenName: string, creds: TwitterCreds, get?: GraphqlGetter, queryIds?: TwitterQueryIds): Promise<TwitterUserInfo>;
|
|
29
|
+
export type TimelineTab = 'tweets' | 'replies' | 'likes';
|
|
30
|
+
export interface TimelineEntry {
|
|
31
|
+
id: string;
|
|
32
|
+
tweet: ParsedData;
|
|
33
|
+
/** 文字/视频/图片/转推/回复 中的若干类 */
|
|
34
|
+
kinds: ('text' | 'video' | 'image' | 'retweet' | 'reply')[];
|
|
35
|
+
isRetweet: boolean;
|
|
36
|
+
isReply: boolean;
|
|
37
|
+
replyToId?: string;
|
|
38
|
+
url: string;
|
|
39
|
+
}
|
|
40
|
+
/** 纯函数:把单条 tweet result 归类(含转推解包) */
|
|
41
|
+
export declare function categorizeTweetResult(result: any): {
|
|
42
|
+
inner: any;
|
|
43
|
+
isRetweet: boolean;
|
|
44
|
+
isReply: boolean;
|
|
45
|
+
hasVideo: boolean;
|
|
46
|
+
hasImage: boolean;
|
|
47
|
+
isText: boolean;
|
|
48
|
+
id: string;
|
|
49
|
+
replyToId?: string;
|
|
50
|
+
};
|
|
51
|
+
/** 纯函数:从 timeline instructions 抽取推文结果与底部游标 */
|
|
52
|
+
export declare function parseTimeline(instructions: any[]): {
|
|
53
|
+
results: any[];
|
|
54
|
+
bottomCursor?: string;
|
|
55
|
+
};
|
|
56
|
+
/** 用户时间线(tab:tweets=推文/转推;replies=含回复;likes=当前登录用户的点赞) */
|
|
57
|
+
export declare function fetchUserTimeline(opts: {
|
|
58
|
+
screenName?: string;
|
|
59
|
+
userId?: string;
|
|
60
|
+
tab?: TimelineTab;
|
|
61
|
+
limit?: number;
|
|
62
|
+
creds: TwitterCreds;
|
|
63
|
+
get?: GraphqlGetter;
|
|
64
|
+
queryIds?: TwitterQueryIds;
|
|
65
|
+
}): Promise<TimelineEntry[]>;
|
|
66
|
+
export interface TwitterConnectionUser {
|
|
67
|
+
name: string;
|
|
68
|
+
screenName: string;
|
|
69
|
+
description: string;
|
|
70
|
+
followers: number;
|
|
71
|
+
verified: boolean;
|
|
72
|
+
followedBy: boolean;
|
|
73
|
+
}
|
|
74
|
+
/** 纯函数:从 Followers/Following instructions 抽取用户与底部游标 */
|
|
75
|
+
export declare function parseConnections(instructions: any[]): {
|
|
76
|
+
users: TwitterConnectionUser[];
|
|
77
|
+
bottomCursor?: string;
|
|
78
|
+
};
|
|
79
|
+
/** 关注列表(following)或粉丝列表(followers) */
|
|
80
|
+
export declare function fetchUserConnections(opts: {
|
|
81
|
+
screenName: string;
|
|
82
|
+
type: 'followers' | 'following';
|
|
83
|
+
limit?: number;
|
|
84
|
+
creds: TwitterCreds;
|
|
85
|
+
get?: GraphqlGetter;
|
|
86
|
+
queryIds?: TwitterQueryIds;
|
|
87
|
+
}): Promise<TwitterConnectionUser[]>;
|
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DEFAULT_QUERY_IDS = void 0;
|
|
4
|
+
exports.resolveTwitterUser = resolveTwitterUser;
|
|
5
|
+
exports.categorizeTweetResult = categorizeTweetResult;
|
|
6
|
+
exports.parseTimeline = parseTimeline;
|
|
7
|
+
exports.fetchUserTimeline = fetchUserTimeline;
|
|
8
|
+
exports.parseConnections = parseConnections;
|
|
9
|
+
exports.fetchUserConnections = fetchUserConnections;
|
|
10
|
+
const tls_client_1 = require("../utils/tls-client");
|
|
11
|
+
const twitter_1 = require("./twitter");
|
|
12
|
+
// x.com 网页端公开 bearer(与 twitter.ts 同源固定值)
|
|
13
|
+
const WEB_BEARER = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
|
|
14
|
+
/** 默认 queryId 清单(漂移时可用 queryIds 参数整体覆盖) */
|
|
15
|
+
exports.DEFAULT_QUERY_IDS = {
|
|
16
|
+
UserByScreenName: 'sLVLhk0bGj3MVFEKTdax1w',
|
|
17
|
+
UserTweets: 'HuTx74BxAnezK1gWvYY7zg',
|
|
18
|
+
UserTweetsAndReplies: 'RIWc55YCNyUJ-U3HHGYkdg',
|
|
19
|
+
Likes: 'nXEl0lfN_XSznVMlprThgQ',
|
|
20
|
+
Followers: 'pd8Tt1qUz1YWrICegqZ8cw',
|
|
21
|
+
Following: 'wjvx62Hye2dGVvnvVco0xA',
|
|
22
|
+
};
|
|
23
|
+
const TIMELINE_FEATURES = {
|
|
24
|
+
blue_business_profile_image_shape_enabled: true,
|
|
25
|
+
creator_subscriptions_tweet_preview_api_enabled: true,
|
|
26
|
+
freedom_of_speech_not_reach_fetch_enabled: true,
|
|
27
|
+
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
|
|
28
|
+
graphql_timeline_v2_bookmark_timeline: true,
|
|
29
|
+
hidden_profile_likes_enabled: true,
|
|
30
|
+
highlights_tweets_tab_ui_enabled: true,
|
|
31
|
+
interactive_text_enabled: true,
|
|
32
|
+
longform_notetweets_consumption_enabled: true,
|
|
33
|
+
longform_notetweets_inline_media_enabled: true,
|
|
34
|
+
longform_notetweets_rich_text_read_enabled: true,
|
|
35
|
+
longform_notetweets_richtext_consumption_enabled: true,
|
|
36
|
+
profile_foundations_tweet_stats_enabled: true,
|
|
37
|
+
profile_foundations_tweet_stats_tweet_frequency: true,
|
|
38
|
+
responsive_web_birdwatch_note_limit_enabled: true,
|
|
39
|
+
responsive_web_edit_tweet_api_enabled: true,
|
|
40
|
+
responsive_web_enhance_cards_enabled: false,
|
|
41
|
+
responsive_web_graphql_exclude_directive_enabled: true,
|
|
42
|
+
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
|
|
43
|
+
responsive_web_graphql_timeline_navigation_enabled: true,
|
|
44
|
+
responsive_web_media_download_video_enabled: false,
|
|
45
|
+
responsive_web_text_conversations_enabled: false,
|
|
46
|
+
responsive_web_twitter_article_data_v2_enabled: true,
|
|
47
|
+
responsive_web_twitter_article_tweet_consumption_enabled: false,
|
|
48
|
+
responsive_web_twitter_blue_verified_badge_is_enabled: true,
|
|
49
|
+
rweb_lists_timeline_redesign_enabled: true,
|
|
50
|
+
spaces_2022_h2_clipping: true,
|
|
51
|
+
spaces_2022_h2_spaces_communities: true,
|
|
52
|
+
standardized_nudges_misinfo: true,
|
|
53
|
+
subscriptions_verification_info_verified_since_enabled: true,
|
|
54
|
+
tweet_awards_web_tipping_enabled: false,
|
|
55
|
+
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
|
|
56
|
+
tweetypie_unmention_optimization_enabled: true,
|
|
57
|
+
verified_phone_label_enabled: false,
|
|
58
|
+
vibe_api_enabled: true,
|
|
59
|
+
view_counts_everywhere_api_enabled: true,
|
|
60
|
+
};
|
|
61
|
+
async function gql(opName, queryId, variables, creds, get) {
|
|
62
|
+
const url = `https://x.com/i/api/graphql/${queryId}/${opName}` +
|
|
63
|
+
`?variables=${encodeURIComponent(JSON.stringify(variables))}&features=${encodeURIComponent(JSON.stringify(TIMELINE_FEATURES))}`;
|
|
64
|
+
let res;
|
|
65
|
+
try {
|
|
66
|
+
res = await get(url, {
|
|
67
|
+
headers: {
|
|
68
|
+
authorization: `Bearer ${WEB_BEARER}`,
|
|
69
|
+
'x-csrf-token': creds.ct0,
|
|
70
|
+
'x-twitter-auth-type': 'OAuth2Session',
|
|
71
|
+
'x-twitter-active-user': 'yes',
|
|
72
|
+
'x-twitter-client-language': 'en',
|
|
73
|
+
},
|
|
74
|
+
cookies: { auth_token: creds.authToken, ct0: creds.ct0 },
|
|
75
|
+
timeout: 30000,
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
catch (e) {
|
|
79
|
+
throw new Error(`X GraphQL 请求失败:${e?.message || e}`);
|
|
80
|
+
}
|
|
81
|
+
if (res.status === 403 || res.status === 429) {
|
|
82
|
+
throw new Error(`X GraphQL 被 Cloudflare 拦截 (HTTP ${res.status}):TLS 指纹校验未通过。` +
|
|
83
|
+
`请确认已安装可选依赖 @char46/tlsget-rs(npm i @char46/tlsget-rs),它提供浏览器级 TLS 指纹。`);
|
|
84
|
+
}
|
|
85
|
+
if (res.status !== 200) {
|
|
86
|
+
throw new Error(`X GraphQL ${opName} 返回 HTTP ${res.status}:${typeof res.data === 'string' ? res.data.slice(0, 120) : JSON.stringify(res.data || {}).slice(0, 120)}`);
|
|
87
|
+
}
|
|
88
|
+
return res.data;
|
|
89
|
+
}
|
|
90
|
+
async function resolveTwitterUser(screenName, creds, get = tls_client_1.tlsGet, queryIds = {}) {
|
|
91
|
+
const ids = { ...exports.DEFAULT_QUERY_IDS, ...queryIds };
|
|
92
|
+
const variables = {
|
|
93
|
+
screen_name: screenName.replace(/^@/, ''),
|
|
94
|
+
withSafetyModeUserFields: true,
|
|
95
|
+
withSuperFollowsUserFields: true,
|
|
96
|
+
};
|
|
97
|
+
const data = await gql('UserByScreenName', ids.UserByScreenName, variables, creds, get);
|
|
98
|
+
const user = data?.data?.user?.result;
|
|
99
|
+
if (!user || user.__typename === 'UserUnavailable') {
|
|
100
|
+
throw new Error(`用户 @${screenName} 不可访问(不存在、被封或需登录)`);
|
|
101
|
+
}
|
|
102
|
+
const legacy = user.legacy || user.core || {};
|
|
103
|
+
return {
|
|
104
|
+
userId: String(user.rest_id ?? legacy.id_str ?? ''),
|
|
105
|
+
name: String(legacy.name ?? ''),
|
|
106
|
+
screenName: String(legacy.screen_name ?? screenName.replace(/^@/, '')),
|
|
107
|
+
description: String(user.profile_bio?.description ?? legacy.description ?? ''),
|
|
108
|
+
followers: Number(legacy.followers_count ?? 0) || 0,
|
|
109
|
+
friends: Number(legacy.friends_count ?? 0) || 0,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
/** 纯函数:把单条 tweet result 归类(含转推解包) */
|
|
113
|
+
function categorizeTweetResult(result) {
|
|
114
|
+
const raw = (0, twitter_1.unwrapTweetResult)(result);
|
|
115
|
+
const rtInner = raw?.legacy?.retweeted_status_result?.result;
|
|
116
|
+
const inner = rtInner ? (0, twitter_1.unwrapTweetResult)(rtInner) : raw;
|
|
117
|
+
const legacy = inner?.legacy || {};
|
|
118
|
+
const media = Array.isArray(legacy.extended_entities?.media) ? legacy.extended_entities.media : [];
|
|
119
|
+
const hasImage = media.some((m) => m?.type === 'photo');
|
|
120
|
+
const hasVideo = media.some((m) => m?.type === 'video' || m?.type === 'animated_gif');
|
|
121
|
+
const isRetweet = !!rtInner;
|
|
122
|
+
const replyToId = typeof legacy.in_reply_to_status_id_str === 'string' ? legacy.in_reply_to_status_id_str : undefined;
|
|
123
|
+
const isReply = !!replyToId;
|
|
124
|
+
const isText = !hasImage && !hasVideo && !isRetweet;
|
|
125
|
+
return {
|
|
126
|
+
inner,
|
|
127
|
+
isRetweet,
|
|
128
|
+
isReply,
|
|
129
|
+
hasVideo,
|
|
130
|
+
hasImage,
|
|
131
|
+
isText,
|
|
132
|
+
id: String(inner?.rest_id ?? legacy.id_str ?? ''),
|
|
133
|
+
replyToId,
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
/** 纯函数:从 timeline instructions 抽取推文结果与底部游标 */
|
|
137
|
+
function parseTimeline(instructions) {
|
|
138
|
+
const results = [];
|
|
139
|
+
let bottomCursor;
|
|
140
|
+
for (const ins of instructions || []) {
|
|
141
|
+
const entries = ins?.entries || ins?.entry ? [].concat(ins.entry, ins.entries).filter(Boolean) : [];
|
|
142
|
+
for (const entry of entries) {
|
|
143
|
+
const content = entry?.content;
|
|
144
|
+
if (!content)
|
|
145
|
+
continue;
|
|
146
|
+
if (content.entryType === 'TimelineTimelineCursor' && content.contentType === 'Bottom') {
|
|
147
|
+
bottomCursor = String(content.value || bottomCursor || '');
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
if (content.entryType === 'TimelineTimelineItem' && content.itemContent?.tweet_results?.result) {
|
|
151
|
+
results.push(content.itemContent.tweet_results.result);
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
if (content.entryType === 'TimelineTimelineModule') {
|
|
155
|
+
for (const item of content.items || []) {
|
|
156
|
+
const r = item?.itemContent?.tweet_results?.result;
|
|
157
|
+
if (r)
|
|
158
|
+
results.push(r);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
return { results, bottomCursor };
|
|
164
|
+
}
|
|
165
|
+
function timelineOf(data) {
|
|
166
|
+
const user = data?.data?.user?.result;
|
|
167
|
+
const timeline = user?.timeline_v2?.timeline || user?.timeline?.timeline;
|
|
168
|
+
return timeline?.instructions || [];
|
|
169
|
+
}
|
|
170
|
+
/** 用户时间线(tab:tweets=推文/转推;replies=含回复;likes=当前登录用户的点赞) */
|
|
171
|
+
async function fetchUserTimeline(opts) {
|
|
172
|
+
const tab = opts.tab || 'tweets';
|
|
173
|
+
const limit = Math.max(1, Math.min(200, opts.limit ?? 20));
|
|
174
|
+
const get = opts.get || tls_client_1.tlsGet;
|
|
175
|
+
const ids = { ...exports.DEFAULT_QUERY_IDS, ...opts.queryIds };
|
|
176
|
+
let userId = opts.userId;
|
|
177
|
+
if (!userId) {
|
|
178
|
+
const u = await resolveTwitterUser(opts.screenName, opts.creds, get, opts.queryIds);
|
|
179
|
+
userId = u.userId;
|
|
180
|
+
}
|
|
181
|
+
const [opName, queryId] = tab === 'likes'
|
|
182
|
+
? ['Likes', ids.Likes]
|
|
183
|
+
: tab === 'replies'
|
|
184
|
+
? ['UserTweetsAndReplies', ids.UserTweetsAndReplies]
|
|
185
|
+
: ['UserTweets', ids.UserTweets];
|
|
186
|
+
const out = [];
|
|
187
|
+
const seen = new Set();
|
|
188
|
+
let cursor;
|
|
189
|
+
let pages = 0;
|
|
190
|
+
while (out.length < limit && pages < 20) {
|
|
191
|
+
pages++;
|
|
192
|
+
const variables = {
|
|
193
|
+
userId,
|
|
194
|
+
count: 20,
|
|
195
|
+
includePromotedContent: false,
|
|
196
|
+
withQuickPromoteEligibilityTweetQuery: false,
|
|
197
|
+
withVoice: true,
|
|
198
|
+
withV2Timeline: true,
|
|
199
|
+
};
|
|
200
|
+
if (cursor)
|
|
201
|
+
variables.cursor = cursor;
|
|
202
|
+
const data = await gql(opName, queryId, variables, opts.creds, get);
|
|
203
|
+
const { results, bottomCursor } = parseTimeline(timelineOf(data));
|
|
204
|
+
for (const r of results) {
|
|
205
|
+
const cat = categorizeTweetResult(r);
|
|
206
|
+
if (!cat.id || seen.has(cat.id))
|
|
207
|
+
continue;
|
|
208
|
+
seen.add(cat.id);
|
|
209
|
+
const kinds = [];
|
|
210
|
+
if (cat.isRetweet)
|
|
211
|
+
kinds.push('retweet');
|
|
212
|
+
if (cat.isReply)
|
|
213
|
+
kinds.push('reply');
|
|
214
|
+
if (cat.hasVideo)
|
|
215
|
+
kinds.push('video');
|
|
216
|
+
else if (cat.hasImage)
|
|
217
|
+
kinds.push('image');
|
|
218
|
+
else if (!cat.isRetweet)
|
|
219
|
+
kinds.push('text');
|
|
220
|
+
out.push({
|
|
221
|
+
id: cat.id,
|
|
222
|
+
tweet: (0, twitter_1.mapGraphql)(cat.inner),
|
|
223
|
+
kinds,
|
|
224
|
+
isRetweet: cat.isRetweet,
|
|
225
|
+
isReply: cat.isReply,
|
|
226
|
+
replyToId: cat.replyToId,
|
|
227
|
+
url: `https://x.com/i/web/status/${cat.id}`,
|
|
228
|
+
});
|
|
229
|
+
if (out.length >= limit)
|
|
230
|
+
break;
|
|
231
|
+
}
|
|
232
|
+
if (!bottomCursor || !results.length)
|
|
233
|
+
break;
|
|
234
|
+
cursor = bottomCursor;
|
|
235
|
+
}
|
|
236
|
+
return out;
|
|
237
|
+
}
|
|
238
|
+
function userOfEntry(itemContent) {
|
|
239
|
+
const user = itemContent?.user_results?.result;
|
|
240
|
+
if (!user || user.__typename === 'UserUnavailable')
|
|
241
|
+
return null;
|
|
242
|
+
const legacy = user.legacy || {};
|
|
243
|
+
return {
|
|
244
|
+
name: String(legacy.name ?? user.core?.name ?? ''),
|
|
245
|
+
screenName: String(legacy.screen_name ?? user.core?.screen_name ?? ''),
|
|
246
|
+
description: String(user.profile_bio?.description ?? legacy.description ?? ''),
|
|
247
|
+
followers: Number(legacy.followers_count ?? 0) || 0,
|
|
248
|
+
verified: !!user.is_blue_verified || !!legacy.verified,
|
|
249
|
+
followedBy: !!legacy.followed_by,
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
/** 纯函数:从 Followers/Following instructions 抽取用户与底部游标 */
|
|
253
|
+
function parseConnections(instructions) {
|
|
254
|
+
const users = [];
|
|
255
|
+
let bottomCursor;
|
|
256
|
+
for (const ins of instructions || []) {
|
|
257
|
+
for (const entry of ins?.entries || []) {
|
|
258
|
+
const content = entry?.content;
|
|
259
|
+
if (!content)
|
|
260
|
+
continue;
|
|
261
|
+
if (content.entryType === 'TimelineTimelineCursor' && content.contentType === 'Bottom') {
|
|
262
|
+
bottomCursor = String(content.value || bottomCursor || '');
|
|
263
|
+
continue;
|
|
264
|
+
}
|
|
265
|
+
if (content.entryType === 'TimelineTimelineItem' && content.itemContent) {
|
|
266
|
+
const u = userOfEntry(content.itemContent);
|
|
267
|
+
if (u && u.screenName)
|
|
268
|
+
users.push(u);
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
return { users, bottomCursor };
|
|
273
|
+
}
|
|
274
|
+
/** 关注列表(following)或粉丝列表(followers) */
|
|
275
|
+
async function fetchUserConnections(opts) {
|
|
276
|
+
const limit = Math.max(1, Math.min(1000, opts.limit ?? 50));
|
|
277
|
+
const get = opts.get || tls_client_1.tlsGet;
|
|
278
|
+
const ids = { ...exports.DEFAULT_QUERY_IDS, ...opts.queryIds };
|
|
279
|
+
const u = await resolveTwitterUser(opts.screenName, opts.creds, get, opts.queryIds);
|
|
280
|
+
const [opName, queryId] = opts.type === 'following'
|
|
281
|
+
? ['Following', ids.Following]
|
|
282
|
+
: ['Followers', ids.Followers];
|
|
283
|
+
const out = [];
|
|
284
|
+
const seen = new Set();
|
|
285
|
+
let cursor;
|
|
286
|
+
let pages = 0;
|
|
287
|
+
while (out.length < limit && pages < 50) {
|
|
288
|
+
pages++;
|
|
289
|
+
const variables = { userId: u.userId, count: 50, includePromotedContent: false };
|
|
290
|
+
if (cursor)
|
|
291
|
+
variables.cursor = cursor;
|
|
292
|
+
const data = await gql(opName, queryId, variables, opts.creds, get);
|
|
293
|
+
const { users, bottomCursor } = parseConnections(timelineOf(data));
|
|
294
|
+
for (const user of users) {
|
|
295
|
+
if (seen.has(user.screenName))
|
|
296
|
+
continue;
|
|
297
|
+
seen.add(user.screenName);
|
|
298
|
+
out.push(user);
|
|
299
|
+
if (out.length >= limit)
|
|
300
|
+
break;
|
|
301
|
+
}
|
|
302
|
+
if (!bottomCursor || !users.length)
|
|
303
|
+
break;
|
|
304
|
+
cursor = bottomCursor;
|
|
305
|
+
}
|
|
306
|
+
return out;
|
|
307
|
+
}
|
package/lib/engine/twitter.d.ts
CHANGED
|
@@ -23,4 +23,29 @@ export interface GrokTranslation {
|
|
|
23
23
|
* x-client-transaction-id / cf_clearance。任何失败返回 null 由调用方回落通用翻译。
|
|
24
24
|
*/
|
|
25
25
|
export declare function fetchGrokTranslation(url: string, targetLang: string, creds: TwitterCreds, get?: GraphqlGetter): Promise<GrokTranslation | null>;
|
|
26
|
+
export declare function extractTweetId(url: string): string | null;
|
|
27
|
+
/** 把 GraphQL TweetResultByRestId 响应映射为 ParsedData */
|
|
28
|
+
export declare function mapGraphql(rawResult: any): ParsedData;
|
|
29
|
+
/** 解除 TweetWithVisibilityResults 包裹(NSFW/受限推文) */
|
|
30
|
+
export declare function unwrapTweetResult(result: any): any;
|
|
31
|
+
/** 鉴权 GraphQL 原始获取:返回未映射的 tweet result 节点(树构建复用) */
|
|
32
|
+
export declare function fetchGraphqlRaw(id: string, creds: TwitterCreds, get: GraphqlGetter): Promise<any>;
|
|
26
33
|
export declare function parseTwitter(url: string, http: AxiosInstance, creds?: TwitterCreds, getGraphql?: GraphqlGetter): Promise<ParsedData>;
|
|
34
|
+
/** 推文树节点:quoted = 本推引用的推文(递归);replyTo = 本推回复的目标(向根方向递归) */
|
|
35
|
+
export interface TweetTree {
|
|
36
|
+
id: string;
|
|
37
|
+
tweet: ParsedData;
|
|
38
|
+
quoted?: TweetTree;
|
|
39
|
+
replyTo?: TweetTree;
|
|
40
|
+
}
|
|
41
|
+
/** 游客态(syndication)取被引用推文 ID:显式字段优先;否则启发式——引用链的
|
|
42
|
+
* t.co 短链由 X 追加在正文末尾,expanded_url 为推文永久链。 */
|
|
43
|
+
export declare function guestQuoteId(tw: any): string | undefined;
|
|
44
|
+
/** 游客态(syndication)取回复目标推文 ID */
|
|
45
|
+
export declare function guestReplyToId(tw: any): string | undefined;
|
|
46
|
+
/**
|
|
47
|
+
* 拉取推文树:引用链(quoted,向下递归)+ 回复链(replyTo,向根递归)。
|
|
48
|
+
* - 有登录态:GraphQL 单次响应自带嵌套 quoted_status_result;父级逐次拉取
|
|
49
|
+
* - 无登录态:syndication 逐节点拉取(引用 ID 走启发式);不可达分支静默截断
|
|
50
|
+
*/
|
|
51
|
+
export declare function fetchTweetTree(url: string, http: AxiosInstance, creds?: TwitterCreds, getGraphql?: GraphqlGetter): Promise<TweetTree>;
|
package/lib/engine/twitter.js
CHANGED
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.fetchGrokTranslation = fetchGrokTranslation;
|
|
4
|
+
exports.extractTweetId = extractTweetId;
|
|
5
|
+
exports.mapGraphql = mapGraphql;
|
|
6
|
+
exports.unwrapTweetResult = unwrapTweetResult;
|
|
7
|
+
exports.fetchGraphqlRaw = fetchGraphqlRaw;
|
|
4
8
|
exports.parseTwitter = parseTwitter;
|
|
9
|
+
exports.guestQuoteId = guestQuoteId;
|
|
10
|
+
exports.guestReplyToId = guestReplyToId;
|
|
11
|
+
exports.fetchTweetTree = fetchTweetTree;
|
|
5
12
|
const tls_client_1 = require("../utils/tls-client");
|
|
6
13
|
/**
|
|
7
14
|
* X / Twitter 原生解析器。
|
|
@@ -370,8 +377,12 @@ function mapGraphql(rawResult) {
|
|
|
370
377
|
p.title = '';
|
|
371
378
|
return p;
|
|
372
379
|
}
|
|
373
|
-
/**
|
|
374
|
-
|
|
380
|
+
/** 解除 TweetWithVisibilityResults 包裹(NSFW/受限推文) */
|
|
381
|
+
function unwrapTweetResult(result) {
|
|
382
|
+
return result && result.__typename === 'TweetWithVisibilityResults' && result.tweet ? result.tweet : result;
|
|
383
|
+
}
|
|
384
|
+
/** 鉴权 GraphQL 原始获取:返回未映射的 tweet result 节点(树构建复用) */
|
|
385
|
+
async function fetchGraphqlRaw(id, creds, get) {
|
|
375
386
|
const variables = { tweetId: id, includePromotedContent: true, withBirdwatchNotes: true, withVoice: true, withCommunity: true };
|
|
376
387
|
const url = `https://x.com/i/api/graphql/${TWEET_RESULT_QUERY_ID}/TweetResultByRestId` +
|
|
377
388
|
`?variables=${encodeURIComponent(JSON.stringify(variables))}&features=${encodeURIComponent(JSON.stringify(GRAPHQL_FEATURES))}`;
|
|
@@ -406,7 +417,11 @@ async function fetchGraphqlTweet(id, creds, get) {
|
|
|
406
417
|
const tb = result?.tombstone?.text?.text || result?.tombstone?.text;
|
|
407
418
|
throw new Error(`推文不可访问(可能需要登录、已被删除或为非公开内容)${tb ? ':' + tb : ''}`);
|
|
408
419
|
}
|
|
409
|
-
return
|
|
420
|
+
return unwrapTweetResult(result);
|
|
421
|
+
}
|
|
422
|
+
/** 鉴权 GraphQL:仅用 auth_token + ct0,回退取登录受限推文 */
|
|
423
|
+
async function fetchGraphqlTweet(id, creds, get) {
|
|
424
|
+
return mapGraphql(await fetchGraphqlRaw(id, creds, get));
|
|
410
425
|
}
|
|
411
426
|
async function parseTwitter(url, http, creds, getGraphql) {
|
|
412
427
|
const id = extractTweetId(url);
|
|
@@ -441,3 +456,113 @@ async function parseTwitter(url, http, creds, getGraphql) {
|
|
|
441
456
|
const reasonRaw = pick(tw?.tombstone?.text, tw?.tombstone?.name);
|
|
442
457
|
throw new Error(`推文不可访问(可能需要登录、已被删除或为非公开内容)${reasonRaw ? ':' + reasonRaw : ''}`);
|
|
443
458
|
}
|
|
459
|
+
const TREE_QUOTE_DEPTH = 6;
|
|
460
|
+
const TREE_REPLY_DEPTH = 12;
|
|
461
|
+
/** 游客态(syndication)取被引用推文 ID:显式字段优先;否则启发式——引用链的
|
|
462
|
+
* t.co 短链由 X 追加在正文末尾,expanded_url 为推文永久链。 */
|
|
463
|
+
function guestQuoteId(tw) {
|
|
464
|
+
const q = tw?.quoted_tweet || tw?.quoted_status_result;
|
|
465
|
+
if (q) {
|
|
466
|
+
const id = String(pick(q.id_str, q.rest_id, ''));
|
|
467
|
+
if (/^\d+$/.test(id))
|
|
468
|
+
return id;
|
|
469
|
+
}
|
|
470
|
+
const urls = Array.isArray(tw?.entities?.urls) ? tw.entities.urls : [];
|
|
471
|
+
const rawText = String(tw?.text || '');
|
|
472
|
+
const textEnd = rawText.trimEnd();
|
|
473
|
+
for (const u of urls) {
|
|
474
|
+
if (!u?.url || !textEnd.endsWith(u.url))
|
|
475
|
+
continue;
|
|
476
|
+
const m = /(?:x|twitter)\.com\/[^/]+\/status(?:es)?\/(\d+)/.exec(String(u.expanded_url || ''));
|
|
477
|
+
if (m)
|
|
478
|
+
return m[1];
|
|
479
|
+
}
|
|
480
|
+
return undefined;
|
|
481
|
+
}
|
|
482
|
+
/** 游客态(syndication)取回复目标推文 ID */
|
|
483
|
+
function guestReplyToId(tw) {
|
|
484
|
+
const id = pick(tw?.in_reply_to_status_id_str, tw?.in_reply_to_status_id);
|
|
485
|
+
if (typeof id === 'string' && /^\d+$/.test(id))
|
|
486
|
+
return id;
|
|
487
|
+
if (typeof id === 'number')
|
|
488
|
+
return String(id);
|
|
489
|
+
return undefined;
|
|
490
|
+
}
|
|
491
|
+
async function fetchSyndicationRaw(tweetId, http) {
|
|
492
|
+
const res = await http.get(SYNDICATION_URL, {
|
|
493
|
+
params: { id: tweetId, token: 'a' },
|
|
494
|
+
timeout: 30000,
|
|
495
|
+
headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' },
|
|
496
|
+
});
|
|
497
|
+
const tw = res.data;
|
|
498
|
+
return tw && tw.__typename === 'Tweet' && tw.user ? tw : null;
|
|
499
|
+
}
|
|
500
|
+
/**
|
|
501
|
+
* 拉取推文树:引用链(quoted,向下递归)+ 回复链(replyTo,向根递归)。
|
|
502
|
+
* - 有登录态:GraphQL 单次响应自带嵌套 quoted_status_result;父级逐次拉取
|
|
503
|
+
* - 无登录态:syndication 逐节点拉取(引用 ID 走启发式);不可达分支静默截断
|
|
504
|
+
*/
|
|
505
|
+
async function fetchTweetTree(url, http, creds, getGraphql) {
|
|
506
|
+
const id = extractTweetId(url);
|
|
507
|
+
if (!id)
|
|
508
|
+
throw new Error('无法从 X 链接提取推文 ID');
|
|
509
|
+
const visited = new Set();
|
|
510
|
+
async function buildGuest(tweetId, depth) {
|
|
511
|
+
visited.add(tweetId);
|
|
512
|
+
const tw = await fetchSyndicationRaw(tweetId, http);
|
|
513
|
+
if (!tw)
|
|
514
|
+
throw new Error(`推文 ${tweetId} 不可访问(可能需要登录或已删除)`);
|
|
515
|
+
const node = { id: tweetId, tweet: mapSyndication(tw) };
|
|
516
|
+
const qid = guestQuoteId(tw);
|
|
517
|
+
if (qid && !visited.has(qid) && depth < TREE_QUOTE_DEPTH) {
|
|
518
|
+
try {
|
|
519
|
+
node.quoted = await buildGuest(qid, depth + 1);
|
|
520
|
+
}
|
|
521
|
+
catch { /* 引用不可达则截断 */ }
|
|
522
|
+
}
|
|
523
|
+
const rid = guestReplyToId(tw);
|
|
524
|
+
if (rid && !visited.has(rid) && depth < TREE_REPLY_DEPTH) {
|
|
525
|
+
try {
|
|
526
|
+
node.replyTo = await buildGuest(rid, depth + 1);
|
|
527
|
+
}
|
|
528
|
+
catch { /* 父推不可达则截断 */ }
|
|
529
|
+
}
|
|
530
|
+
return node;
|
|
531
|
+
}
|
|
532
|
+
async function buildFromGraphqlResult(raw, depth) {
|
|
533
|
+
raw = unwrapTweetResult(raw);
|
|
534
|
+
const rid0 = String(pick(raw?.rest_id, raw?.legacy?.id_str, ''));
|
|
535
|
+
if (rid0)
|
|
536
|
+
visited.add(rid0);
|
|
537
|
+
const node = { id: rid0, tweet: mapGraphql(raw) };
|
|
538
|
+
const q = unwrapTweetResult(raw?.quoted_status_result?.result);
|
|
539
|
+
if (q && q.legacy && q.__typename !== 'TweetTombstone' && depth < TREE_QUOTE_DEPTH) {
|
|
540
|
+
const qid = String(pick(q.rest_id, q.legacy?.id_str, ''));
|
|
541
|
+
if (!qid || !visited.has(qid))
|
|
542
|
+
node.quoted = await buildFromGraphqlResult(q, depth + 1);
|
|
543
|
+
}
|
|
544
|
+
const rid = raw?.legacy?.in_reply_to_status_id_str;
|
|
545
|
+
if (typeof rid === 'string' && /^\d+$/.test(rid) && !visited.has(rid) && depth < TREE_REPLY_DEPTH) {
|
|
546
|
+
try {
|
|
547
|
+
node.replyTo = await buildGraphqlById(rid, depth + 1);
|
|
548
|
+
}
|
|
549
|
+
catch { /* 父推不可达则截断 */ }
|
|
550
|
+
}
|
|
551
|
+
return node;
|
|
552
|
+
}
|
|
553
|
+
async function buildGraphqlById(tweetId, depth) {
|
|
554
|
+
if (visited.has(tweetId))
|
|
555
|
+
throw new Error('引用环');
|
|
556
|
+
const raw = await fetchGraphqlRaw(tweetId, creds, getGraphql || tls_client_1.tlsGet);
|
|
557
|
+
return buildFromGraphqlResult(raw, depth);
|
|
558
|
+
}
|
|
559
|
+
if (creds && creds.authToken && creds.ct0) {
|
|
560
|
+
try {
|
|
561
|
+
return await buildGraphqlById(id, 0);
|
|
562
|
+
}
|
|
563
|
+
catch {
|
|
564
|
+
/* 登录态失败(如 Cloudflare)回退游客路径 */
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
return buildGuest(id, 0);
|
|
568
|
+
}
|
package/lib/index.d.ts
CHANGED
|
@@ -23,6 +23,7 @@ export * from './engine/config';
|
|
|
23
23
|
export * from './engine/platform-config';
|
|
24
24
|
export * from './engine/translate';
|
|
25
25
|
export * from './engine/twitter';
|
|
26
|
+
export * from './engine/twitter-user';
|
|
26
27
|
export * from './engine/parser';
|
|
27
28
|
export * from './engine/fetcher';
|
|
28
29
|
export * from './engine/runtime';
|
package/lib/index.js
CHANGED
|
@@ -39,6 +39,7 @@ __exportStar(require("./engine/config"), exports);
|
|
|
39
39
|
__exportStar(require("./engine/platform-config"), exports);
|
|
40
40
|
__exportStar(require("./engine/translate"), exports);
|
|
41
41
|
__exportStar(require("./engine/twitter"), exports);
|
|
42
|
+
__exportStar(require("./engine/twitter-user"), exports);
|
|
42
43
|
__exportStar(require("./engine/parser"), exports);
|
|
43
44
|
__exportStar(require("./engine/fetcher"), exports);
|
|
44
45
|
__exportStar(require("./engine/runtime"), exports);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sns-parse/core",
|
|
3
|
-
"version": "0.4.0-alpha.
|
|
3
|
+
"version": "0.4.0-alpha.7+upstream.1.6.7",
|
|
4
4
|
"description": "sns-parse 核心:契约、宿主抽象、配置 DSL 与解析引擎(fetcher/parser/翻译/X 原生解析/合并/GIF/TLS 指纹客户端)",
|
|
5
5
|
"main": "lib/index.js",
|
|
6
6
|
"types": "lib/index.d.ts",
|