@sns-parse/core 0.4.0-alpha.5 → 0.4.0-alpha.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/engine/config.js +2 -2
- package/lib/engine/twitter-user.d.ts +87 -0
- package/lib/engine/twitter-user.js +307 -0
- package/lib/engine/twitter.d.ts +25 -0
- package/lib/engine/twitter.js +134 -8
- package/lib/index.d.ts +1 -0
- package/lib/index.js +1 -0
- package/lib/utils/format.js +6 -1
- package/package.json +1 -1
package/lib/engine/config.js
CHANGED
|
@@ -104,8 +104,8 @@ function engineConfigContributions() {
|
|
|
104
104
|
fields: [
|
|
105
105
|
{
|
|
106
106
|
key: 'unifiedMessageFormat', type: 'string', role: 'textarea',
|
|
107
|
-
default: '标题:${标题}\n作者:${作者}\n简介:${简介}\n翻译(${翻译提供方},${原文语言}):${翻译}\n音乐标题:${音乐标题}\n音乐作者:${音乐作者}\n点赞:${点赞数}\n收藏:${收藏数}\n转发:${转发数}\n播放:${播放数}\n评论:${评论数}\n图片数量:${图片数量}',
|
|
108
|
-
description: '消息文字模板,可用变量:${标题} ${作者} ${简介} ${翻译} ${翻译提供方} ${原文语言} ${视频时长} ${点赞数} ${收藏数} ${转发数} ${播放数} ${评论数} ${发布时间} ${图片数量} ${作者ID} ${音乐标题} ${音乐作者}(空行自动隐藏)',
|
|
107
|
+
default: '标题:${标题}\n作者:${作者}\n简介:${简介}\n正文:${正文}\n翻译(${翻译提供方},${原文语言}):${翻译}\n音乐标题:${音乐标题}\n音乐作者:${音乐作者}\n点赞:${点赞数}\n收藏:${收藏数}\n转发:${转发数}\n播放:${播放数}\n评论:${评论数}\n图片数量:${图片数量}',
|
|
108
|
+
description: '消息文字模板,可用变量:${标题} ${作者} ${简介} ${正文}(纯文字作品正文,启用后该类作品不再显示为简介) ${翻译} ${翻译提供方} ${原文语言} ${视频时长} ${点赞数} ${收藏数} ${转发数} ${播放数} ${评论数} ${发布时间} ${图片数量} ${作者ID} ${音乐标题} ${音乐作者}(空行自动隐藏)',
|
|
109
109
|
},
|
|
110
110
|
],
|
|
111
111
|
}),
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* X 用户维度查询:时间线(推文/回复/点赞)与关注/粉丝列表。
|
|
3
|
+
*
|
|
4
|
+
* 全部走鉴权 GraphQL(auth_token + ct0,TLS 指纹由 tlsget-rs 提供)。
|
|
5
|
+
* queryId 随 x.com web 版本漂移:默认值取自活跃维护的开源清单
|
|
6
|
+
* (twitter-api-client),并支持调用方整体覆盖(queryIds 参数)。
|
|
7
|
+
*/
|
|
8
|
+
import type { ParsedData } from '../types';
|
|
9
|
+
import { type GraphqlGetter, type TwitterCreds } from './twitter';
|
|
10
|
+
export interface TwitterQueryIds {
|
|
11
|
+
UserByScreenName?: string;
|
|
12
|
+
UserTweets?: string;
|
|
13
|
+
UserTweetsAndReplies?: string;
|
|
14
|
+
Likes?: string;
|
|
15
|
+
Followers?: string;
|
|
16
|
+
Following?: string;
|
|
17
|
+
}
|
|
18
|
+
/** 默认 queryId 清单(漂移时可用 queryIds 参数整体覆盖) */
|
|
19
|
+
export declare const DEFAULT_QUERY_IDS: Required<TwitterQueryIds>;
|
|
20
|
+
export interface TwitterUserInfo {
|
|
21
|
+
userId: string;
|
|
22
|
+
name: string;
|
|
23
|
+
screenName: string;
|
|
24
|
+
description: string;
|
|
25
|
+
followers: number;
|
|
26
|
+
friends: number;
|
|
27
|
+
}
|
|
28
|
+
export declare function resolveTwitterUser(screenName: string, creds: TwitterCreds, get?: GraphqlGetter, queryIds?: TwitterQueryIds): Promise<TwitterUserInfo>;
|
|
29
|
+
export type TimelineTab = 'tweets' | 'replies' | 'likes';
|
|
30
|
+
export interface TimelineEntry {
|
|
31
|
+
id: string;
|
|
32
|
+
tweet: ParsedData;
|
|
33
|
+
/** 文字/视频/图片/转推/回复 中的若干类 */
|
|
34
|
+
kinds: ('text' | 'video' | 'image' | 'retweet' | 'reply')[];
|
|
35
|
+
isRetweet: boolean;
|
|
36
|
+
isReply: boolean;
|
|
37
|
+
replyToId?: string;
|
|
38
|
+
url: string;
|
|
39
|
+
}
|
|
40
|
+
/** 纯函数:把单条 tweet result 归类(含转推解包) */
|
|
41
|
+
export declare function categorizeTweetResult(result: any): {
|
|
42
|
+
inner: any;
|
|
43
|
+
isRetweet: boolean;
|
|
44
|
+
isReply: boolean;
|
|
45
|
+
hasVideo: boolean;
|
|
46
|
+
hasImage: boolean;
|
|
47
|
+
isText: boolean;
|
|
48
|
+
id: string;
|
|
49
|
+
replyToId?: string;
|
|
50
|
+
};
|
|
51
|
+
/** 纯函数:从 timeline instructions 抽取推文结果与底部游标 */
|
|
52
|
+
export declare function parseTimeline(instructions: any[]): {
|
|
53
|
+
results: any[];
|
|
54
|
+
bottomCursor?: string;
|
|
55
|
+
};
|
|
56
|
+
/** 用户时间线(tab:tweets=推文/转推;replies=含回复;likes=当前登录用户的点赞) */
|
|
57
|
+
export declare function fetchUserTimeline(opts: {
|
|
58
|
+
screenName?: string;
|
|
59
|
+
userId?: string;
|
|
60
|
+
tab?: TimelineTab;
|
|
61
|
+
limit?: number;
|
|
62
|
+
creds: TwitterCreds;
|
|
63
|
+
get?: GraphqlGetter;
|
|
64
|
+
queryIds?: TwitterQueryIds;
|
|
65
|
+
}): Promise<TimelineEntry[]>;
|
|
66
|
+
export interface TwitterConnectionUser {
|
|
67
|
+
name: string;
|
|
68
|
+
screenName: string;
|
|
69
|
+
description: string;
|
|
70
|
+
followers: number;
|
|
71
|
+
verified: boolean;
|
|
72
|
+
followedBy: boolean;
|
|
73
|
+
}
|
|
74
|
+
/** 纯函数:从 Followers/Following instructions 抽取用户与底部游标 */
|
|
75
|
+
export declare function parseConnections(instructions: any[]): {
|
|
76
|
+
users: TwitterConnectionUser[];
|
|
77
|
+
bottomCursor?: string;
|
|
78
|
+
};
|
|
79
|
+
/** 关注列表(following)或粉丝列表(followers) */
|
|
80
|
+
export declare function fetchUserConnections(opts: {
|
|
81
|
+
screenName: string;
|
|
82
|
+
type: 'followers' | 'following';
|
|
83
|
+
limit?: number;
|
|
84
|
+
creds: TwitterCreds;
|
|
85
|
+
get?: GraphqlGetter;
|
|
86
|
+
queryIds?: TwitterQueryIds;
|
|
87
|
+
}): Promise<TwitterConnectionUser[]>;
|
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DEFAULT_QUERY_IDS = void 0;
|
|
4
|
+
exports.resolveTwitterUser = resolveTwitterUser;
|
|
5
|
+
exports.categorizeTweetResult = categorizeTweetResult;
|
|
6
|
+
exports.parseTimeline = parseTimeline;
|
|
7
|
+
exports.fetchUserTimeline = fetchUserTimeline;
|
|
8
|
+
exports.parseConnections = parseConnections;
|
|
9
|
+
exports.fetchUserConnections = fetchUserConnections;
|
|
10
|
+
const tls_client_1 = require("../utils/tls-client");
|
|
11
|
+
const twitter_1 = require("./twitter");
|
|
12
|
+
// x.com 网页端公开 bearer(与 twitter.ts 同源固定值)
|
|
13
|
+
const WEB_BEARER = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
|
|
14
|
+
/** 默认 queryId 清单(漂移时可用 queryIds 参数整体覆盖) */
|
|
15
|
+
exports.DEFAULT_QUERY_IDS = {
|
|
16
|
+
UserByScreenName: 'sLVLhk0bGj3MVFEKTdax1w',
|
|
17
|
+
UserTweets: 'HuTx74BxAnezK1gWvYY7zg',
|
|
18
|
+
UserTweetsAndReplies: 'RIWc55YCNyUJ-U3HHGYkdg',
|
|
19
|
+
Likes: 'nXEl0lfN_XSznVMlprThgQ',
|
|
20
|
+
Followers: 'pd8Tt1qUz1YWrICegqZ8cw',
|
|
21
|
+
Following: 'wjvx62Hye2dGVvnvVco0xA',
|
|
22
|
+
};
|
|
23
|
+
const TIMELINE_FEATURES = {
|
|
24
|
+
blue_business_profile_image_shape_enabled: true,
|
|
25
|
+
creator_subscriptions_tweet_preview_api_enabled: true,
|
|
26
|
+
freedom_of_speech_not_reach_fetch_enabled: true,
|
|
27
|
+
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
|
|
28
|
+
graphql_timeline_v2_bookmark_timeline: true,
|
|
29
|
+
hidden_profile_likes_enabled: true,
|
|
30
|
+
highlights_tweets_tab_ui_enabled: true,
|
|
31
|
+
interactive_text_enabled: true,
|
|
32
|
+
longform_notetweets_consumption_enabled: true,
|
|
33
|
+
longform_notetweets_inline_media_enabled: true,
|
|
34
|
+
longform_notetweets_rich_text_read_enabled: true,
|
|
35
|
+
longform_notetweets_richtext_consumption_enabled: true,
|
|
36
|
+
profile_foundations_tweet_stats_enabled: true,
|
|
37
|
+
profile_foundations_tweet_stats_tweet_frequency: true,
|
|
38
|
+
responsive_web_birdwatch_note_limit_enabled: true,
|
|
39
|
+
responsive_web_edit_tweet_api_enabled: true,
|
|
40
|
+
responsive_web_enhance_cards_enabled: false,
|
|
41
|
+
responsive_web_graphql_exclude_directive_enabled: true,
|
|
42
|
+
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
|
|
43
|
+
responsive_web_graphql_timeline_navigation_enabled: true,
|
|
44
|
+
responsive_web_media_download_video_enabled: false,
|
|
45
|
+
responsive_web_text_conversations_enabled: false,
|
|
46
|
+
responsive_web_twitter_article_data_v2_enabled: true,
|
|
47
|
+
responsive_web_twitter_article_tweet_consumption_enabled: false,
|
|
48
|
+
responsive_web_twitter_blue_verified_badge_is_enabled: true,
|
|
49
|
+
rweb_lists_timeline_redesign_enabled: true,
|
|
50
|
+
spaces_2022_h2_clipping: true,
|
|
51
|
+
spaces_2022_h2_spaces_communities: true,
|
|
52
|
+
standardized_nudges_misinfo: true,
|
|
53
|
+
subscriptions_verification_info_verified_since_enabled: true,
|
|
54
|
+
tweet_awards_web_tipping_enabled: false,
|
|
55
|
+
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
|
|
56
|
+
tweetypie_unmention_optimization_enabled: true,
|
|
57
|
+
verified_phone_label_enabled: false,
|
|
58
|
+
vibe_api_enabled: true,
|
|
59
|
+
view_counts_everywhere_api_enabled: true,
|
|
60
|
+
};
|
|
61
|
+
async function gql(opName, queryId, variables, creds, get) {
|
|
62
|
+
const url = `https://x.com/i/api/graphql/${queryId}/${opName}` +
|
|
63
|
+
`?variables=${encodeURIComponent(JSON.stringify(variables))}&features=${encodeURIComponent(JSON.stringify(TIMELINE_FEATURES))}`;
|
|
64
|
+
let res;
|
|
65
|
+
try {
|
|
66
|
+
res = await get(url, {
|
|
67
|
+
headers: {
|
|
68
|
+
authorization: `Bearer ${WEB_BEARER}`,
|
|
69
|
+
'x-csrf-token': creds.ct0,
|
|
70
|
+
'x-twitter-auth-type': 'OAuth2Session',
|
|
71
|
+
'x-twitter-active-user': 'yes',
|
|
72
|
+
'x-twitter-client-language': 'en',
|
|
73
|
+
},
|
|
74
|
+
cookies: { auth_token: creds.authToken, ct0: creds.ct0 },
|
|
75
|
+
timeout: 30000,
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
catch (e) {
|
|
79
|
+
throw new Error(`X GraphQL 请求失败:${e?.message || e}`);
|
|
80
|
+
}
|
|
81
|
+
if (res.status === 403 || res.status === 429) {
|
|
82
|
+
throw new Error(`X GraphQL 被 Cloudflare 拦截 (HTTP ${res.status}):TLS 指纹校验未通过。` +
|
|
83
|
+
`请确认已安装可选依赖 @char46/tlsget-rs(npm i @char46/tlsget-rs),它提供浏览器级 TLS 指纹。`);
|
|
84
|
+
}
|
|
85
|
+
if (res.status !== 200) {
|
|
86
|
+
throw new Error(`X GraphQL ${opName} 返回 HTTP ${res.status}:${typeof res.data === 'string' ? res.data.slice(0, 120) : JSON.stringify(res.data || {}).slice(0, 120)}`);
|
|
87
|
+
}
|
|
88
|
+
return res.data;
|
|
89
|
+
}
|
|
90
|
+
async function resolveTwitterUser(screenName, creds, get = tls_client_1.tlsGet, queryIds = {}) {
|
|
91
|
+
const ids = { ...exports.DEFAULT_QUERY_IDS, ...queryIds };
|
|
92
|
+
const variables = {
|
|
93
|
+
screen_name: screenName.replace(/^@/, ''),
|
|
94
|
+
withSafetyModeUserFields: true,
|
|
95
|
+
withSuperFollowsUserFields: true,
|
|
96
|
+
};
|
|
97
|
+
const data = await gql('UserByScreenName', ids.UserByScreenName, variables, creds, get);
|
|
98
|
+
const user = data?.data?.user?.result;
|
|
99
|
+
if (!user || user.__typename === 'UserUnavailable') {
|
|
100
|
+
throw new Error(`用户 @${screenName} 不可访问(不存在、被封或需登录)`);
|
|
101
|
+
}
|
|
102
|
+
const legacy = user.legacy || user.core || {};
|
|
103
|
+
return {
|
|
104
|
+
userId: String(user.rest_id ?? legacy.id_str ?? ''),
|
|
105
|
+
name: String(legacy.name ?? ''),
|
|
106
|
+
screenName: String(legacy.screen_name ?? screenName.replace(/^@/, '')),
|
|
107
|
+
description: String(user.profile_bio?.description ?? legacy.description ?? ''),
|
|
108
|
+
followers: Number(legacy.followers_count ?? 0) || 0,
|
|
109
|
+
friends: Number(legacy.friends_count ?? 0) || 0,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
/** 纯函数:把单条 tweet result 归类(含转推解包) */
|
|
113
|
+
function categorizeTweetResult(result) {
|
|
114
|
+
const raw = (0, twitter_1.unwrapTweetResult)(result);
|
|
115
|
+
const rtInner = raw?.legacy?.retweeted_status_result?.result;
|
|
116
|
+
const inner = rtInner ? (0, twitter_1.unwrapTweetResult)(rtInner) : raw;
|
|
117
|
+
const legacy = inner?.legacy || {};
|
|
118
|
+
const media = Array.isArray(legacy.extended_entities?.media) ? legacy.extended_entities.media : [];
|
|
119
|
+
const hasImage = media.some((m) => m?.type === 'photo');
|
|
120
|
+
const hasVideo = media.some((m) => m?.type === 'video' || m?.type === 'animated_gif');
|
|
121
|
+
const isRetweet = !!rtInner;
|
|
122
|
+
const replyToId = typeof legacy.in_reply_to_status_id_str === 'string' ? legacy.in_reply_to_status_id_str : undefined;
|
|
123
|
+
const isReply = !!replyToId;
|
|
124
|
+
const isText = !hasImage && !hasVideo && !isRetweet;
|
|
125
|
+
return {
|
|
126
|
+
inner,
|
|
127
|
+
isRetweet,
|
|
128
|
+
isReply,
|
|
129
|
+
hasVideo,
|
|
130
|
+
hasImage,
|
|
131
|
+
isText,
|
|
132
|
+
id: String(inner?.rest_id ?? legacy.id_str ?? ''),
|
|
133
|
+
replyToId,
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
/** 纯函数:从 timeline instructions 抽取推文结果与底部游标 */
|
|
137
|
+
function parseTimeline(instructions) {
|
|
138
|
+
const results = [];
|
|
139
|
+
let bottomCursor;
|
|
140
|
+
for (const ins of instructions || []) {
|
|
141
|
+
const entries = ins?.entries || ins?.entry ? [].concat(ins.entry, ins.entries).filter(Boolean) : [];
|
|
142
|
+
for (const entry of entries) {
|
|
143
|
+
const content = entry?.content;
|
|
144
|
+
if (!content)
|
|
145
|
+
continue;
|
|
146
|
+
if (content.entryType === 'TimelineTimelineCursor' && content.contentType === 'Bottom') {
|
|
147
|
+
bottomCursor = String(content.value || bottomCursor || '');
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
if (content.entryType === 'TimelineTimelineItem' && content.itemContent?.tweet_results?.result) {
|
|
151
|
+
results.push(content.itemContent.tweet_results.result);
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
if (content.entryType === 'TimelineTimelineModule') {
|
|
155
|
+
for (const item of content.items || []) {
|
|
156
|
+
const r = item?.itemContent?.tweet_results?.result;
|
|
157
|
+
if (r)
|
|
158
|
+
results.push(r);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
return { results, bottomCursor };
|
|
164
|
+
}
|
|
165
|
+
function timelineOf(data) {
|
|
166
|
+
const user = data?.data?.user?.result;
|
|
167
|
+
const timeline = user?.timeline_v2?.timeline || user?.timeline?.timeline;
|
|
168
|
+
return timeline?.instructions || [];
|
|
169
|
+
}
|
|
170
|
+
/** 用户时间线(tab:tweets=推文/转推;replies=含回复;likes=当前登录用户的点赞) */
|
|
171
|
+
async function fetchUserTimeline(opts) {
|
|
172
|
+
const tab = opts.tab || 'tweets';
|
|
173
|
+
const limit = Math.max(1, Math.min(200, opts.limit ?? 20));
|
|
174
|
+
const get = opts.get || tls_client_1.tlsGet;
|
|
175
|
+
const ids = { ...exports.DEFAULT_QUERY_IDS, ...opts.queryIds };
|
|
176
|
+
let userId = opts.userId;
|
|
177
|
+
if (!userId) {
|
|
178
|
+
const u = await resolveTwitterUser(opts.screenName, opts.creds, get, opts.queryIds);
|
|
179
|
+
userId = u.userId;
|
|
180
|
+
}
|
|
181
|
+
const [opName, queryId] = tab === 'likes'
|
|
182
|
+
? ['Likes', ids.Likes]
|
|
183
|
+
: tab === 'replies'
|
|
184
|
+
? ['UserTweetsAndReplies', ids.UserTweetsAndReplies]
|
|
185
|
+
: ['UserTweets', ids.UserTweets];
|
|
186
|
+
const out = [];
|
|
187
|
+
const seen = new Set();
|
|
188
|
+
let cursor;
|
|
189
|
+
let pages = 0;
|
|
190
|
+
while (out.length < limit && pages < 20) {
|
|
191
|
+
pages++;
|
|
192
|
+
const variables = {
|
|
193
|
+
userId,
|
|
194
|
+
count: 20,
|
|
195
|
+
includePromotedContent: false,
|
|
196
|
+
withQuickPromoteEligibilityTweetQuery: false,
|
|
197
|
+
withVoice: true,
|
|
198
|
+
withV2Timeline: true,
|
|
199
|
+
};
|
|
200
|
+
if (cursor)
|
|
201
|
+
variables.cursor = cursor;
|
|
202
|
+
const data = await gql(opName, queryId, variables, opts.creds, get);
|
|
203
|
+
const { results, bottomCursor } = parseTimeline(timelineOf(data));
|
|
204
|
+
for (const r of results) {
|
|
205
|
+
const cat = categorizeTweetResult(r);
|
|
206
|
+
if (!cat.id || seen.has(cat.id))
|
|
207
|
+
continue;
|
|
208
|
+
seen.add(cat.id);
|
|
209
|
+
const kinds = [];
|
|
210
|
+
if (cat.isRetweet)
|
|
211
|
+
kinds.push('retweet');
|
|
212
|
+
if (cat.isReply)
|
|
213
|
+
kinds.push('reply');
|
|
214
|
+
if (cat.hasVideo)
|
|
215
|
+
kinds.push('video');
|
|
216
|
+
else if (cat.hasImage)
|
|
217
|
+
kinds.push('image');
|
|
218
|
+
else if (!cat.isRetweet)
|
|
219
|
+
kinds.push('text');
|
|
220
|
+
out.push({
|
|
221
|
+
id: cat.id,
|
|
222
|
+
tweet: (0, twitter_1.mapGraphql)(cat.inner),
|
|
223
|
+
kinds,
|
|
224
|
+
isRetweet: cat.isRetweet,
|
|
225
|
+
isReply: cat.isReply,
|
|
226
|
+
replyToId: cat.replyToId,
|
|
227
|
+
url: `https://x.com/i/web/status/${cat.id}`,
|
|
228
|
+
});
|
|
229
|
+
if (out.length >= limit)
|
|
230
|
+
break;
|
|
231
|
+
}
|
|
232
|
+
if (!bottomCursor || !results.length)
|
|
233
|
+
break;
|
|
234
|
+
cursor = bottomCursor;
|
|
235
|
+
}
|
|
236
|
+
return out;
|
|
237
|
+
}
|
|
238
|
+
function userOfEntry(itemContent) {
|
|
239
|
+
const user = itemContent?.user_results?.result;
|
|
240
|
+
if (!user || user.__typename === 'UserUnavailable')
|
|
241
|
+
return null;
|
|
242
|
+
const legacy = user.legacy || {};
|
|
243
|
+
return {
|
|
244
|
+
name: String(legacy.name ?? user.core?.name ?? ''),
|
|
245
|
+
screenName: String(legacy.screen_name ?? user.core?.screen_name ?? ''),
|
|
246
|
+
description: String(user.profile_bio?.description ?? legacy.description ?? ''),
|
|
247
|
+
followers: Number(legacy.followers_count ?? 0) || 0,
|
|
248
|
+
verified: !!user.is_blue_verified || !!legacy.verified,
|
|
249
|
+
followedBy: !!legacy.followed_by,
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
/** 纯函数:从 Followers/Following instructions 抽取用户与底部游标 */
|
|
253
|
+
function parseConnections(instructions) {
|
|
254
|
+
const users = [];
|
|
255
|
+
let bottomCursor;
|
|
256
|
+
for (const ins of instructions || []) {
|
|
257
|
+
for (const entry of ins?.entries || []) {
|
|
258
|
+
const content = entry?.content;
|
|
259
|
+
if (!content)
|
|
260
|
+
continue;
|
|
261
|
+
if (content.entryType === 'TimelineTimelineCursor' && content.contentType === 'Bottom') {
|
|
262
|
+
bottomCursor = String(content.value || bottomCursor || '');
|
|
263
|
+
continue;
|
|
264
|
+
}
|
|
265
|
+
if (content.entryType === 'TimelineTimelineItem' && content.itemContent) {
|
|
266
|
+
const u = userOfEntry(content.itemContent);
|
|
267
|
+
if (u && u.screenName)
|
|
268
|
+
users.push(u);
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
return { users, bottomCursor };
|
|
273
|
+
}
|
|
274
|
+
/** 关注列表(following)或粉丝列表(followers) */
|
|
275
|
+
async function fetchUserConnections(opts) {
|
|
276
|
+
const limit = Math.max(1, Math.min(1000, opts.limit ?? 50));
|
|
277
|
+
const get = opts.get || tls_client_1.tlsGet;
|
|
278
|
+
const ids = { ...exports.DEFAULT_QUERY_IDS, ...opts.queryIds };
|
|
279
|
+
const u = await resolveTwitterUser(opts.screenName, opts.creds, get, opts.queryIds);
|
|
280
|
+
const [opName, queryId] = opts.type === 'following'
|
|
281
|
+
? ['Following', ids.Following]
|
|
282
|
+
: ['Followers', ids.Followers];
|
|
283
|
+
const out = [];
|
|
284
|
+
const seen = new Set();
|
|
285
|
+
let cursor;
|
|
286
|
+
let pages = 0;
|
|
287
|
+
while (out.length < limit && pages < 50) {
|
|
288
|
+
pages++;
|
|
289
|
+
const variables = { userId: u.userId, count: 50, includePromotedContent: false };
|
|
290
|
+
if (cursor)
|
|
291
|
+
variables.cursor = cursor;
|
|
292
|
+
const data = await gql(opName, queryId, variables, opts.creds, get);
|
|
293
|
+
const { users, bottomCursor } = parseConnections(timelineOf(data));
|
|
294
|
+
for (const user of users) {
|
|
295
|
+
if (seen.has(user.screenName))
|
|
296
|
+
continue;
|
|
297
|
+
seen.add(user.screenName);
|
|
298
|
+
out.push(user);
|
|
299
|
+
if (out.length >= limit)
|
|
300
|
+
break;
|
|
301
|
+
}
|
|
302
|
+
if (!bottomCursor || !users.length)
|
|
303
|
+
break;
|
|
304
|
+
cursor = bottomCursor;
|
|
305
|
+
}
|
|
306
|
+
return out;
|
|
307
|
+
}
|
package/lib/engine/twitter.d.ts
CHANGED
|
@@ -23,4 +23,29 @@ export interface GrokTranslation {
|
|
|
23
23
|
* x-client-transaction-id / cf_clearance。任何失败返回 null 由调用方回落通用翻译。
|
|
24
24
|
*/
|
|
25
25
|
export declare function fetchGrokTranslation(url: string, targetLang: string, creds: TwitterCreds, get?: GraphqlGetter): Promise<GrokTranslation | null>;
|
|
26
|
+
export declare function extractTweetId(url: string): string | null;
|
|
27
|
+
/** 把 GraphQL TweetResultByRestId 响应映射为 ParsedData */
|
|
28
|
+
export declare function mapGraphql(rawResult: any): ParsedData;
|
|
29
|
+
/** 解除 TweetWithVisibilityResults 包裹(NSFW/受限推文) */
|
|
30
|
+
export declare function unwrapTweetResult(result: any): any;
|
|
31
|
+
/** 鉴权 GraphQL 原始获取:返回未映射的 tweet result 节点(树构建复用) */
|
|
32
|
+
export declare function fetchGraphqlRaw(id: string, creds: TwitterCreds, get: GraphqlGetter): Promise<any>;
|
|
26
33
|
export declare function parseTwitter(url: string, http: AxiosInstance, creds?: TwitterCreds, getGraphql?: GraphqlGetter): Promise<ParsedData>;
|
|
34
|
+
/** 推文树节点:quoted = 本推引用的推文(递归);replyTo = 本推回复的目标(向根方向递归) */
|
|
35
|
+
export interface TweetTree {
|
|
36
|
+
id: string;
|
|
37
|
+
tweet: ParsedData;
|
|
38
|
+
quoted?: TweetTree;
|
|
39
|
+
replyTo?: TweetTree;
|
|
40
|
+
}
|
|
41
|
+
/** 游客态(syndication)取被引用推文 ID:显式字段优先;否则启发式——引用链的
|
|
42
|
+
* t.co 短链由 X 追加在正文末尾,expanded_url 为推文永久链。 */
|
|
43
|
+
export declare function guestQuoteId(tw: any): string | undefined;
|
|
44
|
+
/** 游客态(syndication)取回复目标推文 ID */
|
|
45
|
+
export declare function guestReplyToId(tw: any): string | undefined;
|
|
46
|
+
/**
|
|
47
|
+
* 拉取推文树:引用链(quoted,向下递归)+ 回复链(replyTo,向根递归)。
|
|
48
|
+
* - 有登录态:GraphQL 单次响应自带嵌套 quoted_status_result;父级逐次拉取
|
|
49
|
+
* - 无登录态:syndication 逐节点拉取(引用 ID 走启发式);不可达分支静默截断
|
|
50
|
+
*/
|
|
51
|
+
export declare function fetchTweetTree(url: string, http: AxiosInstance, creds?: TwitterCreds, getGraphql?: GraphqlGetter): Promise<TweetTree>;
|
package/lib/engine/twitter.js
CHANGED
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.fetchGrokTranslation = fetchGrokTranslation;
|
|
4
|
+
exports.extractTweetId = extractTweetId;
|
|
5
|
+
exports.mapGraphql = mapGraphql;
|
|
6
|
+
exports.unwrapTweetResult = unwrapTweetResult;
|
|
7
|
+
exports.fetchGraphqlRaw = fetchGraphqlRaw;
|
|
4
8
|
exports.parseTwitter = parseTwitter;
|
|
9
|
+
exports.guestQuoteId = guestQuoteId;
|
|
10
|
+
exports.guestReplyToId = guestReplyToId;
|
|
11
|
+
exports.fetchTweetTree = fetchTweetTree;
|
|
5
12
|
const tls_client_1 = require("../utils/tls-client");
|
|
6
13
|
/**
|
|
7
14
|
* X / Twitter 原生解析器。
|
|
@@ -168,14 +175,15 @@ function deepFindLongestText(node) {
|
|
|
168
175
|
}
|
|
169
176
|
return best;
|
|
170
177
|
}
|
|
171
|
-
/** 清理描述中的 t.co 短链(Twitter 自动附加的截断 URL
|
|
178
|
+
/** 清理描述中的 t.co 短链(Twitter 自动附加的截断 URL,无实际内容价值)。
|
|
179
|
+
* 注意保留段落换行:仅折叠水平空白,≥3 连续换行压成空行一档。 */
|
|
172
180
|
function cleanDesc(text) {
|
|
173
181
|
if (!text)
|
|
174
182
|
return text;
|
|
175
|
-
//
|
|
176
|
-
let cleaned = text.replace(
|
|
177
|
-
//
|
|
178
|
-
cleaned = cleaned.replace(
|
|
183
|
+
// 去掉全部 t.co URL(文末自动附加 + 文中内嵌)
|
|
184
|
+
let cleaned = text.replace(/https?:\/\/t\.co\/[A-Za-z0-9]+(?:\?[^\s]*)?/g, '');
|
|
185
|
+
// 仅折叠空格/制表(不动换行);去掉行首尾空格;连续空行压成一个空行
|
|
186
|
+
cleaned = cleaned.replace(/[^\S\n]+/g, ' ').replace(/ *\n */g, '\n').replace(/\n{3,}/g, '\n\n').trim();
|
|
179
187
|
return cleaned || text;
|
|
180
188
|
}
|
|
181
189
|
function baseParsed() {
|
|
@@ -369,8 +377,12 @@ function mapGraphql(rawResult) {
|
|
|
369
377
|
p.title = '';
|
|
370
378
|
return p;
|
|
371
379
|
}
|
|
372
|
-
/**
|
|
373
|
-
|
|
380
|
+
/** 解除 TweetWithVisibilityResults 包裹(NSFW/受限推文) */
|
|
381
|
+
function unwrapTweetResult(result) {
|
|
382
|
+
return result && result.__typename === 'TweetWithVisibilityResults' && result.tweet ? result.tweet : result;
|
|
383
|
+
}
|
|
384
|
+
/** 鉴权 GraphQL 原始获取:返回未映射的 tweet result 节点(树构建复用) */
|
|
385
|
+
async function fetchGraphqlRaw(id, creds, get) {
|
|
374
386
|
const variables = { tweetId: id, includePromotedContent: true, withBirdwatchNotes: true, withVoice: true, withCommunity: true };
|
|
375
387
|
const url = `https://x.com/i/api/graphql/${TWEET_RESULT_QUERY_ID}/TweetResultByRestId` +
|
|
376
388
|
`?variables=${encodeURIComponent(JSON.stringify(variables))}&features=${encodeURIComponent(JSON.stringify(GRAPHQL_FEATURES))}`;
|
|
@@ -405,7 +417,11 @@ async function fetchGraphqlTweet(id, creds, get) {
|
|
|
405
417
|
const tb = result?.tombstone?.text?.text || result?.tombstone?.text;
|
|
406
418
|
throw new Error(`推文不可访问(可能需要登录、已被删除或为非公开内容)${tb ? ':' + tb : ''}`);
|
|
407
419
|
}
|
|
408
|
-
return
|
|
420
|
+
return unwrapTweetResult(result);
|
|
421
|
+
}
|
|
422
|
+
/** 鉴权 GraphQL:仅用 auth_token + ct0,回退取登录受限推文 */
|
|
423
|
+
async function fetchGraphqlTweet(id, creds, get) {
|
|
424
|
+
return mapGraphql(await fetchGraphqlRaw(id, creds, get));
|
|
409
425
|
}
|
|
410
426
|
async function parseTwitter(url, http, creds, getGraphql) {
|
|
411
427
|
const id = extractTweetId(url);
|
|
@@ -440,3 +456,113 @@ async function parseTwitter(url, http, creds, getGraphql) {
|
|
|
440
456
|
const reasonRaw = pick(tw?.tombstone?.text, tw?.tombstone?.name);
|
|
441
457
|
throw new Error(`推文不可访问(可能需要登录、已被删除或为非公开内容)${reasonRaw ? ':' + reasonRaw : ''}`);
|
|
442
458
|
}
|
|
459
|
+
const TREE_QUOTE_DEPTH = 6;
|
|
460
|
+
const TREE_REPLY_DEPTH = 12;
|
|
461
|
+
/** 游客态(syndication)取被引用推文 ID:显式字段优先;否则启发式——引用链的
|
|
462
|
+
* t.co 短链由 X 追加在正文末尾,expanded_url 为推文永久链。 */
|
|
463
|
+
function guestQuoteId(tw) {
|
|
464
|
+
const q = tw?.quoted_tweet || tw?.quoted_status_result;
|
|
465
|
+
if (q) {
|
|
466
|
+
const id = String(pick(q.id_str, q.rest_id, ''));
|
|
467
|
+
if (/^\d+$/.test(id))
|
|
468
|
+
return id;
|
|
469
|
+
}
|
|
470
|
+
const urls = Array.isArray(tw?.entities?.urls) ? tw.entities.urls : [];
|
|
471
|
+
const rawText = String(tw?.text || '');
|
|
472
|
+
const textEnd = rawText.trimEnd();
|
|
473
|
+
for (const u of urls) {
|
|
474
|
+
if (!u?.url || !textEnd.endsWith(u.url))
|
|
475
|
+
continue;
|
|
476
|
+
const m = /(?:x|twitter)\.com\/[^/]+\/status(?:es)?\/(\d+)/.exec(String(u.expanded_url || ''));
|
|
477
|
+
if (m)
|
|
478
|
+
return m[1];
|
|
479
|
+
}
|
|
480
|
+
return undefined;
|
|
481
|
+
}
|
|
482
|
+
/** 游客态(syndication)取回复目标推文 ID */
|
|
483
|
+
function guestReplyToId(tw) {
|
|
484
|
+
const id = pick(tw?.in_reply_to_status_id_str, tw?.in_reply_to_status_id);
|
|
485
|
+
if (typeof id === 'string' && /^\d+$/.test(id))
|
|
486
|
+
return id;
|
|
487
|
+
if (typeof id === 'number')
|
|
488
|
+
return String(id);
|
|
489
|
+
return undefined;
|
|
490
|
+
}
|
|
491
|
+
async function fetchSyndicationRaw(tweetId, http) {
|
|
492
|
+
const res = await http.get(SYNDICATION_URL, {
|
|
493
|
+
params: { id: tweetId, token: 'a' },
|
|
494
|
+
timeout: 30000,
|
|
495
|
+
headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' },
|
|
496
|
+
});
|
|
497
|
+
const tw = res.data;
|
|
498
|
+
return tw && tw.__typename === 'Tweet' && tw.user ? tw : null;
|
|
499
|
+
}
|
|
500
|
+
/**
|
|
501
|
+
* 拉取推文树:引用链(quoted,向下递归)+ 回复链(replyTo,向根递归)。
|
|
502
|
+
* - 有登录态:GraphQL 单次响应自带嵌套 quoted_status_result;父级逐次拉取
|
|
503
|
+
* - 无登录态:syndication 逐节点拉取(引用 ID 走启发式);不可达分支静默截断
|
|
504
|
+
*/
|
|
505
|
+
async function fetchTweetTree(url, http, creds, getGraphql) {
|
|
506
|
+
const id = extractTweetId(url);
|
|
507
|
+
if (!id)
|
|
508
|
+
throw new Error('无法从 X 链接提取推文 ID');
|
|
509
|
+
const visited = new Set();
|
|
510
|
+
async function buildGuest(tweetId, depth) {
|
|
511
|
+
visited.add(tweetId);
|
|
512
|
+
const tw = await fetchSyndicationRaw(tweetId, http);
|
|
513
|
+
if (!tw)
|
|
514
|
+
throw new Error(`推文 ${tweetId} 不可访问(可能需要登录或已删除)`);
|
|
515
|
+
const node = { id: tweetId, tweet: mapSyndication(tw) };
|
|
516
|
+
const qid = guestQuoteId(tw);
|
|
517
|
+
if (qid && !visited.has(qid) && depth < TREE_QUOTE_DEPTH) {
|
|
518
|
+
try {
|
|
519
|
+
node.quoted = await buildGuest(qid, depth + 1);
|
|
520
|
+
}
|
|
521
|
+
catch { /* 引用不可达则截断 */ }
|
|
522
|
+
}
|
|
523
|
+
const rid = guestReplyToId(tw);
|
|
524
|
+
if (rid && !visited.has(rid) && depth < TREE_REPLY_DEPTH) {
|
|
525
|
+
try {
|
|
526
|
+
node.replyTo = await buildGuest(rid, depth + 1);
|
|
527
|
+
}
|
|
528
|
+
catch { /* 父推不可达则截断 */ }
|
|
529
|
+
}
|
|
530
|
+
return node;
|
|
531
|
+
}
|
|
532
|
+
async function buildFromGraphqlResult(raw, depth) {
|
|
533
|
+
raw = unwrapTweetResult(raw);
|
|
534
|
+
const rid0 = String(pick(raw?.rest_id, raw?.legacy?.id_str, ''));
|
|
535
|
+
if (rid0)
|
|
536
|
+
visited.add(rid0);
|
|
537
|
+
const node = { id: rid0, tweet: mapGraphql(raw) };
|
|
538
|
+
const q = unwrapTweetResult(raw?.quoted_status_result?.result);
|
|
539
|
+
if (q && q.legacy && q.__typename !== 'TweetTombstone' && depth < TREE_QUOTE_DEPTH) {
|
|
540
|
+
const qid = String(pick(q.rest_id, q.legacy?.id_str, ''));
|
|
541
|
+
if (!qid || !visited.has(qid))
|
|
542
|
+
node.quoted = await buildFromGraphqlResult(q, depth + 1);
|
|
543
|
+
}
|
|
544
|
+
const rid = raw?.legacy?.in_reply_to_status_id_str;
|
|
545
|
+
if (typeof rid === 'string' && /^\d+$/.test(rid) && !visited.has(rid) && depth < TREE_REPLY_DEPTH) {
|
|
546
|
+
try {
|
|
547
|
+
node.replyTo = await buildGraphqlById(rid, depth + 1);
|
|
548
|
+
}
|
|
549
|
+
catch { /* 父推不可达则截断 */ }
|
|
550
|
+
}
|
|
551
|
+
return node;
|
|
552
|
+
}
|
|
553
|
+
async function buildGraphqlById(tweetId, depth) {
|
|
554
|
+
if (visited.has(tweetId))
|
|
555
|
+
throw new Error('引用环');
|
|
556
|
+
const raw = await fetchGraphqlRaw(tweetId, creds, getGraphql || tls_client_1.tlsGet);
|
|
557
|
+
return buildFromGraphqlResult(raw, depth);
|
|
558
|
+
}
|
|
559
|
+
if (creds && creds.authToken && creds.ct0) {
|
|
560
|
+
try {
|
|
561
|
+
return await buildGraphqlById(id, 0);
|
|
562
|
+
}
|
|
563
|
+
catch {
|
|
564
|
+
/* 登录态失败(如 Cloudflare)回退游客路径 */
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
return buildGuest(id, 0);
|
|
568
|
+
}
|
package/lib/index.d.ts
CHANGED
|
@@ -23,6 +23,7 @@ export * from './engine/config';
|
|
|
23
23
|
export * from './engine/platform-config';
|
|
24
24
|
export * from './engine/translate';
|
|
25
25
|
export * from './engine/twitter';
|
|
26
|
+
export * from './engine/twitter-user';
|
|
26
27
|
export * from './engine/parser';
|
|
27
28
|
export * from './engine/fetcher';
|
|
28
29
|
export * from './engine/runtime';
|
package/lib/index.js
CHANGED
|
@@ -39,6 +39,7 @@ __exportStar(require("./engine/config"), exports);
|
|
|
39
39
|
__exportStar(require("./engine/platform-config"), exports);
|
|
40
40
|
__exportStar(require("./engine/translate"), exports);
|
|
41
41
|
__exportStar(require("./engine/twitter"), exports);
|
|
42
|
+
__exportStar(require("./engine/twitter-user"), exports);
|
|
42
43
|
__exportStar(require("./engine/parser"), exports);
|
|
43
44
|
__exportStar(require("./engine/fetcher"), exports);
|
|
44
45
|
__exportStar(require("./engine/runtime"), exports);
|
package/lib/utils/format.js
CHANGED
|
@@ -28,10 +28,15 @@ function formatPublishTime(ms) {
|
|
|
28
28
|
}
|
|
29
29
|
function generateFormattedText(p, format, index, total) {
|
|
30
30
|
const imageCount = p.images.length || p.live_photo.length;
|
|
31
|
+
// 纯文字作品(type=text):正文即内容。模板含 ${正文} 时启用新语义——
|
|
32
|
+
// 正文=desc、简介置空(自动隐藏),避免把正文标成「简介」;旧模板保持原行为。
|
|
33
|
+
const plainTextWork = p.type === 'text' && !!p.desc;
|
|
34
|
+
const wantsBody = plainTextWork && format.includes('${正文}');
|
|
31
35
|
const vars = {
|
|
32
36
|
'标题': p.title,
|
|
33
37
|
'作者': p.author,
|
|
34
|
-
'简介': p.desc,
|
|
38
|
+
'简介': wantsBody ? '' : p.desc,
|
|
39
|
+
'正文': plainTextWork ? p.desc : '',
|
|
35
40
|
'视频时长': p.duration > 0 ? formatDuration(p.duration) : '',
|
|
36
41
|
'点赞数': String(p.like),
|
|
37
42
|
'收藏数': String(p.collect),
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sns-parse/core",
|
|
3
|
-
"version": "0.4.0-alpha.
|
|
3
|
+
"version": "0.4.0-alpha.7+upstream.1.6.7",
|
|
4
4
|
"description": "sns-parse 核心:契约、宿主抽象、配置 DSL 与解析引擎(fetcher/parser/翻译/X 原生解析/合并/GIF/TLS 指纹客户端)",
|
|
5
5
|
"main": "lib/index.js",
|
|
6
6
|
"types": "lib/index.d.ts",
|