@nite07/search-core 0.1.0 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -0
- package/dist/adapters.d.ts +20 -1
- package/dist/adapters.js +21 -4
- package/dist/browser-backends.d.ts +7 -0
- package/dist/browser-backends.js +24 -0
- package/dist/browser.d.ts +46 -0
- package/dist/browser.js +130 -0
- package/dist/catalog.js +1 -0
- package/dist/config.d.ts +8 -0
- package/dist/config.js +50 -17
- package/dist/engine.js +29 -3
- package/dist/index.d.ts +5 -1
- package/dist/index.js +4 -1
- package/dist/media.d.ts +39 -0
- package/dist/media.js +84 -0
- package/dist/reddit/shape.d.ts +20 -0
- package/dist/reddit/shape.js +50 -0
- package/dist/types.d.ts +12 -2
- package/dist/x/parse.d.ts +4 -0
- package/dist/x/parse.js +25 -2
- package/dist/xhs/index.d.ts +94 -0
- package/dist/xhs/index.js +160 -0
- package/dist/xhs/scripts.d.ts +29 -0
- package/dist/xhs/scripts.js +127 -0
- package/package.json +4 -1
package/dist/media.js
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { mkdir, readdir, stat, unlink, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
const EXTENSIONS = {
|
|
6
|
+
'image/jpeg': 'jpg', 'image/jpg': 'jpg', 'image/png': 'png', 'image/webp': 'webp',
|
|
7
|
+
'image/gif': 'gif', 'image/avif': 'avif',
|
|
8
|
+
};
|
|
9
|
+
const DEFAULT_LIMIT = 24;
|
|
10
|
+
const MAX_LIMIT = 24;
|
|
11
|
+
const DEFAULT_MAX_BYTES = 8 * 1024 * 1024;
|
|
12
|
+
const DEFAULT_MAX_AGE_MS = 60 * 60 * 1000;
|
|
13
|
+
const DEFAULT_TOTAL_BYTES = 48 * 1024 * 1024;
|
|
14
|
+
/** The scratch directory these files live in, so a caller can point a reader at it. */
|
|
15
|
+
export function mediaDir() {
|
|
16
|
+
return join(tmpdir(), 'dsh-search-media');
|
|
17
|
+
}
|
|
18
|
+
async function sweep(dir, maxAgeMs) {
|
|
19
|
+
const now = Date.now();
|
|
20
|
+
let names;
|
|
21
|
+
try {
|
|
22
|
+
names = await readdir(dir);
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
await Promise.all(names.map(async (name) => {
|
|
28
|
+
const path = join(dir, name);
|
|
29
|
+
try {
|
|
30
|
+
const info = await stat(path);
|
|
31
|
+
if (now - info.mtimeMs > maxAgeMs)
|
|
32
|
+
await unlink(path);
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
// Another reader removed it first, or it is not ours.
|
|
36
|
+
}
|
|
37
|
+
}));
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Download images to disk, one entry per input, in input order. A failed download is reported in
|
|
41
|
+
* place rather than dropped, because a caller that asked for pictures should learn why it got none.
|
|
42
|
+
*/
|
|
43
|
+
export async function materialiseImages(urls, options = {}) {
|
|
44
|
+
const limit = Math.min(Math.max(Math.trunc(options.limit ?? DEFAULT_LIMIT), 0), MAX_LIMIT);
|
|
45
|
+
const totalBudget = options.totalBytes ?? DEFAULT_TOTAL_BYTES;
|
|
46
|
+
const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
|
|
47
|
+
const dir = options.dir ?? mediaDir();
|
|
48
|
+
const unique = [...new Set(urls.filter(url => typeof url === 'string' && /^https?:\/\//.test(url)))].slice(0, limit);
|
|
49
|
+
if (!unique.length)
|
|
50
|
+
return [];
|
|
51
|
+
const fetchImpl = options.fetch ?? fetch;
|
|
52
|
+
await mkdir(dir, { recursive: true });
|
|
53
|
+
await sweep(dir, options.maxAgeMs ?? DEFAULT_MAX_AGE_MS);
|
|
54
|
+
let spent = 0;
|
|
55
|
+
return Promise.all(unique.map(async (url) => {
|
|
56
|
+
try {
|
|
57
|
+
const response = await fetchImpl(url, {
|
|
58
|
+
redirect: 'follow',
|
|
59
|
+
...(options.signal ? { signal: options.signal } : {}),
|
|
60
|
+
headers: { accept: 'image/avif,image/webp,image/png,image/jpeg,image/gif;q=0.9,*/*;q=0.5' },
|
|
61
|
+
});
|
|
62
|
+
if (!response.ok)
|
|
63
|
+
return { url, error: `HTTP ${response.status}` };
|
|
64
|
+
const contentType = (response.headers.get('content-type') ?? '').split(';')[0].trim().toLowerCase();
|
|
65
|
+
const extension = EXTENSIONS[contentType];
|
|
66
|
+
if (!extension)
|
|
67
|
+
return { url, error: `not an image (${contentType || 'unknown type'})` };
|
|
68
|
+
const buffer = Buffer.from(await response.arrayBuffer());
|
|
69
|
+
if (buffer.byteLength > maxBytes)
|
|
70
|
+
return { url, error: `${buffer.byteLength} bytes exceeds the ${maxBytes} byte limit` };
|
|
71
|
+
if (buffer.byteLength === 0)
|
|
72
|
+
return { url, error: 'empty response' };
|
|
73
|
+
if (spent + buffer.byteLength > totalBudget)
|
|
74
|
+
return { url, error: `not downloaded: the ${totalBudget} byte budget for one call is used up` };
|
|
75
|
+
spent += buffer.byteLength;
|
|
76
|
+
const path = join(dir, `${createHash('sha1').update(url).digest('hex').slice(0, 16)}.${extension}`);
|
|
77
|
+
await writeFile(path, buffer);
|
|
78
|
+
return { url, path, contentType, bytes: buffer.byteLength };
|
|
79
|
+
}
|
|
80
|
+
catch (error) {
|
|
81
|
+
return { url, error: error instanceof Error ? error.message : String(error) };
|
|
82
|
+
}
|
|
83
|
+
}));
|
|
84
|
+
}
|
package/dist/reddit/shape.d.ts
CHANGED
|
@@ -43,6 +43,24 @@ export interface GqlPostNode {
|
|
|
43
43
|
content?: {
|
|
44
44
|
markdown?: string | null;
|
|
45
45
|
} | null;
|
|
46
|
+
media?: {
|
|
47
|
+
image?: {
|
|
48
|
+
url?: string | null;
|
|
49
|
+
} | null;
|
|
50
|
+
gallery?: {
|
|
51
|
+
items?: Array<{
|
|
52
|
+
url?: string | null;
|
|
53
|
+
}> | null;
|
|
54
|
+
} | null;
|
|
55
|
+
} | null;
|
|
56
|
+
preview?: {
|
|
57
|
+
images?: Array<{
|
|
58
|
+
source?: {
|
|
59
|
+
url?: string | null;
|
|
60
|
+
} | null;
|
|
61
|
+
}> | null;
|
|
62
|
+
} | null;
|
|
63
|
+
urlOverriddenByDest?: string | null;
|
|
46
64
|
}
|
|
47
65
|
export interface RedditComment {
|
|
48
66
|
author: string;
|
|
@@ -63,6 +81,8 @@ export interface PostData {
|
|
|
63
81
|
commentCount?: number;
|
|
64
82
|
createdUtc?: number;
|
|
65
83
|
selfText?: string;
|
|
84
|
+
/** Pictures the post carries, best effort: a direct link, a gallery, or the preview. */
|
|
85
|
+
images: string[];
|
|
66
86
|
}
|
|
67
87
|
/** GraphQL created timestamps look like 2026-10-30T06:10:28.665000+0000; Reddit JSON uses epoch seconds. */
|
|
68
88
|
export declare function isoToEpochSeconds(value: unknown): number | undefined;
|
package/dist/reddit/shape.js
CHANGED
|
@@ -85,6 +85,27 @@ export function searchPostNodes(data) {
|
|
|
85
85
|
return node ? [node] : [];
|
|
86
86
|
});
|
|
87
87
|
}
|
|
88
|
+
/** Media on a GraphQL post: a single picture, a gallery, a preview, or a direct image link. */
|
|
89
|
+
function graphqlImages(node) {
|
|
90
|
+
const found = [];
|
|
91
|
+
const direct = str(node.urlOverriddenByDest ?? node.url);
|
|
92
|
+
if (/\.(?:jpe?g|png|gif|webp)(?:\?|$)/i.test(direct))
|
|
93
|
+
found.push(direct);
|
|
94
|
+
const single = str(node.media?.image?.url);
|
|
95
|
+
if (single)
|
|
96
|
+
found.push(single);
|
|
97
|
+
for (const item of node.media?.gallery?.items ?? []) {
|
|
98
|
+
const url = str(item?.url);
|
|
99
|
+
if (url)
|
|
100
|
+
found.push(url);
|
|
101
|
+
}
|
|
102
|
+
for (const image of node.preview?.images ?? []) {
|
|
103
|
+
const url = str(image?.source?.url).replaceAll('&', '&');
|
|
104
|
+
if (url)
|
|
105
|
+
found.push(url);
|
|
106
|
+
}
|
|
107
|
+
return [...new Set(found)].slice(0, 10);
|
|
108
|
+
}
|
|
88
109
|
export function gqlPost(node) {
|
|
89
110
|
const id = str(node.id).replace(/^t3_/, '');
|
|
90
111
|
const permalink = str(node.permalink);
|
|
@@ -98,6 +119,8 @@ export function gqlPost(node) {
|
|
|
98
119
|
commentCount: num(node.commentCount),
|
|
99
120
|
createdUtc: isoToEpochSeconds(node.createdAt),
|
|
100
121
|
selfText: str(node.content?.markdown),
|
|
122
|
+
// The GraphQL node nests media under the legacy shape it was projected from.
|
|
123
|
+
images: graphqlImages(node),
|
|
101
124
|
};
|
|
102
125
|
}
|
|
103
126
|
// ---------------------------------------------------------------------------
|
|
@@ -184,6 +207,32 @@ export function restThreadPost(payload) {
|
|
|
184
207
|
throw new SearchError('protocol', 'Reddit returned no post for this thread.');
|
|
185
208
|
return post.data;
|
|
186
209
|
}
|
|
210
|
+
/** Reddit wraps media URLs in HTML entities and offers several shapes depending on the post type. */
|
|
211
|
+
function imagesOf(data) {
|
|
212
|
+
const unescape = (value) => value.replaceAll('&', '&');
|
|
213
|
+
const found = [];
|
|
214
|
+
const direct = str(data.url_overridden_by_dest) || str(data.url);
|
|
215
|
+
if (/\.(?:jpe?g|png|gif|webp)(?:\?|$)/i.test(direct))
|
|
216
|
+
found.push(direct);
|
|
217
|
+
const gallery = data.media_metadata;
|
|
218
|
+
if (gallery && typeof gallery === 'object') {
|
|
219
|
+
for (const item of Object.values(gallery)) {
|
|
220
|
+
const source = item?.s;
|
|
221
|
+
const url = str(source?.u) || str(source?.gif);
|
|
222
|
+
if (url)
|
|
223
|
+
found.push(unescape(url));
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
const preview = data.preview?.images;
|
|
227
|
+
if (Array.isArray(preview)) {
|
|
228
|
+
for (const image of preview) {
|
|
229
|
+
const url = str(image?.source?.url);
|
|
230
|
+
if (url)
|
|
231
|
+
found.push(unescape(url));
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
return [...new Set(found)].slice(0, 10);
|
|
235
|
+
}
|
|
187
236
|
export function restPost(data) {
|
|
188
237
|
const id = str(data.id);
|
|
189
238
|
const permalink = str(data.permalink);
|
|
@@ -197,6 +246,7 @@ export function restPost(data) {
|
|
|
197
246
|
commentCount: num(data.num_comments),
|
|
198
247
|
createdUtc: num(data.created_utc),
|
|
199
248
|
selfText: str(data.selftext),
|
|
249
|
+
images: imagesOf(data),
|
|
200
250
|
};
|
|
201
251
|
}
|
|
202
252
|
/** Convert the already-nested REST comment listing; `replies` is "" or another listing. */
|
package/dist/types.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type AccessMode = 'anonymous' | 'key' | 'oauth' | 'session';
|
|
1
|
+
export type AccessMode = 'anonymous' | 'key' | 'oauth' | 'session' | 'browser';
|
|
2
2
|
export type CostTier = 'free' | 'allowance' | 'trial' | 'paid' | 'self-hosted' | 'account' | 'unverified';
|
|
3
3
|
export type Localized = {
|
|
4
4
|
en: string;
|
|
@@ -48,7 +48,6 @@ export interface BackendConfig {
|
|
|
48
48
|
enabled: boolean;
|
|
49
49
|
mode: AccessMode;
|
|
50
50
|
endpoint: string;
|
|
51
|
-
credentialRef: string;
|
|
52
51
|
timeoutMs: number;
|
|
53
52
|
rateLimitPerMinute: number;
|
|
54
53
|
dailyAttemptLimit: number;
|
|
@@ -69,6 +68,8 @@ export interface SearchConfig {
|
|
|
69
68
|
dedup: 'url' | 'url-title';
|
|
70
69
|
fusion: 'rrf' | 'ordered';
|
|
71
70
|
retry: RetryConfig;
|
|
71
|
+
/** Shared by every browser-backed reader; credentials use the fixed browser token name. */
|
|
72
|
+
browser: BrowserConfig;
|
|
72
73
|
cache: {
|
|
73
74
|
enabled: boolean;
|
|
74
75
|
ttlSeconds: number;
|
|
@@ -90,6 +91,15 @@ export interface SearchConfig {
|
|
|
90
91
|
};
|
|
91
92
|
backends: Record<string, BackendConfig>;
|
|
92
93
|
}
|
|
94
|
+
/**
|
|
95
|
+
* One browser, shared by every backend that reads through one. A backend that set its own endpoint
|
|
96
|
+
* keeps working: the shared endpoint wins only when it is filled in.
|
|
97
|
+
*/
|
|
98
|
+
export interface BrowserConfig {
|
|
99
|
+
endpoint: string;
|
|
100
|
+
statePath: string;
|
|
101
|
+
timeoutMs: number;
|
|
102
|
+
}
|
|
93
103
|
export interface Source {
|
|
94
104
|
url: string;
|
|
95
105
|
title?: string;
|
package/dist/x/parse.d.ts
CHANGED
|
@@ -15,6 +15,8 @@ export interface ParsedTweet {
|
|
|
15
15
|
likes: number;
|
|
16
16
|
reposts: number;
|
|
17
17
|
replies: number;
|
|
18
|
+
/** Pictures the post carries; a post can be nothing but pictures. */
|
|
19
|
+
images: string[];
|
|
18
20
|
}
|
|
19
21
|
export declare function parseTweet(result: Node): ParsedTweet | undefined;
|
|
20
22
|
/** Walk the timeline instructions and pull every tweet that has a body worth quoting. */
|
|
@@ -35,6 +37,8 @@ export interface XThreadPost {
|
|
|
35
37
|
text: string;
|
|
36
38
|
url: string;
|
|
37
39
|
createdAt?: string;
|
|
40
|
+
/** Pictures the post carries; a post can be nothing but pictures. */
|
|
41
|
+
images?: string[];
|
|
38
42
|
}
|
|
39
43
|
export interface ParsedThread {
|
|
40
44
|
post: XThreadPost;
|
package/dist/x/parse.js
CHANGED
|
@@ -73,8 +73,28 @@ export function parseTweet(result) {
|
|
|
73
73
|
likes: num(counts.favorite_count ?? legacy.favorite_count),
|
|
74
74
|
reposts: num(counts.retweet_count ?? legacy.retweet_count),
|
|
75
75
|
replies: num(counts.reply_count ?? legacy.reply_count),
|
|
76
|
+
images: tweetImages(legacy),
|
|
76
77
|
};
|
|
77
78
|
}
|
|
79
|
+
/**
|
|
80
|
+
* Photos attached to a tweet. X keeps them under `extended_entities` for real tweets and under
|
|
81
|
+
* `entities` in some projections; only photos are kept, because a video needs a player rather than
|
|
82
|
+
* an image reader.
|
|
83
|
+
*/
|
|
84
|
+
function tweetImages(legacy) {
|
|
85
|
+
const found = [];
|
|
86
|
+
for (const holder of [legacy.extended_entities, legacy.entities]) {
|
|
87
|
+
for (const item of list(node(holder).media)) {
|
|
88
|
+
const media = node(item);
|
|
89
|
+
if (str(media.type) !== 'photo')
|
|
90
|
+
continue;
|
|
91
|
+
const url = str(media.media_url_https) ?? str(media.media_url);
|
|
92
|
+
if (url)
|
|
93
|
+
found.push(url);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return [...new Set(found)];
|
|
97
|
+
}
|
|
78
98
|
/** Walk the timeline instructions and pull every tweet that has a body worth quoting. */
|
|
79
99
|
export function parseInstructions(data, limit) {
|
|
80
100
|
const timeline = node(node(node(data.data).search_by_raw_query).search_timeline).timeline;
|
|
@@ -85,7 +105,7 @@ export function parseInstructions(data, limit) {
|
|
|
85
105
|
if (!result)
|
|
86
106
|
continue;
|
|
87
107
|
const parsed = parseTweet(result);
|
|
88
|
-
if (!parsed || parsed.text.trim() === '')
|
|
108
|
+
if (!parsed || (parsed.text.trim() === '' && parsed.images.length === 0))
|
|
89
109
|
continue;
|
|
90
110
|
tweets.push(parsed);
|
|
91
111
|
if (tweets.length >= limit)
|
|
@@ -106,6 +126,8 @@ export function toSource(tweet) {
|
|
|
106
126
|
}
|
|
107
127
|
if (tweet.createdAt)
|
|
108
128
|
source.publishedAt = tweet.createdAt;
|
|
129
|
+
if (tweet.images.length)
|
|
130
|
+
source.summary = `${source.summary ? `${source.summary} · ` : ''}${tweet.images.length} image${tweet.images.length === 1 ? '' : 's'}`;
|
|
109
131
|
return source;
|
|
110
132
|
}
|
|
111
133
|
/**
|
|
@@ -148,7 +170,7 @@ export function parseThread(data, options) {
|
|
|
148
170
|
const collected = [];
|
|
149
171
|
conversationTweets(node(entry), 0, collected);
|
|
150
172
|
for (const item of collected) {
|
|
151
|
-
if (item.tweet.text.trim() === '')
|
|
173
|
+
if (item.tweet.text.trim() === '' && item.tweet.images.length === 0)
|
|
152
174
|
continue;
|
|
153
175
|
const url = `https://x.com/${item.tweet.handle ?? 'i/web'}/status/${item.tweet.id}`;
|
|
154
176
|
if (!post) {
|
|
@@ -158,6 +180,7 @@ export function parseThread(data, options) {
|
|
|
158
180
|
url,
|
|
159
181
|
...(item.tweet.handle ? { handle: item.tweet.handle } : {}),
|
|
160
182
|
...(item.tweet.createdAt ? { createdAt: item.tweet.createdAt } : {}),
|
|
183
|
+
...(item.tweet.images.length ? { images: item.tweet.images } : {}),
|
|
161
184
|
};
|
|
162
185
|
continue;
|
|
163
186
|
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import type { Source } from '../types.js';
|
|
2
|
+
import { BrowserSession, type BrowserSessionOptions } from '../browser.js';
|
|
3
|
+
/**
|
|
4
|
+
* Xiaohongshu reads through the user's own browser. Everything here is a read: search, one note,
|
|
5
|
+
* its comments. Nothing writes, nothing marks anything read, and every call is capped.
|
|
6
|
+
*/
|
|
7
|
+
export interface XhsContext {
|
|
8
|
+
wsEndpoint: string;
|
|
9
|
+
token?: string;
|
|
10
|
+
siteUrl?: string;
|
|
11
|
+
/** File holding the signed-in session, so a fresh process starts already logged in. */
|
|
12
|
+
statePath?: string;
|
|
13
|
+
timeoutMs: number;
|
|
14
|
+
signal: AbortSignal;
|
|
15
|
+
/** Injected in tests; production uses the real browser connection. */
|
|
16
|
+
browser?: Pick<BrowserSession, 'evaluate' | 'target' | 'persist' | 'close'>;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Default session file, kept beside the browser container's volume so the session the container
|
|
20
|
+
* saves is the session a read starts from. Override with the statePath option or XHS_STATE_PATH.
|
|
21
|
+
*/
|
|
22
|
+
export declare const DEFAULT_STATE_PATH: string;
|
|
23
|
+
export declare const DEFAULT_SITE_URL = "https://www.xiaohongshu.com/explore";
|
|
24
|
+
export interface XhsNoteSummary {
|
|
25
|
+
id: string;
|
|
26
|
+
title: string;
|
|
27
|
+
author: string;
|
|
28
|
+
likes: string;
|
|
29
|
+
url: string;
|
|
30
|
+
}
|
|
31
|
+
export interface XhsNote {
|
|
32
|
+
id: string;
|
|
33
|
+
title: string;
|
|
34
|
+
desc: string;
|
|
35
|
+
author: string;
|
|
36
|
+
likes: string;
|
|
37
|
+
collects: string;
|
|
38
|
+
comments: string;
|
|
39
|
+
shares: string;
|
|
40
|
+
publishedAt?: string;
|
|
41
|
+
tags: string[];
|
|
42
|
+
/** Picture URLs, in the order the note shows them. */
|
|
43
|
+
imageUrls: string[];
|
|
44
|
+
url: string;
|
|
45
|
+
}
|
|
46
|
+
export interface XhsComment {
|
|
47
|
+
id: string;
|
|
48
|
+
author: string;
|
|
49
|
+
content: string;
|
|
50
|
+
likes: string;
|
|
51
|
+
replies: string;
|
|
52
|
+
ipLocation?: string;
|
|
53
|
+
}
|
|
54
|
+
export declare function optionsOf(ctx: XhsContext): BrowserSessionOptions;
|
|
55
|
+
/** Page answers carry their own vocabulary; translate once, here. */
|
|
56
|
+
export declare function unwrapAnswer<T>(answer: unknown): T;
|
|
57
|
+
/** Read the search page, which also yields the per-note token that detail and comment reads need. */
|
|
58
|
+
export declare function readXhsSearchNotes(input: {
|
|
59
|
+
query: string;
|
|
60
|
+
limit: number;
|
|
61
|
+
sort?: string;
|
|
62
|
+
}, ctx: XhsContext): Promise<XhsNoteSummary[]>;
|
|
63
|
+
/** Search as the engine sees it: citeable sources, one per note. */
|
|
64
|
+
export declare function searchXhs(input: {
|
|
65
|
+
query: string;
|
|
66
|
+
maxResults: number;
|
|
67
|
+
sort?: string;
|
|
68
|
+
}, ctx: XhsContext): Promise<Source[]>;
|
|
69
|
+
/** A note's detail. The token comes from the argument, the share link, or an earlier search. */
|
|
70
|
+
export declare function fetchXhsNote(input: {
|
|
71
|
+
url?: string;
|
|
72
|
+
id?: string;
|
|
73
|
+
token?: string;
|
|
74
|
+
}, ctx: XhsContext): Promise<XhsNote>;
|
|
75
|
+
export declare function fetchXhsComments(input: {
|
|
76
|
+
url?: string;
|
|
77
|
+
id?: string;
|
|
78
|
+
token?: string;
|
|
79
|
+
limit?: number;
|
|
80
|
+
cursor?: string;
|
|
81
|
+
}, ctx: XhsContext): Promise<{
|
|
82
|
+
comments: XhsComment[];
|
|
83
|
+
hasMore: boolean;
|
|
84
|
+
cursor?: string;
|
|
85
|
+
}>;
|
|
86
|
+
/** Accept a share link, a bare id, or an explicit token. Share links embed xsec_token. */
|
|
87
|
+
export declare function resolveTarget(input: {
|
|
88
|
+
url?: string;
|
|
89
|
+
id?: string;
|
|
90
|
+
token?: string;
|
|
91
|
+
}): {
|
|
92
|
+
id: string;
|
|
93
|
+
token: string;
|
|
94
|
+
};
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { homedir } from 'node:os';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { SearchError } from '../transport.js';
|
|
4
|
+
import { sessionFor } from '../browser.js';
|
|
5
|
+
import { commentsScript, healthScript, noteScript, searchScript } from './scripts.js';
|
|
6
|
+
/**
|
|
7
|
+
* Default session file, kept beside the browser container's volume so the session the container
|
|
8
|
+
* saves is the session a read starts from. Override with the statePath option or XHS_STATE_PATH.
|
|
9
|
+
*/
|
|
10
|
+
export const DEFAULT_STATE_PATH = join(homedir(), '.config', 'dsh-search', 'browser', 'state.json');
|
|
11
|
+
export const DEFAULT_SITE_URL = 'https://www.xiaohongshu.com/explore';
|
|
12
|
+
/**
|
|
13
|
+
* One connection per endpoint, kept for the life of the process.
|
|
14
|
+
*
|
|
15
|
+
* A connection owns its browser context, and that context is the window a human signs into over
|
|
16
|
+
* VNC. Dropping it after every read would close that window and leave no way to log in, so the
|
|
17
|
+
* connection is reused until the host shuts it down.
|
|
18
|
+
*/
|
|
19
|
+
/**
|
|
20
|
+
* Search results carry the per-note token that detail and comment reads require, but a Source must
|
|
21
|
+
* not expose it. Keeping it here is what lets a caller read a note it just saw in a search.
|
|
22
|
+
*/
|
|
23
|
+
const noteTokens = new Map();
|
|
24
|
+
const TOKEN_CACHE_LIMIT = 500;
|
|
25
|
+
function rememberToken(id, token) {
|
|
26
|
+
if (!token)
|
|
27
|
+
return;
|
|
28
|
+
noteTokens.set(id, token);
|
|
29
|
+
if (noteTokens.size > TOKEN_CACHE_LIMIT) {
|
|
30
|
+
const oldest = noteTokens.keys().next();
|
|
31
|
+
if (!oldest.done)
|
|
32
|
+
noteTokens.delete(oldest.value);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
const browserFor = (ctx) => ctx.browser ?? sessionFor(`xhs|${ctx.wsEndpoint}|${ctx.token ?? ''}`, optionsOf(ctx));
|
|
36
|
+
export function optionsOf(ctx) {
|
|
37
|
+
return {
|
|
38
|
+
wsEndpoint: ctx.wsEndpoint,
|
|
39
|
+
siteUrl: ctx.siteUrl ?? DEFAULT_SITE_URL,
|
|
40
|
+
timeoutMs: ctx.timeoutMs,
|
|
41
|
+
statePath: ctx.statePath ?? process.env.XHS_STATE_PATH ?? DEFAULT_STATE_PATH,
|
|
42
|
+
...(ctx.token ? { token: ctx.token } : {}),
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
/** Page answers carry their own vocabulary; translate once, here. */
|
|
46
|
+
export function unwrapAnswer(answer) {
|
|
47
|
+
const body = (answer ?? {});
|
|
48
|
+
if (body.ok === true)
|
|
49
|
+
return answer;
|
|
50
|
+
const code = body.code ?? '';
|
|
51
|
+
const detail = body.message ?? 'no detail';
|
|
52
|
+
if (code === 'NOT_READY' || code === 'NO_RUNTIME' || code === 'NO_CLIENT' || code === 'NO_REGISTRATION' || code === 'BAD_CLIENT') {
|
|
53
|
+
return fail('protocol', `The browser page has not loaded Xiaohongshu, so the site runtime was unavailable (${code}). Open xiaohongshu.com in that browser once.`);
|
|
54
|
+
}
|
|
55
|
+
if (code === 'EMPTY')
|
|
56
|
+
return fail('protocol', `Xiaohongshu returned no note for that id (${detail}).`);
|
|
57
|
+
const status = Number(body.status ?? 0);
|
|
58
|
+
if (/登录|login/i.test(detail)) {
|
|
59
|
+
return fail('authentication', `Xiaohongshu says the browser is not signed in (${detail}). Open the browser container's VNC page, sign in once in the window that is already open, and ask again.`);
|
|
60
|
+
}
|
|
61
|
+
if (status === 401 || status === 406)
|
|
62
|
+
return fail('authentication', `Xiaohongshu refused the read with HTTP ${status}; the browser session is not signed in or was challenged. Sign in again in that browser.`);
|
|
63
|
+
if (status === 403)
|
|
64
|
+
return fail('authentication', `Xiaohongshu denied this read with HTTP 403 (${detail}). The subject may be private, or the session lacks permission.`);
|
|
65
|
+
if (status === 429)
|
|
66
|
+
return fail('rate-limit', 'Xiaohongshu rate-limited the read; wait before asking again.');
|
|
67
|
+
return fail('protocol', `Xiaohongshu answered something unexpected (${code || 'no code'}): ${detail}`);
|
|
68
|
+
}
|
|
69
|
+
function fail(kind, message) {
|
|
70
|
+
throw new SearchError(kind, message);
|
|
71
|
+
}
|
|
72
|
+
const guard = (ctx) => {
|
|
73
|
+
if (!ctx.wsEndpoint)
|
|
74
|
+
throw new SearchError('configuration', 'The Xiaohongshu backend needs a Playwright endpoint. Set the browser endpoint option or XHS_WS_ENDPOINT to the URL the browser prints at startup.');
|
|
75
|
+
if (ctx.signal.aborted)
|
|
76
|
+
throw new SearchError('cancelled', 'The Xiaohongshu read was cancelled.');
|
|
77
|
+
};
|
|
78
|
+
/** Read the search page, which also yields the per-note token that detail and comment reads need. */
|
|
79
|
+
export async function readXhsSearchNotes(input, ctx) {
|
|
80
|
+
guard(ctx);
|
|
81
|
+
const browser = browserFor(ctx);
|
|
82
|
+
try {
|
|
83
|
+
await unwrapAnswer(await browser.evaluate(healthScript()));
|
|
84
|
+
const answer = unwrapAnswer(await browser.evaluate(searchScript({ query: input.query, limit: input.limit, page: 1, sort: input.sort ?? 'general' })));
|
|
85
|
+
for (const item of answer.items)
|
|
86
|
+
rememberToken(item.id, item.xsecToken);
|
|
87
|
+
return answer.items;
|
|
88
|
+
}
|
|
89
|
+
finally {
|
|
90
|
+
await browser.persist().catch(() => undefined);
|
|
91
|
+
await browser.close().catch(() => undefined);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
/** Search as the engine sees it: citeable sources, one per note. */
|
|
95
|
+
export async function searchXhs(input, ctx) {
|
|
96
|
+
const notes = await readXhsSearchNotes({ query: input.query, limit: Math.min(Math.max(input.maxResults, 1), 40), ...(input.sort ? { sort: input.sort } : {}) }, ctx);
|
|
97
|
+
return notes.map(note => ({
|
|
98
|
+
url: note.url,
|
|
99
|
+
title: note.title === '' ? `小红书 note by ${note.author}` : note.title,
|
|
100
|
+
summary: `${note.author} · ${note.likes} likes`,
|
|
101
|
+
}));
|
|
102
|
+
}
|
|
103
|
+
/** A note's detail. The token comes from the argument, the share link, or an earlier search. */
|
|
104
|
+
export async function fetchXhsNote(input, ctx) {
|
|
105
|
+
guard(ctx);
|
|
106
|
+
const { id, token } = resolveTarget({ ...input, token: input.token ?? noteTokens.get(trimNoteId(input)) });
|
|
107
|
+
const browser = browserFor(ctx);
|
|
108
|
+
try {
|
|
109
|
+
await unwrapAnswer(await browser.evaluate(healthScript()));
|
|
110
|
+
const answer = unwrapAnswer(await browser.evaluate(noteScript({ id, token })));
|
|
111
|
+
return answer.note;
|
|
112
|
+
}
|
|
113
|
+
finally {
|
|
114
|
+
await browser.persist().catch(() => undefined);
|
|
115
|
+
await browser.close().catch(() => undefined);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
export async function fetchXhsComments(input, ctx) {
|
|
119
|
+
guard(ctx);
|
|
120
|
+
const { id, token } = resolveTarget({ ...input, token: input.token ?? noteTokens.get(trimNoteId(input)) });
|
|
121
|
+
const browser = browserFor(ctx);
|
|
122
|
+
try {
|
|
123
|
+
await unwrapAnswer(await browser.evaluate(healthScript()));
|
|
124
|
+
return unwrapAnswer(await browser.evaluate(commentsScript({ id, token, cursor: input.cursor ?? '', limit: Math.min(Math.max(input.limit ?? 20, 1), 100) })));
|
|
125
|
+
}
|
|
126
|
+
finally {
|
|
127
|
+
await browser.persist().catch(() => undefined);
|
|
128
|
+
await browser.close().catch(() => undefined);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
/** Best-effort id from whatever the caller passed, used only to look up a remembered token. */
|
|
132
|
+
function trimNoteId(input) {
|
|
133
|
+
const raw = (input.url ?? input.id ?? '').trim();
|
|
134
|
+
const match = /\/(?:explore|discovery\/item)\/([0-9a-zA-Z]+)/.exec(raw);
|
|
135
|
+
return match?.[1] ?? raw;
|
|
136
|
+
}
|
|
137
|
+
/** Accept a share link, a bare id, or an explicit token. Share links embed xsec_token. */
|
|
138
|
+
export function resolveTarget(input) {
|
|
139
|
+
const raw = (input.url ?? input.id ?? '').trim();
|
|
140
|
+
if (raw === '')
|
|
141
|
+
throw new SearchError('configuration', 'Pass a note URL or a note id.');
|
|
142
|
+
let id = raw;
|
|
143
|
+
let token = input.token ?? '';
|
|
144
|
+
if (raw.includes('/')) {
|
|
145
|
+
try {
|
|
146
|
+
const url = new URL(raw);
|
|
147
|
+
const match = /\/(?:explore|discovery\/item)\/([0-9a-zA-Z]+)/.exec(url.pathname);
|
|
148
|
+
id = match?.[1] ?? url.pathname.split('/').filter(Boolean).pop() ?? '';
|
|
149
|
+
token = token || url.searchParams.get('xsec_token') || url.searchParams.get('xsecToken') || '';
|
|
150
|
+
}
|
|
151
|
+
catch {
|
|
152
|
+
throw new SearchError('configuration', `"${raw}" is not a Xiaohongshu note URL.`);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
if (id === '')
|
|
156
|
+
throw new SearchError('configuration', `Could not read a note id from "${raw}".`);
|
|
157
|
+
if (token === '')
|
|
158
|
+
throw new SearchError('configuration', 'A note read needs its xsec_token. Search for the note first, use its share link, or pass the token explicitly.');
|
|
159
|
+
return { id, token };
|
|
160
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Xiaohongshu page scripts. The site's own webpack HTTP client is the only thing that can make
|
|
3
|
+
* these calls — a plain in-page fetch was refused with 406 — so each script locates that client
|
|
4
|
+
* inside the running page and performs one read. Module ids are never hardcoded: they are found
|
|
5
|
+
* by route literals and the registration call, so a rebuild of the site does not break this.
|
|
6
|
+
*/
|
|
7
|
+
/** Identity probe: is the runtime usable, and is the session signed in? */
|
|
8
|
+
export declare function healthScript(): string;
|
|
9
|
+
export interface SearchScriptInput {
|
|
10
|
+
query: string;
|
|
11
|
+
limit: number;
|
|
12
|
+
page: number;
|
|
13
|
+
sort: string;
|
|
14
|
+
}
|
|
15
|
+
export declare function searchScript(input: SearchScriptInput): string;
|
|
16
|
+
export declare function noteScript(input: {
|
|
17
|
+
id: string;
|
|
18
|
+
token: string;
|
|
19
|
+
}): string;
|
|
20
|
+
export declare function commentsScript(input: {
|
|
21
|
+
id: string;
|
|
22
|
+
token: string;
|
|
23
|
+
cursor: string;
|
|
24
|
+
limit: number;
|
|
25
|
+
}): string;
|
|
26
|
+
/** Set the session cookies inside the page, then prove they work before reporting success. */
|
|
27
|
+
export declare function sessionScript(input: {
|
|
28
|
+
cookies: string;
|
|
29
|
+
}): string;
|