ants-move 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +114 -0
- package/dist/index.d.ts +143 -0
- package/dist/index.js +2214 -0
- package/dist/index.js.map +1 -0
- package/package.json +52 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,2214 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// src/index.ts
|
|
4
|
+
import { realpathSync } from "fs";
|
|
5
|
+
import { fileURLToPath } from "url";
|
|
6
|
+
|
|
7
|
+
// src/cli.ts
|
|
8
|
+
import { Command, CommanderError } from "commander";
|
|
9
|
+
import { ZodError } from "zod";
|
|
10
|
+
|
|
11
|
+
// src/hn/command.ts
|
|
12
|
+
import { z } from "zod";
|
|
13
|
+
|
|
14
|
+
// src/output.ts
|
|
15
|
+
import { stripVTControlCharacters } from "util";
|
|
16
|
+
import { table } from "table";
|
|
17
|
+
function sanitizeTableCell(value) {
|
|
18
|
+
return stripVTControlCharacters(String(value)).replace(/[\u0000-\u001f\u007f-\u009f]+/g, " ").replace(/ +/g, " ").trim();
|
|
19
|
+
}
|
|
20
|
+
function renderSafeTable(rows) {
|
|
21
|
+
return table(rows.map((row) => row.map(sanitizeTableCell)));
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// src/hn/output.ts
|
|
25
|
+
function renderHackerNewsStoriesAsJson(result) {
|
|
26
|
+
return JSON.stringify(
|
|
27
|
+
{
|
|
28
|
+
ok: true,
|
|
29
|
+
data: result
|
|
30
|
+
},
|
|
31
|
+
null,
|
|
32
|
+
2
|
|
33
|
+
);
|
|
34
|
+
}
|
|
35
|
+
function renderHackerNewsStoriesAsTable(result) {
|
|
36
|
+
return renderSafeTable([
|
|
37
|
+
["id", "title", "author", "score", "comments", "time", "url"],
|
|
38
|
+
...result.items.map((item) => [
|
|
39
|
+
String(item.id),
|
|
40
|
+
item.title,
|
|
41
|
+
item.author ?? "",
|
|
42
|
+
item.score === void 0 ? "" : String(item.score),
|
|
43
|
+
item.commentCount === void 0 ? "" : String(item.commentCount),
|
|
44
|
+
item.time?.iso ?? "",
|
|
45
|
+
item.url
|
|
46
|
+
])
|
|
47
|
+
]);
|
|
48
|
+
}
|
|
49
|
+
function renderHackerNewsSearchAsJson(result) {
|
|
50
|
+
return JSON.stringify(
|
|
51
|
+
{
|
|
52
|
+
ok: true,
|
|
53
|
+
data: result
|
|
54
|
+
},
|
|
55
|
+
null,
|
|
56
|
+
2
|
|
57
|
+
);
|
|
58
|
+
}
|
|
59
|
+
function renderHackerNewsSearchAsTable(result) {
|
|
60
|
+
return renderSafeTable([
|
|
61
|
+
["id", "title", "author", "score", "comments", "time", "url"],
|
|
62
|
+
...result.items.map((item) => [
|
|
63
|
+
String(item.id),
|
|
64
|
+
item.title,
|
|
65
|
+
item.author ?? "",
|
|
66
|
+
item.score === void 0 ? "" : String(item.score),
|
|
67
|
+
item.commentCount === void 0 ? "" : String(item.commentCount),
|
|
68
|
+
item.time?.iso ?? "",
|
|
69
|
+
item.url
|
|
70
|
+
])
|
|
71
|
+
]);
|
|
72
|
+
}
|
|
73
|
+
function renderHackerNewsCommandErrorAsJson(code, message, details) {
|
|
74
|
+
return JSON.stringify(
|
|
75
|
+
{
|
|
76
|
+
ok: false,
|
|
77
|
+
error: {
|
|
78
|
+
code,
|
|
79
|
+
message,
|
|
80
|
+
...details === void 0 ? {} : { details }
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
null,
|
|
84
|
+
2
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// src/hn/types.ts
|
|
89
|
+
var HackerNewsCommandError = class extends Error {
|
|
90
|
+
constructor(code, message, exitCode, details) {
|
|
91
|
+
super(message);
|
|
92
|
+
this.code = code;
|
|
93
|
+
this.exitCode = exitCode;
|
|
94
|
+
this.details = details;
|
|
95
|
+
this.name = "HackerNewsCommandError";
|
|
96
|
+
}
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
// src/hn/service.ts
|
|
100
|
+
var SUPPORTED_SOURCES = /* @__PURE__ */ new Set(["top", "new", "best"]);
|
|
101
|
+
var HackerNewsService = class {
|
|
102
|
+
constructor(dependencies) {
|
|
103
|
+
this.dependencies = dependencies;
|
|
104
|
+
}
|
|
105
|
+
async stories(options) {
|
|
106
|
+
const source = parseSource(options.source);
|
|
107
|
+
const limit = normalizeLimit(options.limit);
|
|
108
|
+
const runtimeResult = await this.dependencies.runtime.fetchStories({
|
|
109
|
+
limit,
|
|
110
|
+
source
|
|
111
|
+
});
|
|
112
|
+
return {
|
|
113
|
+
...runtimeResult,
|
|
114
|
+
meta: {
|
|
115
|
+
limit,
|
|
116
|
+
totalItems: runtimeResult.items.length
|
|
117
|
+
}
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
async search(options) {
|
|
121
|
+
const query = normalizeQuery(options.query);
|
|
122
|
+
const limit = normalizeLimit(options.limit);
|
|
123
|
+
const sort = options.sort ?? "relevance";
|
|
124
|
+
const runtimeResult = await this.dependencies.runtime.fetchSearch({
|
|
125
|
+
limit,
|
|
126
|
+
query,
|
|
127
|
+
sort
|
|
128
|
+
});
|
|
129
|
+
return {
|
|
130
|
+
...runtimeResult,
|
|
131
|
+
meta: {
|
|
132
|
+
limit,
|
|
133
|
+
sort,
|
|
134
|
+
totalItems: runtimeResult.items.length
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
};
|
|
139
|
+
function parseSource(source) {
|
|
140
|
+
if (SUPPORTED_SOURCES.has(source)) {
|
|
141
|
+
return source;
|
|
142
|
+
}
|
|
143
|
+
throw new HackerNewsCommandError(
|
|
144
|
+
"HN_INVALID_SOURCE",
|
|
145
|
+
"Hacker News list only supports top, new, and best.",
|
|
146
|
+
2,
|
|
147
|
+
{
|
|
148
|
+
source,
|
|
149
|
+
supportedSources: [...SUPPORTED_SOURCES]
|
|
150
|
+
}
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
function normalizeLimit(limit) {
|
|
154
|
+
const value = limit ?? 30;
|
|
155
|
+
if (!Number.isInteger(value) || value < 1 || value > 100) {
|
|
156
|
+
throw new HackerNewsCommandError(
|
|
157
|
+
"HN_INVALID_LIMIT",
|
|
158
|
+
"Hacker News limit must be an integer between 1 and 100.",
|
|
159
|
+
2,
|
|
160
|
+
{
|
|
161
|
+
limit: value
|
|
162
|
+
}
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
return value;
|
|
166
|
+
}
|
|
167
|
+
function normalizeQuery(query) {
|
|
168
|
+
const value = query.trim();
|
|
169
|
+
if (value.length === 0) {
|
|
170
|
+
throw new HackerNewsCommandError(
|
|
171
|
+
"HN_INVALID_QUERY",
|
|
172
|
+
"Hacker News search query must not be empty.",
|
|
173
|
+
2
|
|
174
|
+
);
|
|
175
|
+
}
|
|
176
|
+
return value;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// src/hn/command.ts
|
|
180
|
+
var storiesOptionsSchema = z.object({
|
|
181
|
+
format: z.enum(["json", "table"]).default("json"),
|
|
182
|
+
limit: z.number().int().default(30),
|
|
183
|
+
table: z.boolean().optional()
|
|
184
|
+
});
|
|
185
|
+
var searchOptionsSchema = z.object({
|
|
186
|
+
format: z.enum(["json", "table"]).default("json"),
|
|
187
|
+
limit: z.number().int().default(30),
|
|
188
|
+
sort: z.enum(["relevance", "date"]).default("relevance"),
|
|
189
|
+
table: z.boolean().optional()
|
|
190
|
+
});
|
|
191
|
+
function registerHackerNewsCommands(program, dependencies) {
|
|
192
|
+
const hn = program.command("hn").alias("hackernews").description("Fetch Hacker News stories and search results.");
|
|
193
|
+
const service = new HackerNewsService({ runtime: dependencies.runtime });
|
|
194
|
+
for (const source of ["top", "new", "best"]) {
|
|
195
|
+
hn.command(source).option("--limit <limit>", "number of stories to fetch", parseLimit, 30).option("--format <format>", "output format", "json").option("-t, --table", "render as a table").action(async (options) => {
|
|
196
|
+
const parsed = storiesOptionsSchema.parse(applyTableShortcut(options));
|
|
197
|
+
const result = await service.stories({
|
|
198
|
+
limit: parsed.limit,
|
|
199
|
+
source
|
|
200
|
+
});
|
|
201
|
+
dependencies.stdout(
|
|
202
|
+
parsed.format === "table" ? `${renderHackerNewsStoriesAsTable(result)}
|
|
203
|
+
` : `${renderHackerNewsStoriesAsJson(result)}
|
|
204
|
+
`
|
|
205
|
+
);
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
hn.command("search").argument("<query>", "search query").option("--limit <limit>", "number of search results to fetch", parseLimit, 30).option("--sort <sort>", "search sort: relevance or date", "relevance").option("--format <format>", "output format", "json").option("-t, --table", "render as a table").action(async (query, options) => {
|
|
209
|
+
const parsed = searchOptionsSchema.parse(applyTableShortcut(options));
|
|
210
|
+
const result = await service.search({
|
|
211
|
+
limit: parsed.limit,
|
|
212
|
+
query,
|
|
213
|
+
sort: parsed.sort
|
|
214
|
+
});
|
|
215
|
+
dependencies.stdout(
|
|
216
|
+
parsed.format === "table" ? `${renderHackerNewsSearchAsTable(result)}
|
|
217
|
+
` : `${renderHackerNewsSearchAsJson(result)}
|
|
218
|
+
`
|
|
219
|
+
);
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
function handleHackerNewsCommandError(error, dependencies) {
|
|
223
|
+
if (error instanceof HackerNewsCommandError) {
|
|
224
|
+
dependencies.stderr(
|
|
225
|
+
`${renderHackerNewsCommandErrorAsJson(error.code, error.message, error.details)}
|
|
226
|
+
`
|
|
227
|
+
);
|
|
228
|
+
return error.exitCode;
|
|
229
|
+
}
|
|
230
|
+
return void 0;
|
|
231
|
+
}
|
|
232
|
+
function parseLimit(value) {
|
|
233
|
+
const parsed = Number(value);
|
|
234
|
+
if (!Number.isInteger(parsed)) {
|
|
235
|
+
throw new HackerNewsCommandError(
|
|
236
|
+
"HN_INVALID_LIMIT",
|
|
237
|
+
"Hacker News limit must be an integer between 1 and 100.",
|
|
238
|
+
2,
|
|
239
|
+
{
|
|
240
|
+
limit: value
|
|
241
|
+
}
|
|
242
|
+
);
|
|
243
|
+
}
|
|
244
|
+
return parsed;
|
|
245
|
+
}
|
|
246
|
+
function applyTableShortcut(options) {
|
|
247
|
+
if (options.table) {
|
|
248
|
+
return {
|
|
249
|
+
...options,
|
|
250
|
+
format: "table"
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
return {
|
|
254
|
+
...options,
|
|
255
|
+
format: options.format
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
// src/hn/runtime.ts
|
|
260
|
+
var FIREBASE_BASE_URL = "https://hacker-news.firebaseio.com/v0";
|
|
261
|
+
var ALGOLIA_BASE_URL = "https://hn.algolia.com/api/v1";
|
|
262
|
+
var DEFAULT_DETAIL_CONCURRENCY = 8;
|
|
263
|
+
var DEFAULT_DETAIL_RESPONSE_BYTES = 256 * 1024;
|
|
264
|
+
var DEFAULT_MAX_RESPONSE_BYTES = 5 * 1024 * 1024;
|
|
265
|
+
var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
|
|
266
|
+
var SOURCE_TO_ENDPOINT = {
|
|
267
|
+
best: "beststories",
|
|
268
|
+
new: "newstories",
|
|
269
|
+
top: "topstories"
|
|
270
|
+
};
|
|
271
|
+
function createDefaultHackerNewsRuntime(options = {}) {
|
|
272
|
+
const dependencies = {
|
|
273
|
+
detailConcurrency: normalizeDetailConcurrency(options.detailConcurrency),
|
|
274
|
+
fetch: options.fetch ?? fetch,
|
|
275
|
+
maxResponseBytes: options.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES,
|
|
276
|
+
requestTimeoutMs: options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS
|
|
277
|
+
};
|
|
278
|
+
return {
|
|
279
|
+
fetchSearch: async (request) => fetchSearch(dependencies, request),
|
|
280
|
+
fetchStories: async (request) => fetchStories(dependencies, request)
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
function normalizeDetailConcurrency(value) {
|
|
284
|
+
if (value === void 0 || !Number.isFinite(value)) {
|
|
285
|
+
return DEFAULT_DETAIL_CONCURRENCY;
|
|
286
|
+
}
|
|
287
|
+
return Math.max(1, Math.min(DEFAULT_DETAIL_CONCURRENCY, Math.floor(value)));
|
|
288
|
+
}
|
|
289
|
+
async function fetchStories(dependencies, options) {
|
|
290
|
+
const endpoint = SOURCE_TO_ENDPOINT[options.source];
|
|
291
|
+
const ids = await fetchJson(
|
|
292
|
+
dependencies,
|
|
293
|
+
`${FIREBASE_BASE_URL}/${endpoint}.json`
|
|
294
|
+
);
|
|
295
|
+
const candidateIds = [...new Set(ids.slice(0, options.limit * 2))];
|
|
296
|
+
const rawItems = await mapOrderedBounded(
|
|
297
|
+
candidateIds,
|
|
298
|
+
dependencies.detailConcurrency,
|
|
299
|
+
async (id) => fetchJson(
|
|
300
|
+
dependencies,
|
|
301
|
+
`${FIREBASE_BASE_URL}/item/${id}.json`,
|
|
302
|
+
Math.min(dependencies.maxResponseBytes, DEFAULT_DETAIL_RESPONSE_BYTES)
|
|
303
|
+
)
|
|
304
|
+
);
|
|
305
|
+
return {
|
|
306
|
+
items: rawItems.flatMap(mapFirebaseItem).slice(0, options.limit),
|
|
307
|
+
source: options.source
|
|
308
|
+
};
|
|
309
|
+
}
|
|
310
|
+
async function fetchSearch(dependencies, options) {
|
|
311
|
+
const params = new URLSearchParams({
|
|
312
|
+
hitsPerPage: String(options.limit),
|
|
313
|
+
query: options.query,
|
|
314
|
+
tags: "story"
|
|
315
|
+
});
|
|
316
|
+
const endpoint = options.sort === "date" ? "search_by_date" : "search";
|
|
317
|
+
const response = await fetchJson(
|
|
318
|
+
dependencies,
|
|
319
|
+
`${ALGOLIA_BASE_URL}/${endpoint}?${params.toString()}`
|
|
320
|
+
);
|
|
321
|
+
return {
|
|
322
|
+
items: response.hits.flatMap(mapAlgoliaHit).slice(0, options.limit),
|
|
323
|
+
query: options.query
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
async function fetchJson(dependencies, url, maxResponseBytes = dependencies.maxResponseBytes) {
|
|
327
|
+
const controller = new AbortController();
|
|
328
|
+
const timeout = setTimeout(() => controller.abort(), dependencies.requestTimeoutMs);
|
|
329
|
+
try {
|
|
330
|
+
let response;
|
|
331
|
+
try {
|
|
332
|
+
response = await dependencies.fetch(url, {
|
|
333
|
+
headers: {
|
|
334
|
+
accept: "application/json",
|
|
335
|
+
"user-agent": "ants-move/0.1 HackerNews collector"
|
|
336
|
+
},
|
|
337
|
+
signal: controller.signal
|
|
338
|
+
});
|
|
339
|
+
} catch (error) {
|
|
340
|
+
throw new HackerNewsCommandError(
|
|
341
|
+
"HN_FETCH_FAILED",
|
|
342
|
+
"Failed to fetch Hacker News data.",
|
|
343
|
+
2,
|
|
344
|
+
{
|
|
345
|
+
cause: error instanceof Error ? error.message : String(error),
|
|
346
|
+
url
|
|
347
|
+
}
|
|
348
|
+
);
|
|
349
|
+
}
|
|
350
|
+
if (!response.ok) {
|
|
351
|
+
await cancelResponseBody(response);
|
|
352
|
+
throw new HackerNewsCommandError(
|
|
353
|
+
"HN_FETCH_FAILED",
|
|
354
|
+
`Failed to fetch Hacker News data. HTTP ${response.status}.`,
|
|
355
|
+
2,
|
|
356
|
+
{ url }
|
|
357
|
+
);
|
|
358
|
+
}
|
|
359
|
+
const body = await readBoundedBody(response, maxResponseBytes, url);
|
|
360
|
+
try {
|
|
361
|
+
return JSON.parse(new TextDecoder().decode(body));
|
|
362
|
+
} catch {
|
|
363
|
+
throw new HackerNewsCommandError(
|
|
364
|
+
"HN_PARSE_FAILED",
|
|
365
|
+
"Failed to parse Hacker News JSON.",
|
|
366
|
+
2,
|
|
367
|
+
{ url }
|
|
368
|
+
);
|
|
369
|
+
}
|
|
370
|
+
} finally {
|
|
371
|
+
clearTimeout(timeout);
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
async function readBoundedBody(response, maxBytes, url) {
|
|
375
|
+
const declaredLength = Number(response.headers.get("content-length"));
|
|
376
|
+
if (Number.isFinite(declaredLength) && declaredLength > maxBytes) {
|
|
377
|
+
await cancelResponseBody(response);
|
|
378
|
+
throw responseTooLarge(url, maxBytes);
|
|
379
|
+
}
|
|
380
|
+
const reader = response.body?.getReader();
|
|
381
|
+
if (!reader) {
|
|
382
|
+
return new Uint8Array();
|
|
383
|
+
}
|
|
384
|
+
const chunks = [];
|
|
385
|
+
let totalBytes = 0;
|
|
386
|
+
while (true) {
|
|
387
|
+
const { done, value } = await reader.read();
|
|
388
|
+
if (done) break;
|
|
389
|
+
totalBytes += value.byteLength;
|
|
390
|
+
if (totalBytes > maxBytes) {
|
|
391
|
+
await reader.cancel().catch(() => void 0);
|
|
392
|
+
throw responseTooLarge(url, maxBytes);
|
|
393
|
+
}
|
|
394
|
+
chunks.push(value);
|
|
395
|
+
}
|
|
396
|
+
const body = new Uint8Array(totalBytes);
|
|
397
|
+
let offset = 0;
|
|
398
|
+
for (const chunk of chunks) {
|
|
399
|
+
body.set(chunk, offset);
|
|
400
|
+
offset += chunk.byteLength;
|
|
401
|
+
}
|
|
402
|
+
return body;
|
|
403
|
+
}
|
|
404
|
+
async function cancelResponseBody(response) {
|
|
405
|
+
await response.body?.cancel().catch(() => void 0);
|
|
406
|
+
}
|
|
407
|
+
function responseTooLarge(url, maxBytes) {
|
|
408
|
+
return new HackerNewsCommandError(
|
|
409
|
+
"HN_RESPONSE_TOO_LARGE",
|
|
410
|
+
"Hacker News response exceeded the configured size limit.",
|
|
411
|
+
2,
|
|
412
|
+
{ maxBytes, url }
|
|
413
|
+
);
|
|
414
|
+
}
|
|
415
|
+
async function mapOrderedBounded(values, concurrency, map) {
|
|
416
|
+
if (values.length === 0) {
|
|
417
|
+
return [];
|
|
418
|
+
}
|
|
419
|
+
const results = new Array(values.length);
|
|
420
|
+
let nextIndex = 0;
|
|
421
|
+
let firstError;
|
|
422
|
+
const worker = async () => {
|
|
423
|
+
while (firstError === void 0) {
|
|
424
|
+
const index = nextIndex;
|
|
425
|
+
nextIndex += 1;
|
|
426
|
+
if (index >= values.length) return;
|
|
427
|
+
try {
|
|
428
|
+
results[index] = await map(values[index], index);
|
|
429
|
+
} catch (error) {
|
|
430
|
+
firstError ??= error;
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
};
|
|
434
|
+
const workerCount = Math.max(1, Math.min(
|
|
435
|
+
Math.floor(concurrency),
|
|
436
|
+
values.length
|
|
437
|
+
));
|
|
438
|
+
await Promise.all(Array.from({ length: workerCount }, worker));
|
|
439
|
+
if (firstError !== void 0) {
|
|
440
|
+
throw firstError;
|
|
441
|
+
}
|
|
442
|
+
return results;
|
|
443
|
+
}
|
|
444
|
+
function mapFirebaseItem(item) {
|
|
445
|
+
if (!item || item.deleted || item.dead || item.type !== "story" || !item.id || !item.title) {
|
|
446
|
+
return [];
|
|
447
|
+
}
|
|
448
|
+
return [
|
|
449
|
+
createItem({
|
|
450
|
+
id: item.id,
|
|
451
|
+
title: item.title,
|
|
452
|
+
url: item.url ?? `https://news.ycombinator.com/item?id=${item.id}`,
|
|
453
|
+
...optional("author", item.by),
|
|
454
|
+
...optional("commentCount", item.descendants),
|
|
455
|
+
...optional("score", item.score),
|
|
456
|
+
...optional("text", item.text),
|
|
457
|
+
...optional("timeSeconds", item.time)
|
|
458
|
+
})
|
|
459
|
+
];
|
|
460
|
+
}
|
|
461
|
+
function mapAlgoliaHit(hit) {
|
|
462
|
+
const id = hit.story_id ?? parseNumericId(hit.objectID);
|
|
463
|
+
if (!id || !hit.title) {
|
|
464
|
+
return [];
|
|
465
|
+
}
|
|
466
|
+
return [
|
|
467
|
+
createItem({
|
|
468
|
+
id,
|
|
469
|
+
title: hit.title,
|
|
470
|
+
url: hit.url ?? `https://news.ycombinator.com/item?id=${id}`,
|
|
471
|
+
...optional("author", hit.author),
|
|
472
|
+
...optional("commentCount", hit.num_comments),
|
|
473
|
+
...optional("score", hit.points),
|
|
474
|
+
...optional("timeSeconds", hit.created_at_i)
|
|
475
|
+
})
|
|
476
|
+
];
|
|
477
|
+
}
|
|
478
|
+
function createItem(input) {
|
|
479
|
+
const item = {
|
|
480
|
+
id: input.id,
|
|
481
|
+
title: input.title,
|
|
482
|
+
type: "story",
|
|
483
|
+
url: input.url
|
|
484
|
+
};
|
|
485
|
+
if (input.author !== void 0) item.author = input.author;
|
|
486
|
+
if (input.commentCount !== void 0) item.commentCount = input.commentCount;
|
|
487
|
+
if (input.score !== void 0) item.score = input.score;
|
|
488
|
+
if (input.text !== void 0) item.text = input.text;
|
|
489
|
+
if (input.timeSeconds !== void 0) {
|
|
490
|
+
item.time = {
|
|
491
|
+
iso: new Date(input.timeSeconds * 1e3).toISOString(),
|
|
492
|
+
seconds: input.timeSeconds
|
|
493
|
+
};
|
|
494
|
+
}
|
|
495
|
+
return item;
|
|
496
|
+
}
|
|
497
|
+
function optional(key, value) {
|
|
498
|
+
return value === void 0 ? {} : { [key]: value };
|
|
499
|
+
}
|
|
500
|
+
function parseNumericId(value) {
|
|
501
|
+
if (!value || !/^\d+$/.test(value)) {
|
|
502
|
+
return void 0;
|
|
503
|
+
}
|
|
504
|
+
return Number(value);
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
// src/36kr/command.ts
|
|
508
|
+
import { z as z2 } from "zod";
|
|
509
|
+
|
|
510
|
+
// src/36kr/output.ts
|
|
511
|
+
function renderKr36ArticleAsJson(article) {
|
|
512
|
+
return JSON.stringify(
|
|
513
|
+
{
|
|
514
|
+
ok: true,
|
|
515
|
+
data: article
|
|
516
|
+
},
|
|
517
|
+
null,
|
|
518
|
+
2
|
|
519
|
+
);
|
|
520
|
+
}
|
|
521
|
+
function renderKr36InformationListAsJson(list) {
|
|
522
|
+
return JSON.stringify(
|
|
523
|
+
{
|
|
524
|
+
ok: true,
|
|
525
|
+
data: list
|
|
526
|
+
},
|
|
527
|
+
null,
|
|
528
|
+
2
|
|
529
|
+
);
|
|
530
|
+
}
|
|
531
|
+
function renderKr36InformationListAsTable(list) {
|
|
532
|
+
return renderSafeTable([
|
|
533
|
+
["id", "title", "author", "publishTime", "url"],
|
|
534
|
+
...list.items.map((item) => [
|
|
535
|
+
String(item.id),
|
|
536
|
+
item.title,
|
|
537
|
+
item.authorName ?? "",
|
|
538
|
+
item.publishTime?.local ?? "",
|
|
539
|
+
item.url
|
|
540
|
+
])
|
|
541
|
+
]);
|
|
542
|
+
}
|
|
543
|
+
function renderKr36CommandErrorAsJson(code, message, details) {
|
|
544
|
+
return JSON.stringify(
|
|
545
|
+
{
|
|
546
|
+
ok: false,
|
|
547
|
+
error: {
|
|
548
|
+
code,
|
|
549
|
+
message,
|
|
550
|
+
...details === void 0 ? {} : { details }
|
|
551
|
+
}
|
|
552
|
+
},
|
|
553
|
+
null,
|
|
554
|
+
2
|
|
555
|
+
);
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
// src/36kr/types.ts
|
|
559
|
+
var Kr36CommandError = class extends Error {
|
|
560
|
+
constructor(code, message, exitCode, details) {
|
|
561
|
+
super(message);
|
|
562
|
+
this.code = code;
|
|
563
|
+
this.exitCode = exitCode;
|
|
564
|
+
this.details = details;
|
|
565
|
+
this.name = "Kr36CommandError";
|
|
566
|
+
}
|
|
567
|
+
};
|
|
568
|
+
|
|
569
|
+
// src/36kr/service.ts
|
|
570
|
+
var KR36_BROWSER_HEADERS = {
|
|
571
|
+
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
|
|
572
|
+
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
|
573
|
+
"Cache-Control": "no-cache",
|
|
574
|
+
Pragma: "no-cache",
|
|
575
|
+
Referer: "https://36kr.com/",
|
|
576
|
+
"Sec-Fetch-Dest": "document",
|
|
577
|
+
"Sec-Fetch-Mode": "navigate",
|
|
578
|
+
"Sec-Fetch-Site": "same-origin",
|
|
579
|
+
"Sec-Fetch-User": "?1",
|
|
580
|
+
"Upgrade-Insecure-Requests": "1",
|
|
581
|
+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
|
|
582
|
+
};
|
|
583
|
+
var KR36_INFORMATION_ENDPOINT = "https://gateway.36kr.com/api/mis/nav/ifm/subNav/flow";
|
|
584
|
+
var KR36_ARTICLE_MAX_RETAINED_BYTES = 20 * 1024 * 1024;
|
|
585
|
+
var KR36_INFORMATION_MAX_RETAINED_BYTES = 5 * 1024 * 1024;
|
|
586
|
+
var KR36_INFORMATION_PAGE_SIZE = 30;
|
|
587
|
+
var KR36_SUPPORTED_CHANNELS = /* @__PURE__ */ new Set(["AI", "technology"]);
|
|
588
|
+
var KR36_JSON_HEADERS = {
|
|
589
|
+
Accept: "application/json, text/plain, */*",
|
|
590
|
+
"Accept-Language": KR36_BROWSER_HEADERS["Accept-Language"],
|
|
591
|
+
"Content-Type": "application/json;charset=UTF-8",
|
|
592
|
+
Origin: "https://36kr.com",
|
|
593
|
+
"User-Agent": KR36_BROWSER_HEADERS["User-Agent"]
|
|
594
|
+
};
|
|
595
|
+
var Kr36ArticleService = class {
|
|
596
|
+
constructor(runtime) {
|
|
597
|
+
this.runtime = runtime;
|
|
598
|
+
}
|
|
599
|
+
async getArticle(articleId) {
|
|
600
|
+
assertArticleId(articleId);
|
|
601
|
+
const request = buildKr36ArticleRequest(articleId);
|
|
602
|
+
const html = await this.runtime.fetchArticleHtml(request);
|
|
603
|
+
const initialState = parseInitialState(html);
|
|
604
|
+
const articleDetail = initialState.articleDetail;
|
|
605
|
+
const rawArticle = articleDetail?.articleDetailData?.data;
|
|
606
|
+
if (!rawArticle) {
|
|
607
|
+
throw new Kr36CommandError(
|
|
608
|
+
"KR36_PARSE_ERROR",
|
|
609
|
+
"36kr article data was not found in window.initialState.",
|
|
610
|
+
2
|
|
611
|
+
);
|
|
612
|
+
}
|
|
613
|
+
const contentHtml = rawArticle.widgetContent ?? "";
|
|
614
|
+
const recommend = articleDetail?.articleRecommendData ?? {};
|
|
615
|
+
const publishTime = rawArticle.publishTime ?? 0;
|
|
616
|
+
const article = {
|
|
617
|
+
author: buildAuthor(rawArticle, recommend),
|
|
618
|
+
content: {
|
|
619
|
+
html: contentHtml,
|
|
620
|
+
paragraphs: extractParagraphs(contentHtml)
|
|
621
|
+
},
|
|
622
|
+
id: String(rawArticle.itemId ?? articleId),
|
|
623
|
+
imageSources: rawArticle.imgSources ?? [],
|
|
624
|
+
images: extractImages(contentHtml),
|
|
625
|
+
latestArticles: (articleDetail?.latestArticle?.articleLatestList ?? []).map((item) => ({
|
|
626
|
+
id: item.id,
|
|
627
|
+
title: item.title
|
|
628
|
+
})),
|
|
629
|
+
newestArticles: (recommend.newestItemList ?? []).map((item) => {
|
|
630
|
+
const listItem = {
|
|
631
|
+
id: item.itemId,
|
|
632
|
+
title: item.itemTitle
|
|
633
|
+
};
|
|
634
|
+
if (item.itemContent !== void 0) {
|
|
635
|
+
listItem.content = item.itemContent;
|
|
636
|
+
}
|
|
637
|
+
if (item.publishTime !== void 0) {
|
|
638
|
+
listItem.publishTime = item.publishTime;
|
|
639
|
+
}
|
|
640
|
+
if (item.itemRoute !== void 0) {
|
|
641
|
+
listItem.route = item.itemRoute;
|
|
642
|
+
}
|
|
643
|
+
return listItem;
|
|
644
|
+
}),
|
|
645
|
+
organizations: articleDetail?.organArticleData?.data?.organizationList ?? [],
|
|
646
|
+
publishTime: {
|
|
647
|
+
iso: publishTime ? new Date(publishTime).toISOString() : "",
|
|
648
|
+
local: publishTime ? formatChinaTime(publishTime) : "",
|
|
649
|
+
ms: publishTime
|
|
650
|
+
},
|
|
651
|
+
relatedArticles: (recommend.relateArticleList ?? []).map((item) => {
|
|
652
|
+
const listItem = {
|
|
653
|
+
id: item.itemId,
|
|
654
|
+
title: item.widgetTitle
|
|
655
|
+
};
|
|
656
|
+
const author = item.author ?? item.authorName;
|
|
657
|
+
if (author !== void 0) {
|
|
658
|
+
listItem.author = author;
|
|
659
|
+
}
|
|
660
|
+
if (item.widgetImage !== void 0) {
|
|
661
|
+
listItem.image = item.widgetImage;
|
|
662
|
+
}
|
|
663
|
+
if (item.route !== void 0) {
|
|
664
|
+
listItem.route = item.route;
|
|
665
|
+
}
|
|
666
|
+
return listItem;
|
|
667
|
+
}),
|
|
668
|
+
request,
|
|
669
|
+
stats: buildStats(articleDetail?.likeCount, articleDetail?.favoriteCount, recommend),
|
|
670
|
+
summary: rawArticle.summary ?? "",
|
|
671
|
+
title: rawArticle.widgetTitle ?? "",
|
|
672
|
+
url: request.url
|
|
673
|
+
};
|
|
674
|
+
if (rawArticle.companyCertifyNick !== void 0) {
|
|
675
|
+
article.companyCertifyNick = rawArticle.companyCertifyNick;
|
|
676
|
+
}
|
|
677
|
+
if (rawArticle.popinImage !== void 0) {
|
|
678
|
+
article.coverImage = rawArticle.popinImage;
|
|
679
|
+
}
|
|
680
|
+
if (recommend.nextItem) {
|
|
681
|
+
article.nextArticle = {
|
|
682
|
+
id: recommend.nextItem.itemId,
|
|
683
|
+
title: recommend.nextItem.itemTitle
|
|
684
|
+
};
|
|
685
|
+
if (recommend.nextItem.itemContent !== void 0) {
|
|
686
|
+
article.nextArticle.content = recommend.nextItem.itemContent;
|
|
687
|
+
}
|
|
688
|
+
if (recommend.nextItem.publishTime !== void 0) {
|
|
689
|
+
article.nextArticle.publishTime = recommend.nextItem.publishTime;
|
|
690
|
+
}
|
|
691
|
+
if (recommend.nextItem.itemRoute !== void 0) {
|
|
692
|
+
article.nextArticle.route = recommend.nextItem.itemRoute;
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
if (rawArticle.sourceType !== void 0) {
|
|
696
|
+
article.sourceType = rawArticle.sourceType;
|
|
697
|
+
}
|
|
698
|
+
const retainedBytes = Buffer.byteLength(JSON.stringify(article), "utf8");
|
|
699
|
+
if (retainedBytes > KR36_ARTICLE_MAX_RETAINED_BYTES) {
|
|
700
|
+
throw new Kr36CommandError(
|
|
701
|
+
"KR36_RESOURCE_LIMIT",
|
|
702
|
+
"36kr article result exceeded the configured size limit.",
|
|
703
|
+
2,
|
|
704
|
+
{
|
|
705
|
+
maxRetainedBytes: KR36_ARTICLE_MAX_RETAINED_BYTES,
|
|
706
|
+
retainedBytes
|
|
707
|
+
}
|
|
708
|
+
);
|
|
709
|
+
}
|
|
710
|
+
return article;
|
|
711
|
+
}
|
|
712
|
+
async getInformationList(options) {
|
|
713
|
+
const channel = parseInformationChannel(options.channel);
|
|
714
|
+
const pageLimit = normalizePages(options.pages);
|
|
715
|
+
const firstPageRequest = buildKr36InformationFirstPageRequest(channel);
|
|
716
|
+
const firstPageHtml = await this.runtime.fetchArticleHtml(firstPageRequest);
|
|
717
|
+
const firstPageState = parseInitialState(firstPageHtml);
|
|
718
|
+
const firstPageList = firstPageState.information?.informationList;
|
|
719
|
+
if (!firstPageList) {
|
|
720
|
+
throw new Kr36CommandError(
|
|
721
|
+
"KR36_PARSE_ERROR",
|
|
722
|
+
"36kr information list data was not found in window.initialState.",
|
|
723
|
+
2
|
|
724
|
+
);
|
|
725
|
+
}
|
|
726
|
+
const firstPageItems = mapInformationPageItems(firstPageList.itemList ?? []);
|
|
727
|
+
let retainedBytes = addInformationRetainedBytes(0, firstPageItems);
|
|
728
|
+
const items = [...firstPageItems];
|
|
729
|
+
let pageCallback = firstPageList.pageCallback ?? "";
|
|
730
|
+
let hasNextPage = firstPageList.hasNextPage ?? 0;
|
|
731
|
+
let fetchedPages = 1;
|
|
732
|
+
while (fetchedPages < pageLimit && hasNextPage && pageCallback) {
|
|
733
|
+
const request = buildKr36InformationNextPageRequest(channel, pageCallback);
|
|
734
|
+
const response = parseInformationFlowResponse(await this.runtime.fetchJson(request));
|
|
735
|
+
const nextList = response.data ?? {};
|
|
736
|
+
const nextItems = mapInformationPageItems(nextList.itemList ?? []);
|
|
737
|
+
retainedBytes = addInformationRetainedBytes(retainedBytes, nextItems);
|
|
738
|
+
items.push(...nextItems);
|
|
739
|
+
pageCallback = nextList.pageCallback ?? "";
|
|
740
|
+
hasNextPage = nextList.hasNextPage ?? 0;
|
|
741
|
+
fetchedPages += 1;
|
|
742
|
+
}
|
|
743
|
+
return {
|
|
744
|
+
channel,
|
|
745
|
+
items,
|
|
746
|
+
meta: {
|
|
747
|
+
fetchedPages,
|
|
748
|
+
hasNextPage,
|
|
749
|
+
nextPageCallback: pageCallback,
|
|
750
|
+
pageSize: KR36_INFORMATION_PAGE_SIZE,
|
|
751
|
+
totalItems: items.length
|
|
752
|
+
},
|
|
753
|
+
request: {
|
|
754
|
+
firstPage: firstPageRequest,
|
|
755
|
+
nextPageEndpoint: KR36_INFORMATION_ENDPOINT
|
|
756
|
+
}
|
|
757
|
+
};
|
|
758
|
+
}
|
|
759
|
+
};
|
|
760
|
+
function buildKr36ArticleUrl(articleId) {
|
|
761
|
+
assertArticleId(articleId);
|
|
762
|
+
return `https://36kr.com/p/${articleId}?f=rss`;
|
|
763
|
+
}
|
|
764
|
+
function buildKr36ArticleRequest(articleId) {
|
|
765
|
+
return {
|
|
766
|
+
headers: KR36_BROWSER_HEADERS,
|
|
767
|
+
url: buildKr36ArticleUrl(articleId)
|
|
768
|
+
};
|
|
769
|
+
}
|
|
770
|
+
function buildKr36InformationFirstPageRequest(channel) {
|
|
771
|
+
return {
|
|
772
|
+
headers: KR36_BROWSER_HEADERS,
|
|
773
|
+
url: `https://36kr.com/information/${channel}/`
|
|
774
|
+
};
|
|
775
|
+
}
|
|
776
|
+
function buildKr36InformationNextPageRequest(channel, pageCallback, timestamp = Date.now()) {
|
|
777
|
+
return {
|
|
778
|
+
body: {
|
|
779
|
+
partner_id: "web",
|
|
780
|
+
timestamp,
|
|
781
|
+
param: {
|
|
782
|
+
subnavType: 1,
|
|
783
|
+
subnavNick: channel,
|
|
784
|
+
pageSize: KR36_INFORMATION_PAGE_SIZE,
|
|
785
|
+
pageEvent: 1,
|
|
786
|
+
pageCallback,
|
|
787
|
+
siteId: 1,
|
|
788
|
+
platformId: 2
|
|
789
|
+
}
|
|
790
|
+
},
|
|
791
|
+
headers: {
|
|
792
|
+
...KR36_JSON_HEADERS,
|
|
793
|
+
Referer: `https://36kr.com/information/${channel}/`
|
|
794
|
+
},
|
|
795
|
+
url: KR36_INFORMATION_ENDPOINT
|
|
796
|
+
};
|
|
797
|
+
}
|
|
798
|
+
function assertArticleId(articleId) {
|
|
799
|
+
if (!/^\d+$/.test(articleId)) {
|
|
800
|
+
throw new Kr36CommandError(
|
|
801
|
+
"KR36_INVALID_ARTICLE_ID",
|
|
802
|
+
"36kr article id must contain digits only.",
|
|
803
|
+
2,
|
|
804
|
+
{
|
|
805
|
+
articleId
|
|
806
|
+
}
|
|
807
|
+
);
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
function parseInformationChannel(channel) {
|
|
811
|
+
if (KR36_SUPPORTED_CHANNELS.has(channel)) {
|
|
812
|
+
return channel;
|
|
813
|
+
}
|
|
814
|
+
throw new Kr36CommandError(
|
|
815
|
+
"KR36_INVALID_CHANNEL",
|
|
816
|
+
"36kr list only supports AI and technology channels.",
|
|
817
|
+
2,
|
|
818
|
+
{
|
|
819
|
+
channel,
|
|
820
|
+
supportedChannels: [...KR36_SUPPORTED_CHANNELS]
|
|
821
|
+
}
|
|
822
|
+
);
|
|
823
|
+
}
|
|
824
|
+
function normalizePages(pages) {
|
|
825
|
+
const value = pages ?? 1;
|
|
826
|
+
if (!Number.isInteger(value) || value < 1 || value > 20) {
|
|
827
|
+
throw new Kr36CommandError(
|
|
828
|
+
"KR36_INVALID_PAGES",
|
|
829
|
+
"36kr list pages must be an integer between 1 and 20.",
|
|
830
|
+
2,
|
|
831
|
+
{
|
|
832
|
+
pages: value
|
|
833
|
+
}
|
|
834
|
+
);
|
|
835
|
+
}
|
|
836
|
+
return value;
|
|
837
|
+
}
|
|
838
|
+
function parseInitialState(html) {
|
|
839
|
+
const match = /window\.initialState=(\{[\s\S]*?\})\s*;?\s*<\/script>/.exec(html);
|
|
840
|
+
if (!match?.[1]) {
|
|
841
|
+
throw new Kr36CommandError(
|
|
842
|
+
"KR36_PARSE_ERROR",
|
|
843
|
+
"window.initialState was not found in the 36kr article page.",
|
|
844
|
+
2
|
|
845
|
+
);
|
|
846
|
+
}
|
|
847
|
+
try {
|
|
848
|
+
return JSON.parse(match[1]);
|
|
849
|
+
} catch (error) {
|
|
850
|
+
throw new Kr36CommandError(
|
|
851
|
+
"KR36_PARSE_ERROR",
|
|
852
|
+
"Failed to parse window.initialState from the 36kr article page.",
|
|
853
|
+
2,
|
|
854
|
+
{
|
|
855
|
+
cause: error instanceof Error ? error.message : String(error)
|
|
856
|
+
}
|
|
857
|
+
);
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
function extractImages(html) {
|
|
861
|
+
return [...html.matchAll(/<img\b[^>]*>/gi)].map((match, index) => {
|
|
862
|
+
const tag = match[0];
|
|
863
|
+
return {
|
|
864
|
+
index: index + 1,
|
|
865
|
+
size: getAttribute(tag, "data-img-size-val"),
|
|
866
|
+
url: getAttribute(tag, "src") ?? ""
|
|
867
|
+
};
|
|
868
|
+
}).filter((image) => image.url);
|
|
869
|
+
}
|
|
870
|
+
function parseInformationFlowResponse(jsonText) {
|
|
871
|
+
try {
|
|
872
|
+
const response = JSON.parse(jsonText);
|
|
873
|
+
if (response.code !== 0) {
|
|
874
|
+
throw new Kr36CommandError(
|
|
875
|
+
"KR36_REQUEST_FAILED",
|
|
876
|
+
response.msg || "36kr information flow request failed.",
|
|
877
|
+
1,
|
|
878
|
+
{
|
|
879
|
+
code: response.code
|
|
880
|
+
}
|
|
881
|
+
);
|
|
882
|
+
}
|
|
883
|
+
return response;
|
|
884
|
+
} catch (error) {
|
|
885
|
+
if (error instanceof Kr36CommandError) {
|
|
886
|
+
throw error;
|
|
887
|
+
}
|
|
888
|
+
throw new Kr36CommandError(
|
|
889
|
+
"KR36_PARSE_ERROR",
|
|
890
|
+
"Failed to parse 36kr information flow response.",
|
|
891
|
+
2,
|
|
892
|
+
{
|
|
893
|
+
cause: error instanceof Error ? error.message : String(error)
|
|
894
|
+
}
|
|
895
|
+
);
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
function mapInformationItems(items) {
|
|
899
|
+
return items.map((item) => {
|
|
900
|
+
const material = item.templateMaterial ?? {};
|
|
901
|
+
const mapped = {
|
|
902
|
+
id: item.itemId,
|
|
903
|
+
title: material.widgetTitle ?? "",
|
|
904
|
+
url: `https://36kr.com/p/${item.itemId}`
|
|
905
|
+
};
|
|
906
|
+
if (material.authorName !== void 0) {
|
|
907
|
+
mapped.authorName = material.authorName;
|
|
908
|
+
}
|
|
909
|
+
if (material.authorRoute !== void 0) {
|
|
910
|
+
mapped.authorRoute = material.authorRoute;
|
|
911
|
+
}
|
|
912
|
+
if (material.widgetImage !== void 0) {
|
|
913
|
+
mapped.image = material.widgetImage;
|
|
914
|
+
}
|
|
915
|
+
if (material.publishTime !== void 0) {
|
|
916
|
+
mapped.publishTime = {
|
|
917
|
+
iso: new Date(material.publishTime).toISOString(),
|
|
918
|
+
local: formatChinaTime(material.publishTime),
|
|
919
|
+
ms: material.publishTime
|
|
920
|
+
};
|
|
921
|
+
}
|
|
922
|
+
if (item.route !== void 0) {
|
|
923
|
+
mapped.route = item.route;
|
|
924
|
+
}
|
|
925
|
+
if (material.summary !== void 0) {
|
|
926
|
+
mapped.summary = material.summary;
|
|
927
|
+
}
|
|
928
|
+
return mapped;
|
|
929
|
+
});
|
|
930
|
+
}
|
|
931
|
+
function mapInformationPageItems(items) {
|
|
932
|
+
if (items.length > KR36_INFORMATION_PAGE_SIZE) {
|
|
933
|
+
throw new Kr36CommandError(
|
|
934
|
+
"KR36_RESOURCE_LIMIT",
|
|
935
|
+
"36kr information response exceeded the configured item limit.",
|
|
936
|
+
2,
|
|
937
|
+
{
|
|
938
|
+
maxItemsPerPage: KR36_INFORMATION_PAGE_SIZE,
|
|
939
|
+
receivedItems: items.length
|
|
940
|
+
}
|
|
941
|
+
);
|
|
942
|
+
}
|
|
943
|
+
return mapInformationItems(items);
|
|
944
|
+
}
|
|
945
|
+
function addInformationRetainedBytes(retainedBytes, items) {
|
|
946
|
+
const nextRetainedBytes = retainedBytes + Buffer.byteLength(JSON.stringify(items), "utf8");
|
|
947
|
+
if (nextRetainedBytes > KR36_INFORMATION_MAX_RETAINED_BYTES) {
|
|
948
|
+
throw new Kr36CommandError(
|
|
949
|
+
"KR36_RESOURCE_LIMIT",
|
|
950
|
+
"36kr information results exceeded the configured size limit.",
|
|
951
|
+
2,
|
|
952
|
+
{
|
|
953
|
+
maxRetainedBytes: KR36_INFORMATION_MAX_RETAINED_BYTES,
|
|
954
|
+
retainedBytes: nextRetainedBytes
|
|
955
|
+
}
|
|
956
|
+
);
|
|
957
|
+
}
|
|
958
|
+
return nextRetainedBytes;
|
|
959
|
+
}
|
|
960
|
+
function extractParagraphs(html) {
|
|
961
|
+
return html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, "\n").split(/\n+/).map((value) => decodeHtmlEntities(value).trim()).filter(Boolean);
|
|
962
|
+
}
|
|
963
|
+
function getAttribute(tag, name) {
|
|
964
|
+
const match = new RegExp(`${name}=["']([^"']+)["']`, "i").exec(tag);
|
|
965
|
+
return match?.[1] ?? null;
|
|
966
|
+
}
|
|
967
|
+
function decodeHtmlEntities(value) {
|
|
968
|
+
return value.replace(/ /g, " ").replace(/"/g, '"').replace(/'/g, "'").replace(/</g, "<").replace(/>/g, ">").replace(/&/g, "&");
|
|
969
|
+
}
|
|
970
|
+
function buildStats(likeCount, favoriteCount, recommend) {
|
|
971
|
+
return {
|
|
972
|
+
authorArticleCount: recommend.statArticle ?? 0,
|
|
973
|
+
collect: recommend.statCollect ?? 0,
|
|
974
|
+
comment: recommend.statComment ?? 0,
|
|
975
|
+
favoriteCount: favoriteCount ?? 0,
|
|
976
|
+
likeCount: likeCount ?? 0,
|
|
977
|
+
praise: recommend.statPraise ?? 0
|
|
978
|
+
};
|
|
979
|
+
}
|
|
980
|
+
function buildAuthor(rawArticle, recommend) {
|
|
981
|
+
const author = {
|
|
982
|
+
name: rawArticle.author ?? recommend.authorName ?? ""
|
|
983
|
+
};
|
|
984
|
+
const face = rawArticle.authorFace ?? recommend.authorFace;
|
|
985
|
+
if (face !== void 0) {
|
|
986
|
+
author.face = face;
|
|
987
|
+
}
|
|
988
|
+
if (rawArticle.authorId !== void 0) {
|
|
989
|
+
author.id = rawArticle.authorId;
|
|
990
|
+
}
|
|
991
|
+
if (rawArticle.authorRoute !== void 0) {
|
|
992
|
+
author.route = rawArticle.authorRoute;
|
|
993
|
+
}
|
|
994
|
+
if (recommend.authorSummary !== void 0) {
|
|
995
|
+
author.summary = recommend.authorSummary;
|
|
996
|
+
}
|
|
997
|
+
if (recommend.authorTitle !== void 0) {
|
|
998
|
+
author.title = recommend.authorTitle;
|
|
999
|
+
}
|
|
1000
|
+
return author;
|
|
1001
|
+
}
|
|
1002
|
+
function formatChinaTime(timestampMs) {
|
|
1003
|
+
const formatter = new Intl.DateTimeFormat("sv-SE", {
|
|
1004
|
+
day: "2-digit",
|
|
1005
|
+
hour: "2-digit",
|
|
1006
|
+
hour12: false,
|
|
1007
|
+
minute: "2-digit",
|
|
1008
|
+
month: "2-digit",
|
|
1009
|
+
second: "2-digit",
|
|
1010
|
+
timeZone: "Asia/Shanghai",
|
|
1011
|
+
year: "numeric"
|
|
1012
|
+
});
|
|
1013
|
+
return formatter.format(new Date(timestampMs));
|
|
1014
|
+
}
|
|
1015
|
+
|
|
1016
|
+
// src/36kr/command.ts
|
|
1017
|
+
var articleOptionsSchema = z2.object({
|
|
1018
|
+
format: z2.enum(["json"]).default("json")
|
|
1019
|
+
});
|
|
1020
|
+
var informationListOptionsSchema = z2.object({
|
|
1021
|
+
format: z2.enum(["json", "table"]).default("json"),
|
|
1022
|
+
pages: z2.number().int().min(1).max(20).default(1),
|
|
1023
|
+
table: z2.boolean().optional()
|
|
1024
|
+
});
|
|
1025
|
+
function registerKr36Commands(program, dependencies) {
|
|
1026
|
+
const kr36 = program.command("36kr").description("Fetch and parse 36kr content.");
|
|
1027
|
+
const articleService = new Kr36ArticleService(dependencies.runtime);
|
|
1028
|
+
kr36.command("article").argument("<articleId>", "36kr article id").option("--format <format>", "output format", "json").action(async (articleId, options) => {
|
|
1029
|
+
articleOptionsSchema.parse(options);
|
|
1030
|
+
const article = await articleService.getArticle(articleId);
|
|
1031
|
+
dependencies.stdout(`${renderKr36ArticleAsJson(article)}
|
|
1032
|
+
`);
|
|
1033
|
+
});
|
|
1034
|
+
kr36.command("list").argument("<channel>", "36kr information channel: AI or technology").option("--pages <pages>", "number of pages to fetch", parsePages, 1).option("--format <format>", "output format", "json").option("-t, --table", "render as a table").action(async (channel, options) => {
|
|
1035
|
+
const parsed = informationListOptionsSchema.parse(applyTableShortcut2(options));
|
|
1036
|
+
const list = await articleService.getInformationList({
|
|
1037
|
+
channel,
|
|
1038
|
+
pages: parsed.pages
|
|
1039
|
+
});
|
|
1040
|
+
dependencies.stdout(
|
|
1041
|
+
parsed.format === "table" ? `${renderKr36InformationListAsTable(list)}
|
|
1042
|
+
` : `${renderKr36InformationListAsJson(list)}
|
|
1043
|
+
`
|
|
1044
|
+
);
|
|
1045
|
+
});
|
|
1046
|
+
}
|
|
1047
|
+
function handleKr36CommandError(error, dependencies) {
|
|
1048
|
+
if (error instanceof Kr36CommandError) {
|
|
1049
|
+
dependencies.stderr(
|
|
1050
|
+
`${renderKr36CommandErrorAsJson(error.code, error.message, error.details)}
|
|
1051
|
+
`
|
|
1052
|
+
);
|
|
1053
|
+
return error.exitCode;
|
|
1054
|
+
}
|
|
1055
|
+
return void 0;
|
|
1056
|
+
}
|
|
1057
|
+
function parsePages(value) {
|
|
1058
|
+
const parsed = Number(value);
|
|
1059
|
+
if (!Number.isInteger(parsed)) {
|
|
1060
|
+
throw new Kr36CommandError(
|
|
1061
|
+
"KR36_INVALID_PAGES",
|
|
1062
|
+
"36kr list pages must be an integer between 1 and 20.",
|
|
1063
|
+
2,
|
|
1064
|
+
{
|
|
1065
|
+
pages: value
|
|
1066
|
+
}
|
|
1067
|
+
);
|
|
1068
|
+
}
|
|
1069
|
+
return parsed;
|
|
1070
|
+
}
|
|
1071
|
+
function applyTableShortcut2(options) {
|
|
1072
|
+
if (options.table) {
|
|
1073
|
+
return {
|
|
1074
|
+
...options,
|
|
1075
|
+
format: "table"
|
|
1076
|
+
};
|
|
1077
|
+
}
|
|
1078
|
+
return {
|
|
1079
|
+
...options,
|
|
1080
|
+
format: options.format
|
|
1081
|
+
};
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
// src/36kr/runtime.ts
|
|
1085
|
+
import { execFile as nodeExecFile } from "child_process";
|
|
1086
|
+
import { promisify } from "util";
|
|
1087
|
+
var execFileAsync = promisify(nodeExecFile);
|
|
1088
|
+
function createDefaultKr36Runtime(options = {}) {
|
|
1089
|
+
const execFile = options.execFile ?? execFileAsync;
|
|
1090
|
+
return {
|
|
1091
|
+
fetchArticleHtml: async (request) => fetchWithCurl(execFile, request),
|
|
1092
|
+
fetchJson: async (request) => fetchWithCurl(execFile, request)
|
|
1093
|
+
};
|
|
1094
|
+
}
|
|
1095
|
+
async function fetchWithCurl(execFile, request) {
|
|
1096
|
+
const args = [
|
|
1097
|
+
"--silent",
|
|
1098
|
+
"--show-error",
|
|
1099
|
+
"--location",
|
|
1100
|
+
"--compressed",
|
|
1101
|
+
"--connect-timeout",
|
|
1102
|
+
"10",
|
|
1103
|
+
"--max-time",
|
|
1104
|
+
"45",
|
|
1105
|
+
...Object.entries(request.headers).flatMap(([name, value]) => [
|
|
1106
|
+
"--header",
|
|
1107
|
+
`${name}: ${value}`
|
|
1108
|
+
]),
|
|
1109
|
+
..."body" in request ? ["--data-raw", JSON.stringify(request.body)] : [],
|
|
1110
|
+
request.url
|
|
1111
|
+
];
|
|
1112
|
+
try {
|
|
1113
|
+
const result = await execFile("curl", args, {
|
|
1114
|
+
encoding: "utf8",
|
|
1115
|
+
maxBuffer: 20 * 1024 * 1024
|
|
1116
|
+
});
|
|
1117
|
+
return result.stdout;
|
|
1118
|
+
} catch (error) {
|
|
1119
|
+
const execError = error;
|
|
1120
|
+
const failureReason = execError.stderr?.trim() || "curl request failed.";
|
|
1121
|
+
throw new Kr36CommandError(
|
|
1122
|
+
"KR36_REQUEST_FAILED",
|
|
1123
|
+
`Failed to fetch 36kr page: ${failureReason}`,
|
|
1124
|
+
1,
|
|
1125
|
+
{
|
|
1126
|
+
curlExitCode: execError.code,
|
|
1127
|
+
url: request.url
|
|
1128
|
+
}
|
|
1129
|
+
);
|
|
1130
|
+
}
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
// src/toutiao/command.ts
|
|
1134
|
+
import { z as z3 } from "zod";
|
|
1135
|
+
|
|
1136
|
+
// src/toutiao/output.ts
|
|
1137
|
+
function renderToutiaoListAsJson(result) {
|
|
1138
|
+
return JSON.stringify(
|
|
1139
|
+
{
|
|
1140
|
+
ok: true,
|
|
1141
|
+
data: result
|
|
1142
|
+
},
|
|
1143
|
+
null,
|
|
1144
|
+
2
|
|
1145
|
+
);
|
|
1146
|
+
}
|
|
1147
|
+
function renderToutiaoListAsTable(result) {
|
|
1148
|
+
return renderSafeTable([
|
|
1149
|
+
["id", "title", "author", "publishTime", "comments", "url"],
|
|
1150
|
+
...result.items.map((item) => [
|
|
1151
|
+
item.id,
|
|
1152
|
+
item.title,
|
|
1153
|
+
item.authorName ?? "",
|
|
1154
|
+
item.publishTime?.local ?? "",
|
|
1155
|
+
item.commentCount === void 0 ? "" : String(item.commentCount),
|
|
1156
|
+
item.url
|
|
1157
|
+
])
|
|
1158
|
+
]);
|
|
1159
|
+
}
|
|
1160
|
+
function renderToutiaoArticleAsJson(article) {
|
|
1161
|
+
return JSON.stringify(
|
|
1162
|
+
{
|
|
1163
|
+
ok: true,
|
|
1164
|
+
data: article
|
|
1165
|
+
},
|
|
1166
|
+
null,
|
|
1167
|
+
2
|
|
1168
|
+
);
|
|
1169
|
+
}
|
|
1170
|
+
function renderToutiaoAuthorAsJson(result) {
|
|
1171
|
+
return JSON.stringify(
|
|
1172
|
+
{
|
|
1173
|
+
ok: true,
|
|
1174
|
+
data: result
|
|
1175
|
+
},
|
|
1176
|
+
null,
|
|
1177
|
+
2
|
|
1178
|
+
);
|
|
1179
|
+
}
|
|
1180
|
+
function renderToutiaoAuthorAsTable(result) {
|
|
1181
|
+
return renderSafeTable([
|
|
1182
|
+
["id", "title", "author", "publishTime", "comments", "url"],
|
|
1183
|
+
...result.items.map((item) => [
|
|
1184
|
+
item.id,
|
|
1185
|
+
item.title,
|
|
1186
|
+
item.authorName ?? "",
|
|
1187
|
+
item.publishTime?.local ?? "",
|
|
1188
|
+
item.commentCount === void 0 ? "" : String(item.commentCount),
|
|
1189
|
+
item.url
|
|
1190
|
+
])
|
|
1191
|
+
]);
|
|
1192
|
+
}
|
|
1193
|
+
function renderToutiaoCommandErrorAsJson(code, message, details) {
|
|
1194
|
+
return JSON.stringify(
|
|
1195
|
+
{
|
|
1196
|
+
ok: false,
|
|
1197
|
+
error: {
|
|
1198
|
+
code,
|
|
1199
|
+
message,
|
|
1200
|
+
...details === void 0 ? {} : { details }
|
|
1201
|
+
}
|
|
1202
|
+
},
|
|
1203
|
+
null,
|
|
1204
|
+
2
|
|
1205
|
+
);
|
|
1206
|
+
}
|
|
1207
|
+
|
|
1208
|
+
// src/toutiao/types.ts
|
|
1209
|
+
var ToutiaoCommandError = class extends Error {
|
|
1210
|
+
constructor(code, message, exitCode, details) {
|
|
1211
|
+
super(message);
|
|
1212
|
+
this.code = code;
|
|
1213
|
+
this.exitCode = exitCode;
|
|
1214
|
+
this.details = details;
|
|
1215
|
+
this.name = "ToutiaoCommandError";
|
|
1216
|
+
}
|
|
1217
|
+
};
|
|
1218
|
+
|
|
1219
|
+
// src/toutiao/service.ts
|
|
1220
|
+
var SUPPORTED_SOURCES2 = /* @__PURE__ */ new Set([
|
|
1221
|
+
"tech",
|
|
1222
|
+
"AI",
|
|
1223
|
+
"\u5149\u523B\u673A",
|
|
1224
|
+
"\u82AF\u7247",
|
|
1225
|
+
"\u534A\u5BFC\u4F53"
|
|
1226
|
+
]);
|
|
1227
|
+
var MAX_AUTHOR_CONTENT_ARTICLES = 100;
|
|
1228
|
+
var MAX_AUTHOR_CONTENT_RETAINED_BYTES = 10 * 1024 * 1024;
|
|
1229
|
+
var ToutiaoService = class {
|
|
1230
|
+
constructor(dependencies) {
|
|
1231
|
+
this.dependencies = dependencies;
|
|
1232
|
+
}
|
|
1233
|
+
async article(input) {
|
|
1234
|
+
const url = buildToutiaoArticleUrl(input);
|
|
1235
|
+
return this.dependencies.runtime.withSession(
|
|
1236
|
+
async (session) => session.fetchArticle({ input, url })
|
|
1237
|
+
);
|
|
1238
|
+
}
|
|
1239
|
+
async author(input, options = {}) {
|
|
1240
|
+
const authorToken = buildToutiaoAuthorToken(input);
|
|
1241
|
+
const pages = normalizePages2(options.pages);
|
|
1242
|
+
const url = buildToutiaoAuthorUrl(authorToken);
|
|
1243
|
+
const withContent = options.withContent ?? false;
|
|
1244
|
+
return this.dependencies.runtime.withSession(async (session) => {
|
|
1245
|
+
const runtimeResult = await session.fetchAuthorArticles({
|
|
1246
|
+
authorToken,
|
|
1247
|
+
pages,
|
|
1248
|
+
url
|
|
1249
|
+
});
|
|
1250
|
+
const result = {
|
|
1251
|
+
...runtimeResult,
|
|
1252
|
+
meta: {
|
|
1253
|
+
fetchedPages: pages,
|
|
1254
|
+
totalItems: runtimeResult.items.length,
|
|
1255
|
+
withContent
|
|
1256
|
+
},
|
|
1257
|
+
request: {
|
|
1258
|
+
input,
|
|
1259
|
+
url
|
|
1260
|
+
}
|
|
1261
|
+
};
|
|
1262
|
+
if (!withContent) {
|
|
1263
|
+
return result;
|
|
1264
|
+
}
|
|
1265
|
+
if (runtimeResult.items.length > MAX_AUTHOR_CONTENT_ARTICLES) {
|
|
1266
|
+
throw new ToutiaoCommandError(
|
|
1267
|
+
"TOUTIAO_RESOURCE_LIMIT",
|
|
1268
|
+
"Toutiao author content collection supports at most 100 articles per invocation.",
|
|
1269
|
+
2,
|
|
1270
|
+
{
|
|
1271
|
+
totalItems: runtimeResult.items.length
|
|
1272
|
+
}
|
|
1273
|
+
);
|
|
1274
|
+
}
|
|
1275
|
+
const articles = [];
|
|
1276
|
+
let retainedBytes = 0;
|
|
1277
|
+
for (const item of runtimeResult.items) {
|
|
1278
|
+
const article = await session.fetchArticle({
|
|
1279
|
+
input: item.id,
|
|
1280
|
+
url: buildToutiaoArticleUrl(item.url || item.id)
|
|
1281
|
+
});
|
|
1282
|
+
const nextRetainedBytes = retainedBytes + Buffer.byteLength(JSON.stringify(article), "utf8");
|
|
1283
|
+
if (nextRetainedBytes > MAX_AUTHOR_CONTENT_RETAINED_BYTES) {
|
|
1284
|
+
throw new ToutiaoCommandError(
|
|
1285
|
+
"TOUTIAO_RESOURCE_LIMIT",
|
|
1286
|
+
"Toutiao author content exceeded the configured size limit.",
|
|
1287
|
+
2,
|
|
1288
|
+
{
|
|
1289
|
+
maxRetainedBytes: MAX_AUTHOR_CONTENT_RETAINED_BYTES,
|
|
1290
|
+
retainedBytes: nextRetainedBytes
|
|
1291
|
+
}
|
|
1292
|
+
);
|
|
1293
|
+
}
|
|
1294
|
+
retainedBytes = nextRetainedBytes;
|
|
1295
|
+
articles.push(article);
|
|
1296
|
+
}
|
|
1297
|
+
result.articles = articles;
|
|
1298
|
+
result.meta.totalArticles = articles.length;
|
|
1299
|
+
return result;
|
|
1300
|
+
});
|
|
1301
|
+
}
|
|
1302
|
+
async list(options) {
|
|
1303
|
+
const source = parseSource2(options.source);
|
|
1304
|
+
const pages = normalizePages2(options.pages);
|
|
1305
|
+
return this.dependencies.runtime.withSession(async (session) => {
|
|
1306
|
+
const runtimeResult = source === "tech" ? await session.fetchTechnologyChannel({ pages }) : await session.fetchKeywordInformation({
|
|
1307
|
+
keyword: source,
|
|
1308
|
+
pages,
|
|
1309
|
+
source
|
|
1310
|
+
});
|
|
1311
|
+
return {
|
|
1312
|
+
...runtimeResult,
|
|
1313
|
+
meta: {
|
|
1314
|
+
fetchedPages: pages,
|
|
1315
|
+
totalItems: runtimeResult.items.length
|
|
1316
|
+
}
|
|
1317
|
+
};
|
|
1318
|
+
});
|
|
1319
|
+
}
|
|
1320
|
+
};
|
|
1321
|
+
function buildToutiaoAuthorToken(input) {
|
|
1322
|
+
const trimmed = input.trim();
|
|
1323
|
+
if (/^[A-Za-z0-9._-]+$/.test(trimmed) && trimmed.length >= 12) {
|
|
1324
|
+
return trimmed;
|
|
1325
|
+
}
|
|
1326
|
+
try {
|
|
1327
|
+
const url = new URL(trimmed);
|
|
1328
|
+
const token = /\/c\/user\/token\/([^/?#]+)/.exec(url.pathname)?.[1];
|
|
1329
|
+
if (token && isToutiaoHost(url.hostname)) {
|
|
1330
|
+
return decodeURIComponent(token);
|
|
1331
|
+
}
|
|
1332
|
+
} catch {
|
|
1333
|
+
}
|
|
1334
|
+
throw new ToutiaoCommandError(
|
|
1335
|
+
"TOUTIAO_INVALID_AUTHOR",
|
|
1336
|
+
"Toutiao author must be a user token or a /c/user/token/<token>/ homepage URL.",
|
|
1337
|
+
2,
|
|
1338
|
+
{
|
|
1339
|
+
input
|
|
1340
|
+
}
|
|
1341
|
+
);
|
|
1342
|
+
}
|
|
1343
|
+
function buildToutiaoAuthorUrl(authorToken) {
|
|
1344
|
+
return `https://www.toutiao.com/c/user/token/${encodeURIComponent(authorToken)}/`;
|
|
1345
|
+
}
|
|
1346
|
+
function buildToutiaoArticleUrl(input) {
|
|
1347
|
+
const trimmed = input.trim();
|
|
1348
|
+
if (/^\d+$/.test(trimmed)) {
|
|
1349
|
+
return `https://www.toutiao.com/article/${trimmed}/`;
|
|
1350
|
+
}
|
|
1351
|
+
try {
|
|
1352
|
+
const url = new URL(trimmed);
|
|
1353
|
+
const id = /(?:article|group)\/(\d+)/.exec(url.pathname)?.[1] ?? /^\/a(\d+)/.exec(url.pathname)?.[1];
|
|
1354
|
+
if (id && isToutiaoHost(url.hostname)) {
|
|
1355
|
+
return `https://www.toutiao.com/article/${id}/`;
|
|
1356
|
+
}
|
|
1357
|
+
const nextUrl = url.searchParams.get("url");
|
|
1358
|
+
if (nextUrl) {
|
|
1359
|
+
return buildToutiaoArticleUrl(nextUrl);
|
|
1360
|
+
}
|
|
1361
|
+
} catch {
|
|
1362
|
+
}
|
|
1363
|
+
throw new ToutiaoCommandError(
|
|
1364
|
+
"TOUTIAO_INVALID_ARTICLE",
|
|
1365
|
+
"Toutiao article must be a numeric id or a toutiao article/group URL.",
|
|
1366
|
+
2,
|
|
1367
|
+
{
|
|
1368
|
+
input
|
|
1369
|
+
}
|
|
1370
|
+
);
|
|
1371
|
+
}
|
|
1372
|
+
function isToutiaoHost(hostname) {
|
|
1373
|
+
return hostname === "www.toutiao.com" || hostname === "toutiao.com";
|
|
1374
|
+
}
|
|
1375
|
+
function parseSource2(source) {
|
|
1376
|
+
if (SUPPORTED_SOURCES2.has(source)) {
|
|
1377
|
+
return source;
|
|
1378
|
+
}
|
|
1379
|
+
throw new ToutiaoCommandError(
|
|
1380
|
+
"TOUTIAO_INVALID_SOURCE",
|
|
1381
|
+
"Toutiao list only supports tech, AI, \u5149\u523B\u673A, \u82AF\u7247, and \u534A\u5BFC\u4F53.",
|
|
1382
|
+
2,
|
|
1383
|
+
{
|
|
1384
|
+
source,
|
|
1385
|
+
supportedSources: [...SUPPORTED_SOURCES2]
|
|
1386
|
+
}
|
|
1387
|
+
);
|
|
1388
|
+
}
|
|
1389
|
+
function normalizePages2(pages) {
|
|
1390
|
+
const value = pages ?? 1;
|
|
1391
|
+
if (!Number.isInteger(value) || value < 1 || value > 5) {
|
|
1392
|
+
throw new ToutiaoCommandError(
|
|
1393
|
+
"TOUTIAO_INVALID_PAGES",
|
|
1394
|
+
"Toutiao list pages must be an integer between 1 and 5.",
|
|
1395
|
+
2,
|
|
1396
|
+
{
|
|
1397
|
+
pages: value
|
|
1398
|
+
}
|
|
1399
|
+
);
|
|
1400
|
+
}
|
|
1401
|
+
return value;
|
|
1402
|
+
}
|
|
1403
|
+
|
|
1404
|
+
// src/toutiao/command.ts
|
|
1405
|
+
var listOptionsSchema = z3.object({
|
|
1406
|
+
format: z3.enum(["json", "table"]).default("json"),
|
|
1407
|
+
pages: z3.number().int().min(1).max(5).default(1),
|
|
1408
|
+
table: z3.boolean().optional()
|
|
1409
|
+
});
|
|
1410
|
+
var articleOptionsSchema2 = z3.object({
|
|
1411
|
+
format: z3.enum(["json"]).default("json")
|
|
1412
|
+
});
|
|
1413
|
+
var authorOptionsSchema = z3.object({
|
|
1414
|
+
format: z3.enum(["json", "table"]).default("json"),
|
|
1415
|
+
pages: z3.number().int().min(1).max(5).default(1),
|
|
1416
|
+
table: z3.boolean().optional(),
|
|
1417
|
+
withContent: z3.boolean().default(false)
|
|
1418
|
+
});
|
|
1419
|
+
function registerToutiaoCommands(program, dependencies) {
|
|
1420
|
+
const toutiao = program.command("toutiao").description("Fetch Toutiao channel and search content.");
|
|
1421
|
+
const service = new ToutiaoService({ runtime: dependencies.runtime });
|
|
1422
|
+
toutiao.command("article").argument("<article>", "Toutiao article id or URL").option("--format <format>", "output format", "json").action(async (article, options) => {
|
|
1423
|
+
articleOptionsSchema2.parse(options);
|
|
1424
|
+
const result = await service.article(article);
|
|
1425
|
+
dependencies.stdout(`${renderToutiaoArticleAsJson(result)}
|
|
1426
|
+
`);
|
|
1427
|
+
});
|
|
1428
|
+
toutiao.command("list").argument("<source>", "Toutiao source: tech, AI, \u5149\u523B\u673A, \u82AF\u7247, or \u534A\u5BFC\u4F53").option("--pages <pages>", "number of pages to fetch", parsePages2, 1).option("--format <format>", "output format", "json").option("-t, --table", "render as a table").action(async (source, options) => {
|
|
1429
|
+
const parsed = listOptionsSchema.parse(applyTableShortcut3(options));
|
|
1430
|
+
const result = await service.list({
|
|
1431
|
+
pages: parsed.pages,
|
|
1432
|
+
source
|
|
1433
|
+
});
|
|
1434
|
+
dependencies.stdout(
|
|
1435
|
+
parsed.format === "table" ? `${renderToutiaoListAsTable(result)}
|
|
1436
|
+
` : `${renderToutiaoListAsJson(result)}
|
|
1437
|
+
`
|
|
1438
|
+
);
|
|
1439
|
+
});
|
|
1440
|
+
toutiao.command("author").argument("<author>", "Toutiao author token or /c/user/token/<token>/ homepage URL").option("--pages <pages>", "number of pages to fetch", parsePages2, 1).option("--with-content", "also fetch detail content for each author article").option("--format <format>", "output format", "json").option("-t, --table", "render as a table").action(async (author, options) => {
|
|
1441
|
+
const parsed = authorOptionsSchema.parse(applyTableShortcut3(options));
|
|
1442
|
+
const result = await service.author(author, {
|
|
1443
|
+
pages: parsed.pages,
|
|
1444
|
+
withContent: parsed.withContent
|
|
1445
|
+
});
|
|
1446
|
+
dependencies.stdout(
|
|
1447
|
+
parsed.format === "table" ? `${renderToutiaoAuthorAsTable(result)}
|
|
1448
|
+
` : `${renderToutiaoAuthorAsJson(result)}
|
|
1449
|
+
`
|
|
1450
|
+
);
|
|
1451
|
+
});
|
|
1452
|
+
}
|
|
1453
|
+
function handleToutiaoCommandError(error, dependencies) {
|
|
1454
|
+
if (error instanceof ToutiaoCommandError) {
|
|
1455
|
+
dependencies.stderr(
|
|
1456
|
+
`${renderToutiaoCommandErrorAsJson(error.code, error.message, error.details)}
|
|
1457
|
+
`
|
|
1458
|
+
);
|
|
1459
|
+
return error.exitCode;
|
|
1460
|
+
}
|
|
1461
|
+
return void 0;
|
|
1462
|
+
}
|
|
1463
|
+
function parsePages2(value) {
|
|
1464
|
+
const parsed = Number(value);
|
|
1465
|
+
if (!Number.isInteger(parsed)) {
|
|
1466
|
+
throw new ToutiaoCommandError(
|
|
1467
|
+
"TOUTIAO_INVALID_PAGES",
|
|
1468
|
+
"Toutiao list pages must be an integer between 1 and 5.",
|
|
1469
|
+
2,
|
|
1470
|
+
{
|
|
1471
|
+
pages: value
|
|
1472
|
+
}
|
|
1473
|
+
);
|
|
1474
|
+
}
|
|
1475
|
+
return parsed;
|
|
1476
|
+
}
|
|
1477
|
+
function applyTableShortcut3(options) {
|
|
1478
|
+
if (options.table) {
|
|
1479
|
+
return {
|
|
1480
|
+
...options,
|
|
1481
|
+
format: "table"
|
|
1482
|
+
};
|
|
1483
|
+
}
|
|
1484
|
+
return {
|
|
1485
|
+
...options,
|
|
1486
|
+
format: options.format
|
|
1487
|
+
};
|
|
1488
|
+
}
|
|
1489
|
+
|
|
1490
|
+
// src/toutiao/runtime.ts
|
|
1491
|
+
var TOUTIAO_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36";
|
|
1492
|
+
var TECH_CHANNEL_ID = 3189398999;
|
|
1493
|
+
var DEFAULT_DEADLINE_MS = 10 * 60 * 1e3;
|
|
1494
|
+
var MAX_ARTICLE_TEXT_BYTES = 1024 * 1024;
|
|
1495
|
+
var MAX_FEED_ITEMS_PER_RESPONSE = 100;
|
|
1496
|
+
var MAX_FEED_RESPONSE_BYTES = 5 * 1024 * 1024;
|
|
1497
|
+
var MAX_SEARCH_TEXT_BYTES = 1024 * 1024;
|
|
1498
|
+
var BLOCKED_RESOURCE_TYPES = /* @__PURE__ */ new Set(["font", "image", "media"]);
|
|
1499
|
+
function createDefaultToutiaoRuntime(options = {}) {
|
|
1500
|
+
const deadlineMs = options.deadlineMs ?? DEFAULT_DEADLINE_MS;
|
|
1501
|
+
const playwrightLoader = options.loadPlaywright ?? loadPlaywright;
|
|
1502
|
+
return {
|
|
1503
|
+
withSession: async (operation) => {
|
|
1504
|
+
let browser;
|
|
1505
|
+
let browserClosed = false;
|
|
1506
|
+
let context;
|
|
1507
|
+
let deadlineTriggered = false;
|
|
1508
|
+
let operationPromise;
|
|
1509
|
+
let lifecyclePromise;
|
|
1510
|
+
const createTimeoutError = () => new ToutiaoCommandError(
|
|
1511
|
+
"TOUTIAO_TIMEOUT",
|
|
1512
|
+
"Toutiao collection exceeded the command deadline.",
|
|
1513
|
+
1,
|
|
1514
|
+
{ deadlineMs }
|
|
1515
|
+
);
|
|
1516
|
+
const ensureWithinDeadline = () => {
|
|
1517
|
+
if (deadlineTriggered) {
|
|
1518
|
+
throw createTimeoutError();
|
|
1519
|
+
}
|
|
1520
|
+
};
|
|
1521
|
+
lifecyclePromise = (async () => {
|
|
1522
|
+
try {
|
|
1523
|
+
let playwright;
|
|
1524
|
+
try {
|
|
1525
|
+
playwright = await playwrightLoader();
|
|
1526
|
+
} catch {
|
|
1527
|
+
ensureWithinDeadline();
|
|
1528
|
+
throw new ToutiaoCommandError(
|
|
1529
|
+
"TOUTIAO_BROWSER_UNAVAILABLE",
|
|
1530
|
+
"Toutiao collection requires Playwright to be installed.",
|
|
1531
|
+
2
|
|
1532
|
+
);
|
|
1533
|
+
}
|
|
1534
|
+
ensureWithinDeadline();
|
|
1535
|
+
try {
|
|
1536
|
+
browser = await playwright.chromium.launch({ headless: true });
|
|
1537
|
+
} catch {
|
|
1538
|
+
ensureWithinDeadline();
|
|
1539
|
+
throw new ToutiaoCommandError(
|
|
1540
|
+
"TOUTIAO_BROWSER_UNAVAILABLE",
|
|
1541
|
+
"Toutiao collection requires Playwright Chromium to be installed.",
|
|
1542
|
+
2
|
|
1543
|
+
);
|
|
1544
|
+
}
|
|
1545
|
+
if (deadlineTriggered) {
|
|
1546
|
+
browserClosed = true;
|
|
1547
|
+
await browser.close().catch(() => void 0);
|
|
1548
|
+
throw createTimeoutError();
|
|
1549
|
+
}
|
|
1550
|
+
context = await browser.newContext({
|
|
1551
|
+
locale: "zh-CN",
|
|
1552
|
+
userAgent: TOUTIAO_USER_AGENT
|
|
1553
|
+
});
|
|
1554
|
+
ensureWithinDeadline();
|
|
1555
|
+
const page = await context.newPage();
|
|
1556
|
+
ensureWithinDeadline();
|
|
1557
|
+
await page.route("**/*", async (route) => {
|
|
1558
|
+
if (BLOCKED_RESOURCE_TYPES.has(route.request().resourceType())) {
|
|
1559
|
+
await route.abort();
|
|
1560
|
+
return;
|
|
1561
|
+
}
|
|
1562
|
+
await route.continue();
|
|
1563
|
+
});
|
|
1564
|
+
ensureWithinDeadline();
|
|
1565
|
+
operationPromise = Promise.resolve().then(
|
|
1566
|
+
() => operation(createToutiaoSession(page))
|
|
1567
|
+
);
|
|
1568
|
+
return await operationPromise;
|
|
1569
|
+
} finally {
|
|
1570
|
+
await context?.close().catch(() => void 0);
|
|
1571
|
+
if (!browserClosed) {
|
|
1572
|
+
await browser?.close().catch(() => void 0);
|
|
1573
|
+
}
|
|
1574
|
+
}
|
|
1575
|
+
})();
|
|
1576
|
+
return await new Promise((resolve, reject) => {
|
|
1577
|
+
const deadlineTimer = setTimeout(() => {
|
|
1578
|
+
deadlineTriggered = true;
|
|
1579
|
+
void (async () => {
|
|
1580
|
+
if (browser !== void 0) {
|
|
1581
|
+
browserClosed = true;
|
|
1582
|
+
await browser.close().catch(() => void 0);
|
|
1583
|
+
}
|
|
1584
|
+
if (operationPromise !== void 0) {
|
|
1585
|
+
await operationPromise.then(
|
|
1586
|
+
() => void 0,
|
|
1587
|
+
() => void 0
|
|
1588
|
+
);
|
|
1589
|
+
await lifecyclePromise.then(
|
|
1590
|
+
() => void 0,
|
|
1591
|
+
() => void 0
|
|
1592
|
+
);
|
|
1593
|
+
}
|
|
1594
|
+
reject(createTimeoutError());
|
|
1595
|
+
})();
|
|
1596
|
+
}, deadlineMs);
|
|
1597
|
+
void lifecyclePromise.then(
|
|
1598
|
+
(value) => {
|
|
1599
|
+
if (!deadlineTriggered) {
|
|
1600
|
+
clearTimeout(deadlineTimer);
|
|
1601
|
+
resolve(value);
|
|
1602
|
+
}
|
|
1603
|
+
},
|
|
1604
|
+
(error) => {
|
|
1605
|
+
if (!deadlineTriggered) {
|
|
1606
|
+
clearTimeout(deadlineTimer);
|
|
1607
|
+
reject(error);
|
|
1608
|
+
}
|
|
1609
|
+
}
|
|
1610
|
+
);
|
|
1611
|
+
});
|
|
1612
|
+
}
|
|
1613
|
+
};
|
|
1614
|
+
}
|
|
1615
|
+
function createToutiaoSession(page) {
|
|
1616
|
+
return {
|
|
1617
|
+
fetchArticle: async (request) => fetchArticle(page, request),
|
|
1618
|
+
fetchAuthorArticles: async (options) => fetchAuthorArticles(page, options),
|
|
1619
|
+
fetchKeywordInformation: async (options) => fetchKeywordInformation(page, options),
|
|
1620
|
+
fetchTechnologyChannel: async (options) => fetchTechnologyChannel(page, options)
|
|
1621
|
+
};
|
|
1622
|
+
}
|
|
1623
|
+
async function fetchArticle(page, request) {
|
|
1624
|
+
await page.goto(request.url, {
|
|
1625
|
+
timeout: 45e3,
|
|
1626
|
+
waitUntil: "domcontentloaded"
|
|
1627
|
+
});
|
|
1628
|
+
await page.waitForTimeout(5e3);
|
|
1629
|
+
const title = await page.locator("h1").first().innerText({ timeout: 5e3 }).catch(() => "");
|
|
1630
|
+
const paragraphs = await page.locator("article p").evaluateAll(
|
|
1631
|
+
(nodes) => nodes.map((node) => (node.textContent ?? "").trim()).filter(Boolean)
|
|
1632
|
+
);
|
|
1633
|
+
const bodyText = await page.locator("body").innerText({ timeout: 5e3 });
|
|
1634
|
+
const extractedTextBytes = Math.max(
|
|
1635
|
+
Buffer.byteLength(title, "utf8"),
|
|
1636
|
+
Buffer.byteLength(bodyText, "utf8"),
|
|
1637
|
+
paragraphs.reduce(
|
|
1638
|
+
(total, paragraph) => total + Buffer.byteLength(paragraph, "utf8"),
|
|
1639
|
+
0
|
|
1640
|
+
)
|
|
1641
|
+
);
|
|
1642
|
+
if (extractedTextBytes > MAX_ARTICLE_TEXT_BYTES) {
|
|
1643
|
+
throw new ToutiaoCommandError(
|
|
1644
|
+
"TOUTIAO_RESOURCE_LIMIT",
|
|
1645
|
+
"Toutiao article text exceeded the configured size limit.",
|
|
1646
|
+
2,
|
|
1647
|
+
{
|
|
1648
|
+
maxTextBytes: MAX_ARTICLE_TEXT_BYTES,
|
|
1649
|
+
textBytes: extractedTextBytes
|
|
1650
|
+
}
|
|
1651
|
+
);
|
|
1652
|
+
}
|
|
1653
|
+
const metadata = extractArticleMetadata(bodyText, title);
|
|
1654
|
+
const id = extractArticleId(request.url) ?? extractArticleId(request.input) ?? "";
|
|
1655
|
+
if (!id || !title || paragraphs.length === 0) {
|
|
1656
|
+
throw new ToutiaoCommandError(
|
|
1657
|
+
"TOUTIAO_PARSE_ERROR",
|
|
1658
|
+
"Toutiao article page did not expose title and content.",
|
|
1659
|
+
2,
|
|
1660
|
+
{
|
|
1661
|
+
input: request.input,
|
|
1662
|
+
url: request.url
|
|
1663
|
+
}
|
|
1664
|
+
);
|
|
1665
|
+
}
|
|
1666
|
+
const article = {
|
|
1667
|
+
content: {
|
|
1668
|
+
paragraphs,
|
|
1669
|
+
text: paragraphs.join("\n")
|
|
1670
|
+
},
|
|
1671
|
+
id,
|
|
1672
|
+
request,
|
|
1673
|
+
title,
|
|
1674
|
+
url: page.url()
|
|
1675
|
+
};
|
|
1676
|
+
if (metadata.authorName !== void 0) {
|
|
1677
|
+
article.authorName = metadata.authorName;
|
|
1678
|
+
}
|
|
1679
|
+
if (metadata.publishTimeText !== void 0) {
|
|
1680
|
+
article.publishTimeText = metadata.publishTimeText;
|
|
1681
|
+
}
|
|
1682
|
+
return article;
|
|
1683
|
+
}
|
|
1684
|
+
async function fetchTechnologyChannel(page, options) {
|
|
1685
|
+
const feedResponses = await collectFeedResponses(
|
|
1686
|
+
page,
|
|
1687
|
+
options.pages,
|
|
1688
|
+
(response) => response.url().includes("/api/pc/list/feed"),
|
|
1689
|
+
async () => {
|
|
1690
|
+
await page.goto(
|
|
1691
|
+
`https://www.toutiao.com/?channel=tech&source=ch&channel_id=${TECH_CHANNEL_ID}`,
|
|
1692
|
+
{
|
|
1693
|
+
timeout: 45e3,
|
|
1694
|
+
waitUntil: "domcontentloaded"
|
|
1695
|
+
}
|
|
1696
|
+
);
|
|
1697
|
+
}
|
|
1698
|
+
);
|
|
1699
|
+
const selectedResponses = feedResponses.slice(0, options.pages);
|
|
1700
|
+
const items = deduplicateItems(
|
|
1701
|
+
selectedResponses.flatMap((response) => mapFeedItems(response.data ?? []))
|
|
1702
|
+
);
|
|
1703
|
+
const lastResponse = selectedResponses.at(-1);
|
|
1704
|
+
if (items.length === 0) {
|
|
1705
|
+
throw new ToutiaoCommandError(
|
|
1706
|
+
"TOUTIAO_PARSE_ERROR",
|
|
1707
|
+
"Toutiao technology channel did not return feed items.",
|
|
1708
|
+
2
|
|
1709
|
+
);
|
|
1710
|
+
}
|
|
1711
|
+
const result = {
|
|
1712
|
+
hasMore: lastResponse?.has_more ?? false,
|
|
1713
|
+
items,
|
|
1714
|
+
source: "tech"
|
|
1715
|
+
};
|
|
1716
|
+
addNextPage(result, lastResponse);
|
|
1717
|
+
return result;
|
|
1718
|
+
}
|
|
1719
|
+
async function fetchAuthorArticles(page, options) {
|
|
1720
|
+
const feedResponses = await collectFeedResponses(
|
|
1721
|
+
page,
|
|
1722
|
+
options.pages,
|
|
1723
|
+
(response) => isAuthorFeedResponseUrl(response.url(), options.authorToken),
|
|
1724
|
+
async () => {
|
|
1725
|
+
await page.goto(options.url, {
|
|
1726
|
+
timeout: 45e3,
|
|
1727
|
+
waitUntil: "domcontentloaded"
|
|
1728
|
+
});
|
|
1729
|
+
}
|
|
1730
|
+
);
|
|
1731
|
+
const selectedResponses = feedResponses.slice(0, options.pages);
|
|
1732
|
+
const items = deduplicateItems(
|
|
1733
|
+
selectedResponses.flatMap((response) => mapFeedItems(response.data ?? []))
|
|
1734
|
+
);
|
|
1735
|
+
const lastResponse = selectedResponses.at(-1);
|
|
1736
|
+
if (items.length === 0) {
|
|
1737
|
+
throw new ToutiaoCommandError(
|
|
1738
|
+
"TOUTIAO_PARSE_ERROR",
|
|
1739
|
+
"Toutiao author homepage did not return article feed items.",
|
|
1740
|
+
2,
|
|
1741
|
+
{
|
|
1742
|
+
authorToken: options.authorToken,
|
|
1743
|
+
url: options.url
|
|
1744
|
+
}
|
|
1745
|
+
);
|
|
1746
|
+
}
|
|
1747
|
+
const result = {
|
|
1748
|
+
authorToken: options.authorToken,
|
|
1749
|
+
hasMore: lastResponse?.has_more ?? false,
|
|
1750
|
+
items
|
|
1751
|
+
};
|
|
1752
|
+
addNextPage(result, lastResponse);
|
|
1753
|
+
return result;
|
|
1754
|
+
}
|
|
1755
|
+
async function collectFeedResponses(page, pages, accepts, navigate) {
|
|
1756
|
+
const feedResponses = [];
|
|
1757
|
+
const maxResponseParses = pages * 2;
|
|
1758
|
+
const pendingResponses = /* @__PURE__ */ new Set();
|
|
1759
|
+
const queuedResponses = [];
|
|
1760
|
+
let firstResourceError;
|
|
1761
|
+
let startedResponseParses = 0;
|
|
1762
|
+
const startQueuedResponses = () => {
|
|
1763
|
+
while (firstResourceError === void 0 && queuedResponses.length > 0 && feedResponses.length + pendingResponses.size < pages && startedResponseParses < maxResponseParses) {
|
|
1764
|
+
startResponseParse(queuedResponses.shift());
|
|
1765
|
+
}
|
|
1766
|
+
};
|
|
1767
|
+
const startResponseParse = (response) => {
|
|
1768
|
+
startedResponseParses += 1;
|
|
1769
|
+
const pending = parseFeedResponse(response).then((feedResponse) => {
|
|
1770
|
+
feedResponses.push(feedResponse);
|
|
1771
|
+
}).catch((error) => {
|
|
1772
|
+
if (error instanceof ToutiaoCommandError) {
|
|
1773
|
+
firstResourceError ??= error;
|
|
1774
|
+
}
|
|
1775
|
+
}).finally(() => {
|
|
1776
|
+
pendingResponses.delete(pending);
|
|
1777
|
+
startQueuedResponses();
|
|
1778
|
+
});
|
|
1779
|
+
pendingResponses.add(pending);
|
|
1780
|
+
};
|
|
1781
|
+
const handler = (response) => {
|
|
1782
|
+
if (!accepts(response) || firstResourceError !== void 0 || feedResponses.length >= pages || startedResponseParses + queuedResponses.length >= maxResponseParses) {
|
|
1783
|
+
return;
|
|
1784
|
+
}
|
|
1785
|
+
if (feedResponses.length + pendingResponses.size < pages) {
|
|
1786
|
+
startResponseParse(response);
|
|
1787
|
+
return;
|
|
1788
|
+
}
|
|
1789
|
+
queuedResponses.push(response);
|
|
1790
|
+
};
|
|
1791
|
+
page.on("response", handler);
|
|
1792
|
+
try {
|
|
1793
|
+
await navigate();
|
|
1794
|
+
await waitForFeedResponses(
|
|
1795
|
+
page,
|
|
1796
|
+
feedResponses,
|
|
1797
|
+
pages,
|
|
1798
|
+
() => firstResourceError !== void 0
|
|
1799
|
+
);
|
|
1800
|
+
while (pendingResponses.size > 0) {
|
|
1801
|
+
await Promise.allSettled([...pendingResponses]);
|
|
1802
|
+
}
|
|
1803
|
+
if (firstResourceError !== void 0) {
|
|
1804
|
+
throw firstResourceError;
|
|
1805
|
+
}
|
|
1806
|
+
return feedResponses;
|
|
1807
|
+
} finally {
|
|
1808
|
+
page.off("response", handler);
|
|
1809
|
+
queuedResponses.length = 0;
|
|
1810
|
+
while (pendingResponses.size > 0) {
|
|
1811
|
+
await Promise.allSettled([...pendingResponses]);
|
|
1812
|
+
}
|
|
1813
|
+
}
|
|
1814
|
+
}
|
|
1815
|
+
async function parseFeedResponse(response) {
|
|
1816
|
+
const declaredLength = Number(await response.headerValue("content-length"));
|
|
1817
|
+
if (Number.isFinite(declaredLength) && declaredLength > MAX_FEED_RESPONSE_BYTES) {
|
|
1818
|
+
throw feedResponseTooLarge();
|
|
1819
|
+
}
|
|
1820
|
+
const body = await response.body();
|
|
1821
|
+
if (body.byteLength > MAX_FEED_RESPONSE_BYTES) {
|
|
1822
|
+
throw feedResponseTooLarge();
|
|
1823
|
+
}
|
|
1824
|
+
const feedResponse = JSON.parse(body.toString("utf8"));
|
|
1825
|
+
if (Array.isArray(feedResponse.data) && feedResponse.data.length > MAX_FEED_ITEMS_PER_RESPONSE) {
|
|
1826
|
+
throw new ToutiaoCommandError(
|
|
1827
|
+
"TOUTIAO_RESOURCE_LIMIT",
|
|
1828
|
+
"Toutiao feed response exceeded the configured item limit.",
|
|
1829
|
+
2,
|
|
1830
|
+
{ maxItems: MAX_FEED_ITEMS_PER_RESPONSE }
|
|
1831
|
+
);
|
|
1832
|
+
}
|
|
1833
|
+
return feedResponse;
|
|
1834
|
+
}
|
|
1835
|
+
function feedResponseTooLarge() {
|
|
1836
|
+
return new ToutiaoCommandError(
|
|
1837
|
+
"TOUTIAO_RESPONSE_TOO_LARGE",
|
|
1838
|
+
"Toutiao feed response exceeded the configured size limit.",
|
|
1839
|
+
2,
|
|
1840
|
+
{ maxBytes: MAX_FEED_RESPONSE_BYTES }
|
|
1841
|
+
);
|
|
1842
|
+
}
|
|
1843
|
+
async function fetchKeywordInformation(page, options) {
|
|
1844
|
+
const items = [];
|
|
1845
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1846
|
+
let blocked = false;
|
|
1847
|
+
for (let pageIndex = 0; pageIndex < options.pages; pageIndex += 1) {
|
|
1848
|
+
const url = buildSearchUrl(options.keyword, pageIndex);
|
|
1849
|
+
await page.goto(url, {
|
|
1850
|
+
timeout: 45e3,
|
|
1851
|
+
waitUntil: "domcontentloaded"
|
|
1852
|
+
});
|
|
1853
|
+
await page.waitForTimeout(5e3);
|
|
1854
|
+
blocked = blocked || await page.locator("#pc_captcha").count() > 0;
|
|
1855
|
+
for (const item of await extractSearchItemsFromPage(page, options.keyword)) {
|
|
1856
|
+
if (seen.has(item.id)) {
|
|
1857
|
+
continue;
|
|
1858
|
+
}
|
|
1859
|
+
seen.add(item.id);
|
|
1860
|
+
items.push(item);
|
|
1861
|
+
}
|
|
1862
|
+
}
|
|
1863
|
+
if (blocked && items.length === 0) {
|
|
1864
|
+
throw new ToutiaoCommandError(
|
|
1865
|
+
"TOUTIAO_VERIFICATION_REQUIRED",
|
|
1866
|
+
"Toutiao search requires browser verification before returning usable results.",
|
|
1867
|
+
1,
|
|
1868
|
+
{
|
|
1869
|
+
keyword: options.keyword
|
|
1870
|
+
}
|
|
1871
|
+
);
|
|
1872
|
+
}
|
|
1873
|
+
if (items.length === 0) {
|
|
1874
|
+
throw new ToutiaoCommandError(
|
|
1875
|
+
"TOUTIAO_PARSE_ERROR",
|
|
1876
|
+
"Toutiao search page did not expose result items.",
|
|
1877
|
+
2,
|
|
1878
|
+
{
|
|
1879
|
+
keyword: options.keyword
|
|
1880
|
+
}
|
|
1881
|
+
);
|
|
1882
|
+
}
|
|
1883
|
+
return {
|
|
1884
|
+
hasMore: true,
|
|
1885
|
+
items,
|
|
1886
|
+
keyword: options.keyword,
|
|
1887
|
+
source: options.source
|
|
1888
|
+
};
|
|
1889
|
+
}
|
|
1890
|
+
function buildSearchUrl(keyword, pageIndex) {
|
|
1891
|
+
const params = new URLSearchParams({
|
|
1892
|
+
action_type: pageIndex === 0 ? "search" : "pagination",
|
|
1893
|
+
cur_tab_title: "news",
|
|
1894
|
+
dvpf: "pc",
|
|
1895
|
+
from: "news",
|
|
1896
|
+
keyword,
|
|
1897
|
+
page_num: String(pageIndex),
|
|
1898
|
+
pd: "information",
|
|
1899
|
+
source: pageIndex === 0 ? "input" : "pagination"
|
|
1900
|
+
});
|
|
1901
|
+
return `https://so.toutiao.com/search?${params.toString()}`;
|
|
1902
|
+
}
|
|
1903
|
+
async function waitForFeedResponses(page, responses, pages, shouldStop) {
|
|
1904
|
+
for (let attempt = 0; attempt < 45 && responses.length < pages && !shouldStop(); attempt += 1) {
|
|
1905
|
+
await page.waitForTimeout(1e3);
|
|
1906
|
+
if (responses.length < pages && !shouldStop()) {
|
|
1907
|
+
await page.mouse.wheel(0, 4e3);
|
|
1908
|
+
}
|
|
1909
|
+
}
|
|
1910
|
+
}
|
|
1911
|
+
function mapFeedItems(items) {
|
|
1912
|
+
return items.flatMap((item) => {
|
|
1913
|
+
const id = String(item.group_id ?? item.item_id ?? item.id ?? "");
|
|
1914
|
+
const title = item.title ?? item.Title ?? "";
|
|
1915
|
+
if (!id || !title) {
|
|
1916
|
+
return [];
|
|
1917
|
+
}
|
|
1918
|
+
const mapped = {
|
|
1919
|
+
id,
|
|
1920
|
+
title,
|
|
1921
|
+
url: normalizeArticleUrl(item.article_url ?? item.display_url ?? item.source_url, id)
|
|
1922
|
+
};
|
|
1923
|
+
const abstract = item.Abstract ?? item.abstract;
|
|
1924
|
+
if (abstract !== void 0) {
|
|
1925
|
+
mapped.abstract = abstract;
|
|
1926
|
+
}
|
|
1927
|
+
if (item.comment_count !== void 0) {
|
|
1928
|
+
mapped.commentCount = item.comment_count;
|
|
1929
|
+
}
|
|
1930
|
+
if (item.image_url !== void 0) {
|
|
1931
|
+
mapped.image = item.image_url;
|
|
1932
|
+
}
|
|
1933
|
+
const authorName = item.media_name ?? item.source;
|
|
1934
|
+
if (authorName !== void 0) {
|
|
1935
|
+
mapped.authorName = authorName;
|
|
1936
|
+
}
|
|
1937
|
+
if (item.source_url !== void 0) {
|
|
1938
|
+
mapped.sourceUrl = item.source_url;
|
|
1939
|
+
}
|
|
1940
|
+
if (item.behot_time !== void 0) {
|
|
1941
|
+
mapped.publishTime = {
|
|
1942
|
+
iso: new Date(item.behot_time * 1e3).toISOString(),
|
|
1943
|
+
local: formatChinaTime2(item.behot_time),
|
|
1944
|
+
seconds: item.behot_time
|
|
1945
|
+
};
|
|
1946
|
+
}
|
|
1947
|
+
return [mapped];
|
|
1948
|
+
});
|
|
1949
|
+
}
|
|
1950
|
+
function deduplicateItems(items) {
|
|
1951
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1952
|
+
return items.filter((item) => {
|
|
1953
|
+
if (seen.has(item.id)) {
|
|
1954
|
+
return false;
|
|
1955
|
+
}
|
|
1956
|
+
seen.add(item.id);
|
|
1957
|
+
return true;
|
|
1958
|
+
});
|
|
1959
|
+
}
|
|
1960
|
+
function addNextPage(result, response) {
|
|
1961
|
+
const next = {};
|
|
1962
|
+
if (response?.next?.max_behot_time !== void 0) {
|
|
1963
|
+
next.maxBehotTime = response.next.max_behot_time;
|
|
1964
|
+
}
|
|
1965
|
+
if (response?.offset !== void 0) {
|
|
1966
|
+
next.offset = response.offset;
|
|
1967
|
+
}
|
|
1968
|
+
if (Object.keys(next).length > 0) {
|
|
1969
|
+
result.next = next;
|
|
1970
|
+
}
|
|
1971
|
+
}
|
|
1972
|
+
function normalizeArticleUrl(value, id) {
|
|
1973
|
+
if (!value) {
|
|
1974
|
+
return `https://www.toutiao.com/article/${id}/`;
|
|
1975
|
+
}
|
|
1976
|
+
try {
|
|
1977
|
+
const url = new URL(value, "https://www.toutiao.com");
|
|
1978
|
+
return parseToutiaoArticleLink(url.toString())?.url ?? `https://www.toutiao.com/article/${id}/`;
|
|
1979
|
+
} catch {
|
|
1980
|
+
return `https://www.toutiao.com/article/${id}/`;
|
|
1981
|
+
}
|
|
1982
|
+
}
|
|
1983
|
+
async function extractSearchItemsFromPage(page, keyword) {
|
|
1984
|
+
const anchors = await page.locator("a").evaluateAll(
|
|
1985
|
+
(nodes) => nodes.slice(0, 300).map((node) => ({
|
|
1986
|
+
href: node.href.slice(0, 2048),
|
|
1987
|
+
text: (node.textContent ?? "").trim().replace(/\s+/g, " ").slice(0, 1e3)
|
|
1988
|
+
}))
|
|
1989
|
+
);
|
|
1990
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1991
|
+
const items = [];
|
|
1992
|
+
for (const anchor of anchors) {
|
|
1993
|
+
const articleLink = parseToutiaoArticleLink(anchor.href);
|
|
1994
|
+
if (!articleLink || !anchor.text || anchor.text.length < 6 || seen.has(articleLink.id)) {
|
|
1995
|
+
continue;
|
|
1996
|
+
}
|
|
1997
|
+
seen.add(articleLink.id);
|
|
1998
|
+
items.push({
|
|
1999
|
+
id: articleLink.id,
|
|
2000
|
+
title: anchor.text.slice(0, 180),
|
|
2001
|
+
url: articleLink.url
|
|
2002
|
+
});
|
|
2003
|
+
if (items.length >= 30) {
|
|
2004
|
+
break;
|
|
2005
|
+
}
|
|
2006
|
+
}
|
|
2007
|
+
if (items.length > 0) {
|
|
2008
|
+
return items;
|
|
2009
|
+
}
|
|
2010
|
+
const bodyText = await page.locator("body").innerText({ timeout: 5e3 });
|
|
2011
|
+
const textBytes = Buffer.byteLength(bodyText, "utf8");
|
|
2012
|
+
if (textBytes > MAX_SEARCH_TEXT_BYTES) {
|
|
2013
|
+
throw new ToutiaoCommandError(
|
|
2014
|
+
"TOUTIAO_RESOURCE_LIMIT",
|
|
2015
|
+
"Toutiao search text exceeded the configured size limit.",
|
|
2016
|
+
2,
|
|
2017
|
+
{
|
|
2018
|
+
maxTextBytes: MAX_SEARCH_TEXT_BYTES,
|
|
2019
|
+
textBytes
|
|
2020
|
+
}
|
|
2021
|
+
);
|
|
2022
|
+
}
|
|
2023
|
+
return extractSearchItemsFromText(bodyText, keyword);
|
|
2024
|
+
}
|
|
2025
|
+
function extractSearchItemsFromText(text, keyword) {
|
|
2026
|
+
const blockedPrefixes = /* @__PURE__ */ new Set([
|
|
2027
|
+
"\u65E0\u969C\u788D \u4ECA\u65E5\u5934\u6761\u9996\u9875 \u767B\u5F55 \u7EFC\u5408 \u8D44\u8BAF \u89C6\u9891 \u56FE\u7247 \u7528\u6237 \u5C0F\u89C6\u9891 \u5FAE\u5934\u6761 \u97F3\u4E50 \u53BB\u6296\u97F3\u641C"
|
|
2028
|
+
]);
|
|
2029
|
+
const lines = text.split(/\n+/).map((line) => line.trim()).filter((line) => line.length >= 12 && !blockedPrefixes.has(line));
|
|
2030
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2031
|
+
const items = [];
|
|
2032
|
+
for (const line of lines) {
|
|
2033
|
+
const normalized = line.replace(/\s+/g, " ");
|
|
2034
|
+
if (seen.has(normalized)) {
|
|
2035
|
+
continue;
|
|
2036
|
+
}
|
|
2037
|
+
seen.add(normalized);
|
|
2038
|
+
items.push({
|
|
2039
|
+
id: createSearchItemId(keyword, normalized, items.length),
|
|
2040
|
+
title: normalized.slice(0, 180),
|
|
2041
|
+
url: `https://so.toutiao.com/search?keyword=${encodeURIComponent(keyword)}&pd=information&dvpf=pc&source=input`
|
|
2042
|
+
});
|
|
2043
|
+
if (items.length >= 30) {
|
|
2044
|
+
break;
|
|
2045
|
+
}
|
|
2046
|
+
}
|
|
2047
|
+
return items;
|
|
2048
|
+
}
|
|
2049
|
+
function parseToutiaoArticleLink(href, depth = 0) {
|
|
2050
|
+
if (depth > 3) {
|
|
2051
|
+
return void 0;
|
|
2052
|
+
}
|
|
2053
|
+
const directId = extractArticleId(href);
|
|
2054
|
+
if (directId) {
|
|
2055
|
+
return {
|
|
2056
|
+
id: directId,
|
|
2057
|
+
url: `https://www.toutiao.com/article/${directId}/`
|
|
2058
|
+
};
|
|
2059
|
+
}
|
|
2060
|
+
try {
|
|
2061
|
+
const url = new URL(href);
|
|
2062
|
+
const nextUrl = url.searchParams.get("url");
|
|
2063
|
+
if (nextUrl) {
|
|
2064
|
+
return parseToutiaoArticleLink(nextUrl, depth + 1);
|
|
2065
|
+
}
|
|
2066
|
+
} catch {
|
|
2067
|
+
return void 0;
|
|
2068
|
+
}
|
|
2069
|
+
return void 0;
|
|
2070
|
+
}
|
|
2071
|
+
function extractArticleMetadata(bodyText, title) {
|
|
2072
|
+
const lines = bodyText.split(/\n+/).map((line) => line.trim()).filter(Boolean);
|
|
2073
|
+
const titleIndex = lines.findIndex((line) => line === title);
|
|
2074
|
+
const metadataLine = titleIndex >= 0 ? lines[titleIndex + 1] : void 0;
|
|
2075
|
+
const match = metadataLine?.match(/^(.+?)·(.+)$/);
|
|
2076
|
+
if (!match?.[1] || !match[2]) {
|
|
2077
|
+
return {};
|
|
2078
|
+
}
|
|
2079
|
+
return {
|
|
2080
|
+
authorName: match[2],
|
|
2081
|
+
publishTimeText: match[1]
|
|
2082
|
+
};
|
|
2083
|
+
}
|
|
2084
|
+
function extractArticleId(value) {
|
|
2085
|
+
return /(?:article|group)\/(\d+)/.exec(value)?.[1] ?? /\/a(\d+)/.exec(value)?.[1] ?? (/^\d+$/.test(value) ? value : void 0);
|
|
2086
|
+
}
|
|
2087
|
+
function isAuthorFeedResponseUrl(value, authorToken) {
|
|
2088
|
+
try {
|
|
2089
|
+
const url = new URL(value);
|
|
2090
|
+
const isProfileArticleFeed = url.pathname.includes("/api/pc/list/feed") && (url.searchParams.get("category") === "pc_profile_article" || url.searchParams.get("author_token") === authorToken);
|
|
2091
|
+
const isProfileAllFeed = url.pathname.includes("/api/pc/list/user/feed") && url.searchParams.get("category") === "profile_all" && url.searchParams.get("token") === authorToken;
|
|
2092
|
+
return isProfileArticleFeed || isProfileAllFeed;
|
|
2093
|
+
} catch {
|
|
2094
|
+
return value.includes("/api/pc/list/feed") && (value.includes("pc_profile_article") || value.includes(authorToken)) || value.includes("/api/pc/list/user/feed") && value.includes("profile_all") && value.includes(authorToken);
|
|
2095
|
+
}
|
|
2096
|
+
}
|
|
2097
|
+
function createSearchItemId(keyword, value, index) {
|
|
2098
|
+
let hash = 2166136261;
|
|
2099
|
+
const input = `${keyword}:${index}:${value}`;
|
|
2100
|
+
for (const char of input) {
|
|
2101
|
+
hash ^= char.charCodeAt(0);
|
|
2102
|
+
hash = Math.imul(hash, 16777619);
|
|
2103
|
+
}
|
|
2104
|
+
return `search-${(hash >>> 0).toString(16)}`;
|
|
2105
|
+
}
|
|
2106
|
+
function formatChinaTime2(timestampSeconds) {
|
|
2107
|
+
return new Intl.DateTimeFormat("sv-SE", {
|
|
2108
|
+
day: "2-digit",
|
|
2109
|
+
hour: "2-digit",
|
|
2110
|
+
hour12: false,
|
|
2111
|
+
minute: "2-digit",
|
|
2112
|
+
month: "2-digit",
|
|
2113
|
+
second: "2-digit",
|
|
2114
|
+
timeZone: "Asia/Shanghai",
|
|
2115
|
+
year: "numeric"
|
|
2116
|
+
}).format(new Date(timestampSeconds * 1e3));
|
|
2117
|
+
}
|
|
2118
|
+
async function loadPlaywright() {
|
|
2119
|
+
return await import("playwright");
|
|
2120
|
+
}
|
|
2121
|
+
|
|
2122
|
+
// src/cli.ts
|
|
2123
|
+
function createCli(options = {}) {
|
|
2124
|
+
const io = {
|
|
2125
|
+
stderr: options.stderr ?? ((value) => process.stderr.write(value)),
|
|
2126
|
+
stdout: options.stdout ?? ((value) => process.stdout.write(value))
|
|
2127
|
+
};
|
|
2128
|
+
const program = new Command().name("ants").description("Move data between systems with composable collectors.").showHelpAfterError().exitOverride();
|
|
2129
|
+
program.configureOutput({
|
|
2130
|
+
outputError: () => void 0,
|
|
2131
|
+
writeErr: () => void 0,
|
|
2132
|
+
writeOut: (value) => io.stdout(value)
|
|
2133
|
+
});
|
|
2134
|
+
registerKr36Commands(program, {
|
|
2135
|
+
runtime: options.kr36Runtime ?? createDefaultKr36Runtime(),
|
|
2136
|
+
stderr: io.stderr,
|
|
2137
|
+
stdout: io.stdout
|
|
2138
|
+
});
|
|
2139
|
+
registerHackerNewsCommands(program, {
|
|
2140
|
+
runtime: options.hackerNewsRuntime ?? createDefaultHackerNewsRuntime(),
|
|
2141
|
+
stderr: io.stderr,
|
|
2142
|
+
stdout: io.stdout
|
|
2143
|
+
});
|
|
2144
|
+
registerToutiaoCommands(program, {
|
|
2145
|
+
runtime: options.toutiaoRuntime ?? createDefaultToutiaoRuntime(),
|
|
2146
|
+
stderr: io.stderr,
|
|
2147
|
+
stdout: io.stdout
|
|
2148
|
+
});
|
|
2149
|
+
return {
|
|
2150
|
+
run: async (args) => {
|
|
2151
|
+
try {
|
|
2152
|
+
await program.parseAsync(args, { from: "user" });
|
|
2153
|
+
return 0;
|
|
2154
|
+
} catch (error) {
|
|
2155
|
+
if (error instanceof CommanderError && error.exitCode === 0) {
|
|
2156
|
+
return 0;
|
|
2157
|
+
}
|
|
2158
|
+
const toutiaoExitCode = handleToutiaoCommandError(error, io);
|
|
2159
|
+
if (toutiaoExitCode !== void 0) {
|
|
2160
|
+
return toutiaoExitCode;
|
|
2161
|
+
}
|
|
2162
|
+
const hackerNewsExitCode = handleHackerNewsCommandError(error, io);
|
|
2163
|
+
if (hackerNewsExitCode !== void 0) {
|
|
2164
|
+
return hackerNewsExitCode;
|
|
2165
|
+
}
|
|
2166
|
+
const kr36ExitCode = handleKr36CommandError(error, io);
|
|
2167
|
+
if (kr36ExitCode !== void 0) {
|
|
2168
|
+
return kr36ExitCode;
|
|
2169
|
+
}
|
|
2170
|
+
if (error instanceof ZodError || error instanceof CommanderError) {
|
|
2171
|
+
const message = error instanceof ZodError ? error.issues[0]?.message ?? "Invalid command arguments." : error.message;
|
|
2172
|
+
writeCliError(io, "INVALID_ARGUMENT", message);
|
|
2173
|
+
return 2;
|
|
2174
|
+
}
|
|
2175
|
+
writeCliError(
|
|
2176
|
+
io,
|
|
2177
|
+
"INTERNAL_ERROR",
|
|
2178
|
+
"An unexpected internal error occurred."
|
|
2179
|
+
);
|
|
2180
|
+
return 1;
|
|
2181
|
+
}
|
|
2182
|
+
}
|
|
2183
|
+
};
|
|
2184
|
+
}
|
|
2185
|
+
function writeCliError(io, code, message) {
|
|
2186
|
+
io.stderr(`${JSON.stringify({
|
|
2187
|
+
ok: false,
|
|
2188
|
+
error: {
|
|
2189
|
+
code,
|
|
2190
|
+
message
|
|
2191
|
+
}
|
|
2192
|
+
}, null, 2)}
|
|
2193
|
+
`);
|
|
2194
|
+
}
|
|
2195
|
+
|
|
2196
|
+
// src/index.ts
|
|
2197
|
+
function isMainEntrypoint(moduleUrl, argv) {
|
|
2198
|
+
const scriptPath = argv[1];
|
|
2199
|
+
if (!scriptPath) {
|
|
2200
|
+
return false;
|
|
2201
|
+
}
|
|
2202
|
+
return realpathSync(fileURLToPath(moduleUrl)) === realpathSync(scriptPath);
|
|
2203
|
+
}
|
|
2204
|
+
if (isMainEntrypoint(import.meta.url, process.argv)) {
|
|
2205
|
+
const cli = createCli();
|
|
2206
|
+
void cli.run(process.argv.slice(2)).then((exitCode) => {
|
|
2207
|
+
process.exitCode = exitCode;
|
|
2208
|
+
});
|
|
2209
|
+
}
|
|
2210
|
+
export {
|
|
2211
|
+
createCli,
|
|
2212
|
+
isMainEntrypoint
|
|
2213
|
+
};
|
|
2214
|
+
//# sourceMappingURL=index.js.map
|