@sovovs/bycli 2.1.36 → 2.1.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/README.zh-CN.md +21 -0
- package/cli-manifest.json +642 -0
- package/clis/52pojie/search.js +32 -0
- package/clis/baidu/search.js +68 -0
- package/clis/bing/search.js +58 -0
- package/clis/csdn/search.js +34 -0
- package/clis/gitlab/search.js +36 -0
- package/clis/so/search.js +36 -0
- package/clis/sogou/search.js +35 -0
- package/clis/threads/search.js +33 -0
- package/clis/toutiao/search.js +63 -0
- package/clis/toutiao/utils.js +110 -1
- package/clis/yandex/search.js +43 -0
- package/package.json +1 -1
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
2
|
+
import { ArgumentError, CommandExecutionError } from '@sovovs/bycli/errors';
|
|
3
|
+
import { emptySearchResults, requireBoundedInteger, requireSearchQuery, runBrowserStep } from '../_shared/search-adapter.js';
|
|
4
|
+
|
|
5
|
+
const SORTS = ['relevance', 'latest', 'replies', 'views'];
|
|
6
|
+
|
|
7
|
+
export function buildSearchUrl({ keyword, limit = 10, page = 1, section, sort }) {
|
|
8
|
+
const url = new URL('https://www.52pojie.cn/search.php'); url.searchParams.set('srchtxt', keyword); url.searchParams.set('page', String(page)); url.searchParams.set('perpage', String(limit));
|
|
9
|
+
if (section) url.searchParams.set('section', section); if (sort) url.searchParams.set('sort', sort); return url.toString();
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function extractItems() {
|
|
13
|
+
const body = document.body?.textContent || ''; const blocked = /验证码|访问频繁|captcha|access denied/i.test(body); const items = []; const seen = new Set(); const numberFrom = (card, selector) => { const value = (card.querySelector(selector)?.textContent || '').replace(/[^\d.]/g, ''); return value ? Number(value) : null; };
|
|
14
|
+
for (const card of document.querySelectorAll('.forum-item, .search-list li, .sltm')) {
|
|
15
|
+
const anchor = card.querySelector('.thread-title[href], h3 a[href], h2 a[href]'); if (!anchor || !/^https?:/i.test(anchor.href) || seen.has(anchor.href)) continue; seen.add(anchor.href); const time = card.querySelector('time');
|
|
16
|
+
items.push({ title: (anchor.textContent || '').trim(), url: anchor.href, snippet: (card.querySelector('.summary, .desc, p')?.textContent || '').trim(), displayUrl: new URL(anchor.href).hostname + new URL(anchor.href).pathname, source: '52pojie', resultType: 'thread', author: (card.querySelector('.author, .username')?.textContent || '').trim() || null, publishedAt: time?.getAttribute('datetime') || null, score: null, extra: { replies: numberFrom(card, '.replies, .reply'), views: numberFrom(card, '.views, .view'), section: (card.querySelector('.section, .forum')?.textContent || '').trim() || null } });
|
|
17
|
+
}
|
|
18
|
+
return { blocked, items };
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export const command = cli({
|
|
22
|
+
site: '52pojie', name: 'search', access: 'read', description: 'Search 52pojie', domain: 'www.52pojie.cn', strategy: Strategy.PUBLIC, browser: true,
|
|
23
|
+
args: [{ name: 'keyword', positional: true, required: true, help: 'Search query' }, { name: 'limit', type: 'int', default: 10, help: 'Number of threads (1-50)' }, { name: 'page', type: 'int', default: 1, help: 'Result page number' }, { name: 'section', help: 'Forum section identifier' }, { name: 'sort', help: 'Sort order: relevance, latest, replies, views' }],
|
|
24
|
+
columns: ['rank', 'title', 'url', 'snippet', 'displayUrl', 'source', 'resultType', 'author', 'publishedAt', 'score', 'extra'],
|
|
25
|
+
func: async (page, kwargs) => {
|
|
26
|
+
const keyword = requireSearchQuery(kwargs.keyword); const limit = requireBoundedInteger(kwargs.limit, 10, 1, 50, '--limit'); const pageNumber = requireBoundedInteger(kwargs.page, 1, 1, 100, '--page'); if (kwargs.sort && !SORTS.includes(String(kwargs.sort))) throw new ArgumentError(`--sort must be one of: ${SORTS.join(', ')}`);
|
|
27
|
+
await runBrowserStep('52pojie search navigation', () => page.goto(buildSearchUrl({ keyword, limit, page: pageNumber, section: kwargs.section, sort: kwargs.sort }))); await page.wait({ selector: '.forum-item, .search-list, .sltm', timeout: 8 }).catch(() => page.wait(2).catch(() => {})); const data = await runBrowserStep('52pojie search extraction', () => page.evaluate(`(${extractItems.toString()})()`));
|
|
28
|
+
if (data?.blocked) throw new CommandExecutionError('52pojie search was blocked by a verification page', 'Complete the verification in the browser and retry.'); if (!data?.items?.length) throw emptySearchResults('52pojie', keyword); return data.items.slice(0, limit).map((item, index) => ({ rank: (pageNumber - 1) * limit + index + 1, ...item }));
|
|
29
|
+
},
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
export const __test__ = { command, buildSearchUrl };
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
2
|
+
import { ArgumentError, CommandExecutionError } from '@sovovs/bycli/errors';
|
|
3
|
+
import { emptySearchResults, requireBoundedInteger, requireNonNegativeInteger, requireSearchQuery, runBrowserStep, toHttpsUrl } from '../_shared/search-adapter.js';
|
|
4
|
+
|
|
5
|
+
const TYPES = ['web', 'news', 'image', 'video'];
|
|
6
|
+
const FILETYPES = ['pdf', 'doc', 'xls', 'ppt', 'rtf', 'all'];
|
|
7
|
+
|
|
8
|
+
export function buildSearchUrl({ keyword, limit = 10, page = 1, site, filetype, platform, time }) {
|
|
9
|
+
const url = new URL('https://www.baidu.com/s');
|
|
10
|
+
url.searchParams.set('wd', keyword);
|
|
11
|
+
url.searchParams.set('pn', String((page - 1) * limit));
|
|
12
|
+
if (site) url.searchParams.set('si', site);
|
|
13
|
+
if (filetype) url.searchParams.set('ft', filetype);
|
|
14
|
+
if (platform) url.searchParams.set('ie', platform === 'mobile' ? 'utf-8' : 'utf-8');
|
|
15
|
+
if (time) url.searchParams.set('gpc', `stf=${time}`);
|
|
16
|
+
return url.toString();
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function validateChoice(value, choices, flag) {
|
|
20
|
+
if (value !== undefined && !choices.includes(String(value))) throw new ArgumentError(`--${flag} must be one of: ${choices.join(', ')}`);
|
|
21
|
+
return value === undefined ? undefined : String(value);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function extractItems() {
|
|
25
|
+
const blocked = /验证码|安全验证|百度安全验证|访问过于频繁|请完成验证/.test(document.body?.textContent || '');
|
|
26
|
+
const items = [];
|
|
27
|
+
const seen = new Set();
|
|
28
|
+
for (const card of document.querySelectorAll('.result, .c-container')) {
|
|
29
|
+
const anchor = card.querySelector('h3 a, h3 a[href]');
|
|
30
|
+
if (!anchor) continue;
|
|
31
|
+
const url = anchor.href || '';
|
|
32
|
+
if (!/^https?:/i.test(url) || seen.has(url)) continue;
|
|
33
|
+
seen.add(url);
|
|
34
|
+
items.push({ title: (anchor.textContent || '').trim(), url, snippet: (card.querySelector('.c-abstract, .content-right_8Zs40, .c-span-last')?.textContent || '').trim(), displayUrl: (card.querySelector('.c-showurl, .c-color-gray2')?.textContent || '').trim(), resultType: 'web', extra: {} });
|
|
35
|
+
}
|
|
36
|
+
return { blocked, items };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export const command = cli({
|
|
40
|
+
site: 'baidu', name: 'search', access: 'read', description: 'Search Baidu', domain: 'www.baidu.com', strategy: Strategy.PUBLIC, browser: true,
|
|
41
|
+
args: [
|
|
42
|
+
{ name: 'keyword', positional: true, required: true, help: 'Search query' },
|
|
43
|
+
{ name: 'limit', type: 'int', default: 10, help: 'Number of results (1-50)' },
|
|
44
|
+
{ name: 'page', type: 'int', default: 1, help: 'Result page number' },
|
|
45
|
+
{ name: 'site', help: 'Restrict results to a domain' },
|
|
46
|
+
{ name: 'filetype', help: 'File type: pdf, doc, xls, ppt, rtf, all' },
|
|
47
|
+
{ name: 'platform', help: 'Client platform: pc or mobile' },
|
|
48
|
+
{ name: 'time', help: 'Recent time filter understood by Baidu' },
|
|
49
|
+
],
|
|
50
|
+
columns: ['rank', 'title', 'url', 'snippet', 'displayUrl', 'source', 'resultType', 'author', 'publishedAt', 'score', 'extra'],
|
|
51
|
+
func: async (page, kwargs) => {
|
|
52
|
+
const keyword = requireSearchQuery(kwargs.keyword);
|
|
53
|
+
const limit = requireBoundedInteger(kwargs.limit, 10, 1, 50, '--limit');
|
|
54
|
+
const pageNumber = requireBoundedInteger(kwargs.page, 1, 1, 100, '--page');
|
|
55
|
+
const filetype = validateChoice(kwargs.filetype, FILETYPES, 'filetype');
|
|
56
|
+
const platform = validateChoice(kwargs.platform, ['pc', 'mobile'], 'platform');
|
|
57
|
+
const raw = await runBrowserStep('Baidu search navigation', () => page.goto(buildSearchUrl({ keyword, limit, page: pageNumber, site: kwargs.site, filetype, platform, time: kwargs.time })));
|
|
58
|
+
void raw;
|
|
59
|
+
await page.wait({ selector: '.result, .c-container', timeout: 8 }).catch(() => page.wait(2).catch(() => {}));
|
|
60
|
+
const data = await runBrowserStep('Baidu search extraction', () => page.evaluate(`(${extractItems.toString()})()`));
|
|
61
|
+
if (data?.blocked) throw new CommandExecutionError('Baidu search was blocked by a verification page', 'Complete the verification in the browser and retry.');
|
|
62
|
+
const items = data?.items || [];
|
|
63
|
+
if (!items.length) throw emptySearchResults('Baidu', keyword);
|
|
64
|
+
return items.slice(0, limit).map((item, index) => ({ rank: (pageNumber - 1) * limit + index + 1, ...item, source: 'baidu', author: null, publishedAt: null, score: null }));
|
|
65
|
+
},
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
export const __test__ = { command, buildSearchUrl };
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
2
|
+
import { ArgumentError, CommandExecutionError } from '@sovovs/bycli/errors';
|
|
3
|
+
import { emptySearchResults, requireBoundedInteger, requireSearchQuery, runBrowserStep } from '../_shared/search-adapter.js';
|
|
4
|
+
|
|
5
|
+
const FRESHNESS = ['day', 'week', 'month', 'year'];
|
|
6
|
+
const ANSWERS = ['web', 'news', 'video', 'images'];
|
|
7
|
+
const SAFE = ['off', 'moderate', 'strict'];
|
|
8
|
+
|
|
9
|
+
export function buildSearchUrl({ keyword, limit = 10, page = 1, freshness, market, answer, safe }) {
|
|
10
|
+
const url = new URL('https://www.bing.com/search');
|
|
11
|
+
url.searchParams.set('q', keyword);
|
|
12
|
+
url.searchParams.set('count', String(limit));
|
|
13
|
+
url.searchParams.set('first', String((page - 1) * limit + 1));
|
|
14
|
+
if (freshness) url.searchParams.set('freshness', freshness);
|
|
15
|
+
if (market) url.searchParams.set('cc', String(market).split('-').pop().toUpperCase());
|
|
16
|
+
if (answer && answer !== 'web') url.searchParams.set('scope', answer);
|
|
17
|
+
if (safe) url.searchParams.set('safesearch', safe);
|
|
18
|
+
return url.toString();
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function validate(value, choices, flag) {
|
|
22
|
+
if (value !== undefined && !choices.includes(String(value))) throw new ArgumentError(`--${flag} must be one of: ${choices.join(', ')}`);
|
|
23
|
+
return value === undefined ? undefined : String(value);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function extractItems() {
|
|
27
|
+
const blocked = /unusual traffic|verify you are human|captcha|access denied/i.test(document.body?.textContent || '');
|
|
28
|
+
const items = [];
|
|
29
|
+
const seen = new Set();
|
|
30
|
+
for (const card of document.querySelectorAll('li.b_algo')) {
|
|
31
|
+
const anchor = card.querySelector('h2 a[href]');
|
|
32
|
+
if (!anchor || !/^https?:/i.test(anchor.href) || seen.has(anchor.href)) continue;
|
|
33
|
+
seen.add(anchor.href);
|
|
34
|
+
items.push({ title: (anchor.textContent || '').trim(), url: anchor.href, snippet: (card.querySelector('.b_caption p')?.textContent || '').trim(), displayUrl: (card.querySelector('cite')?.textContent || '').trim(), resultType: 'web', extra: {} });
|
|
35
|
+
}
|
|
36
|
+
return { blocked, items };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export const command = cli({
|
|
40
|
+
site: 'bing', name: 'search', access: 'read', description: 'Search Bing', domain: 'www.bing.com', strategy: Strategy.PUBLIC, browser: true,
|
|
41
|
+
args: [
|
|
42
|
+
{ name: 'keyword', positional: true, required: true, help: 'Search query' }, { name: 'limit', type: 'int', default: 10, help: 'Number of results (1-50)' }, { name: 'page', type: 'int', default: 1, help: 'Result page number' },
|
|
43
|
+
{ name: 'freshness', help: 'Time range: day, week, month, year' }, { name: 'market', help: 'Market code such as en-US or zh-CN' }, { name: 'answer', help: 'Result scope: web, news, video, images' }, { name: 'safe', help: 'Safe search: off, moderate, strict' },
|
|
44
|
+
],
|
|
45
|
+
columns: ['rank', 'title', 'url', 'snippet', 'displayUrl', 'source', 'resultType', 'author', 'publishedAt', 'score', 'extra'],
|
|
46
|
+
func: async (page, kwargs) => {
|
|
47
|
+
const keyword = requireSearchQuery(kwargs.keyword); const limit = requireBoundedInteger(kwargs.limit, 10, 1, 50, '--limit'); const pageNumber = requireBoundedInteger(kwargs.page, 1, 1, 100, '--page');
|
|
48
|
+
const freshness = validate(kwargs.freshness, FRESHNESS, 'freshness'); const answer = validate(kwargs.answer, ANSWERS, 'answer'); const safe = validate(kwargs.safe, SAFE, 'safe');
|
|
49
|
+
await runBrowserStep('Bing search navigation', () => page.goto(buildSearchUrl({ keyword, limit, page: pageNumber, freshness, market: kwargs.market, answer, safe })));
|
|
50
|
+
await page.wait({ selector: 'li.b_algo', timeout: 8 }).catch(() => page.wait(2).catch(() => {}));
|
|
51
|
+
const data = await runBrowserStep('Bing search extraction', () => page.evaluate(`(${extractItems.toString()})()`));
|
|
52
|
+
if (data?.blocked) throw new CommandExecutionError('Bing search was blocked by a verification page', 'Complete the verification in the browser and retry.');
|
|
53
|
+
if (!data?.items?.length) throw emptySearchResults('Bing', keyword);
|
|
54
|
+
return data.items.slice(0, limit).map((item, index) => ({ rank: (pageNumber - 1) * limit + index + 1, ...item, source: 'bing', author: null, publishedAt: null, score: null }));
|
|
55
|
+
},
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
export const __test__ = { command, buildSearchUrl };
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
2
|
+
import { ArgumentError, CommandExecutionError } from '@sovovs/bycli/errors';
|
|
3
|
+
import { emptySearchResults, requireBoundedInteger, requireSearchQuery, runBrowserStep } from '../_shared/search-adapter.js';
|
|
4
|
+
|
|
5
|
+
const TYPES = ['all', 'blog', 'download', 'course']; const SORTS = ['relevance', 'latest', 'hot']; const TIMES = ['day', 'week', 'month', 'year'];
|
|
6
|
+
|
|
7
|
+
export function buildSearchUrl({ keyword, limit = 10, page = 1, contentType = 'all', sort, time }) {
|
|
8
|
+
const url = new URL('https://so.csdn.net/so/search'); url.searchParams.set('q', keyword); url.searchParams.set('p', String(page)); url.searchParams.set('t', contentType); url.searchParams.set('size', String(limit));
|
|
9
|
+
if (sort) url.searchParams.set('sort', sort); if (time) url.searchParams.set('time', time); return url.toString();
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function extractItems() {
|
|
13
|
+
const blocked = /登录|验证码|访问异常|captcha|sign in/i.test(document.body?.textContent || '') && !document.querySelector('.search-result-info, .search-result'); const items = []; const seen = new Set();
|
|
14
|
+
for (const card of document.querySelectorAll('.search-result-info, .search-result')) {
|
|
15
|
+
const anchor = card.querySelector('h3 a[href], h4 a[href]'); if (!anchor || !/^https?:/i.test(anchor.href) || seen.has(anchor.href)) continue;
|
|
16
|
+
seen.add(anchor.href); const time = card.querySelector('time'); items.push({ rank: items.length + 1, title: (anchor.textContent || '').trim(), url: anchor.href, snippet: (card.querySelector('.search-result-desc, p')?.textContent || '').trim(), displayUrl: new URL(anchor.href).hostname + new URL(anchor.href).pathname, source: 'csdn', resultType: 'article', author: (card.querySelector('.author, .user-name')?.textContent || '').trim() || null, publishedAt: time?.getAttribute('datetime') || null, score: null, extra: {} });
|
|
17
|
+
}
|
|
18
|
+
return { blocked, items };
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export const command = cli({
|
|
22
|
+
site: 'csdn', name: 'search', access: 'read', description: 'Search CSDN', domain: 'so.csdn.net', strategy: Strategy.PUBLIC, browser: true,
|
|
23
|
+
args: [{ name: 'keyword', positional: true, required: true, help: 'Search query' }, { name: 'limit', type: 'int', default: 10, help: 'Number of results (1-50)' }, { name: 'page', type: 'int', default: 1, help: 'Result page number' }, { name: 'content-type', help: 'Content type: all, blog, download, course' }, { name: 'sort', help: 'Sort order: relevance, latest, hot' }, { name: 'time', help: 'Time range: day, week, month, year' }],
|
|
24
|
+
columns: ['rank', 'title', 'url', 'snippet', 'displayUrl', 'source', 'resultType', 'author', 'publishedAt', 'score', 'extra'],
|
|
25
|
+
func: async (page, kwargs) => {
|
|
26
|
+
const keyword = requireSearchQuery(kwargs.keyword); const limit = requireBoundedInteger(kwargs.limit, 10, 1, 50, '--limit'); const pageNumber = requireBoundedInteger(kwargs.page, 1, 1, 100, '--page'); const contentType = kwargs['content-type'] ?? kwargs.contentType ?? 'all';
|
|
27
|
+
if (!TYPES.includes(String(contentType))) throw new ArgumentError(`--content-type must be one of: ${TYPES.join(', ')}`); if (kwargs.sort && !SORTS.includes(String(kwargs.sort))) throw new ArgumentError(`--sort must be one of: ${SORTS.join(', ')}`); if (kwargs.time && !TIMES.includes(String(kwargs.time))) throw new ArgumentError(`--time must be one of: ${TIMES.join(', ')}`);
|
|
28
|
+
await runBrowserStep('CSDN search navigation', () => page.goto(buildSearchUrl({ keyword, limit, page: pageNumber, contentType, sort: kwargs.sort, time: kwargs.time }))); await page.wait({ selector: '.search-result-info, .search-result', timeout: 8 }).catch(() => page.wait(2).catch(() => {}));
|
|
29
|
+
const data = await runBrowserStep('CSDN search extraction', () => page.evaluate(`(${extractItems.toString()})()`)); if (data?.blocked) throw new CommandExecutionError('CSDN search requires login or was blocked', 'Open CSDN in the browser, complete any prompt, and retry.'); if (!data?.items?.length) throw emptySearchResults('CSDN', keyword);
|
|
30
|
+
return data.items.slice(0, limit).map((item, index) => ({ ...item, rank: (pageNumber - 1) * limit + index + 1 }));
|
|
31
|
+
},
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
export const __test__ = { command, buildSearchUrl };
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
2
|
+
import { ArgumentError, CommandExecutionError } from '@sovovs/bycli/errors';
|
|
3
|
+
import { emptySearchResults, requireBoundedInteger, requireSearchQuery, runBrowserStep } from '../_shared/search-adapter.js';
|
|
4
|
+
|
|
5
|
+
const SCOPES = ['projects', 'issues', 'merge_requests', 'commits', 'blobs', 'users'];
|
|
6
|
+
const ORDER = ['created_at', 'updated_at', 'latest_activity_at']; const SORT = ['asc', 'desc'];
|
|
7
|
+
|
|
8
|
+
export function buildSearchUrl({ keyword, limit = 10, page = 1, scope = 'projects', orderBy, sort }) {
|
|
9
|
+
const url = new URL('https://gitlab.com/search'); url.searchParams.set('search', keyword); url.searchParams.set('page', String(page)); url.searchParams.set('per_page', String(limit));
|
|
10
|
+
if (scope) url.searchParams.set('scope', scope); if (orderBy) url.searchParams.set('order_by', orderBy); if (sort) url.searchParams.set('sort', sort); return url.toString();
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function extractItems(scope) {
|
|
14
|
+
const blocked = /sign in|login|access denied|captcha/i.test(document.body?.textContent || '') && !document.querySelector('.search-result-row, .search-results'); const items = []; const seen = new Set();
|
|
15
|
+
for (const card of document.querySelectorAll('.search-result-row, .search-result-item, .search-results li')) {
|
|
16
|
+
const anchor = card.querySelector('a.gl-link[href], a[href*="gitlab.com/"]'); if (!anchor || !/^https?:/i.test(anchor.href) || seen.has(anchor.href)) continue;
|
|
17
|
+
seen.add(anchor.href); const time = card.querySelector('time'); const type = scope === 'issues' ? 'issue' : scope === 'merge_requests' ? 'merge_request' : scope === 'projects' ? 'project' : scope || 'result';
|
|
18
|
+
items.push({ title: (anchor.textContent || '').trim(), url: anchor.href, snippet: (card.querySelector('.description, .description p, p')?.textContent || '').trim(), displayUrl: new URL(anchor.href).hostname + new URL(anchor.href).pathname, source: 'gitlab', resultType: type, author: (card.querySelector('.author, [data-testid="author"]')?.textContent || '').trim() || null, publishedAt: time?.getAttribute('datetime') || null, score: null, extra: {} });
|
|
19
|
+
}
|
|
20
|
+
return { blocked, items };
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export const command = cli({
|
|
24
|
+
site: 'gitlab', name: 'search', access: 'read', description: 'Search GitLab', domain: 'gitlab.com', strategy: Strategy.PUBLIC, browser: true,
|
|
25
|
+
args: [{ name: 'keyword', positional: true, required: true, help: 'Search query' }, { name: 'limit', type: 'int', default: 10, help: 'Number of results (1-50)' }, { name: 'page', type: 'int', default: 1, help: 'Result page number' }, { name: 'scope', help: 'Search scope: projects, issues, merge_requests, commits, blobs, users' }, { name: 'order-by', help: 'Order by: created_at, updated_at, latest_activity_at' }, { name: 'sort', help: 'Sort direction: asc or desc' }],
|
|
26
|
+
columns: ['rank', 'title', 'url', 'snippet', 'displayUrl', 'source', 'resultType', 'author', 'publishedAt', 'score', 'extra'],
|
|
27
|
+
func: async (page, kwargs) => {
|
|
28
|
+
const keyword = requireSearchQuery(kwargs.keyword); const limit = requireBoundedInteger(kwargs.limit, 10, 1, 50, '--limit'); const pageNumber = requireBoundedInteger(kwargs.page, 1, 1, 100, '--page'); const scope = kwargs.scope || 'projects'; const orderBy = kwargs['order-by'] ?? kwargs.orderBy; const sort = kwargs.sort;
|
|
29
|
+
if (!SCOPES.includes(String(scope))) throw new ArgumentError(`--scope must be one of: ${SCOPES.join(', ')}`); if (orderBy && !ORDER.includes(String(orderBy))) throw new ArgumentError(`--order-by must be one of: ${ORDER.join(', ')}`); if (sort && !SORT.includes(String(sort))) throw new ArgumentError('--sort must be one of: asc, desc');
|
|
30
|
+
await runBrowserStep('GitLab search navigation', () => page.goto(buildSearchUrl({ keyword, limit, page: pageNumber, scope, orderBy, sort }))); await page.wait({ selector: '.search-result-row, .search-results', timeout: 8 }).catch(() => page.wait(2).catch(() => {}));
|
|
31
|
+
const data = await runBrowserStep('GitLab search extraction', () => page.evaluate(`(${extractItems.toString()})(${JSON.stringify(scope)})`)); if (data?.blocked) throw new CommandExecutionError('GitLab search requires sign-in or was blocked', 'Open GitLab in the browser, sign in if needed, and retry.'); if (!data?.items?.length) throw emptySearchResults('GitLab', keyword);
|
|
32
|
+
return data.items.slice(0, limit).map((item, index) => ({ rank: (pageNumber - 1) * limit + index + 1, ...item }));
|
|
33
|
+
},
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
export const __test__ = { command, buildSearchUrl };
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
2
|
+
import { ArgumentError, CommandExecutionError } from '@sovovs/bycli/errors';
|
|
3
|
+
import { emptySearchResults, requireBoundedInteger, requireSearchQuery, runBrowserStep } from '../_shared/search-adapter.js';
|
|
4
|
+
|
|
5
|
+
const TYPES = ['web', 'news', 'video', 'image']; const SAFE = ['off', 'on'];
|
|
6
|
+
|
|
7
|
+
export function buildSearchUrl({ keyword, limit = 10, page = 1, type, safe }) {
|
|
8
|
+
const url = new URL('https://so.com/s'); url.searchParams.set('q', keyword); url.searchParams.set('pn', String(page)); url.searchParams.set('num', String(limit));
|
|
9
|
+
if (type && type !== 'web') url.searchParams.set('type', type); if (safe) url.searchParams.set('safe', safe); return url.toString();
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function extractItems() {
|
|
13
|
+
const blocked = /验证码|安全验证|访问异常|captcha|access denied/i.test(document.body?.textContent || ''); const items = []; const seen = new Set();
|
|
14
|
+
for (const card of document.querySelectorAll('.result, .res-list')) {
|
|
15
|
+
if (/result-ad|ad-result|推广/.test(card.className || '') || /推广/.test(card.textContent || '')) continue;
|
|
16
|
+
const anchor = card.querySelector('h3 a[href], h2 a[href]'); if (!anchor || !/^https?:/i.test(anchor.href) || seen.has(anchor.href)) continue;
|
|
17
|
+
seen.add(anchor.href); items.push({ title: (anchor.textContent || '').trim(), url: anchor.href, snippet: (card.querySelector('.res-desc, .res-rich, p')?.textContent || '').trim(), displayUrl: (card.querySelector('cite, .res-link')?.textContent || '').trim(), resultType: 'web', extra: {} });
|
|
18
|
+
}
|
|
19
|
+
return { blocked, items };
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export const command = cli({
|
|
23
|
+
site: 'so', name: 'search', access: 'read', description: 'Search 360 Search', domain: 'so.com', strategy: Strategy.PUBLIC, browser: true,
|
|
24
|
+
args: [{ name: 'keyword', positional: true, required: true, help: 'Search query' }, { name: 'limit', type: 'int', default: 10, help: 'Number of results (1-50)' }, { name: 'page', type: 'int', default: 1, help: 'Result page number' }, { name: 'type', help: 'Result type: web, news, video, image' }, { name: 'safe', help: 'Safe search: off or on' }],
|
|
25
|
+
columns: ['rank', 'title', 'url', 'snippet', 'displayUrl', 'source', 'resultType', 'author', 'publishedAt', 'score', 'extra'],
|
|
26
|
+
func: async (page, kwargs) => {
|
|
27
|
+
const keyword = requireSearchQuery(kwargs.keyword); const limit = requireBoundedInteger(kwargs.limit, 10, 1, 50, '--limit'); const pageNumber = requireBoundedInteger(kwargs.page, 1, 1, 100, '--page');
|
|
28
|
+
if (kwargs.type !== undefined && !TYPES.includes(String(kwargs.type))) throw new ArgumentError(`--type must be one of: ${TYPES.join(', ')}`); if (kwargs.safe !== undefined && !SAFE.includes(String(kwargs.safe))) throw new ArgumentError('--safe must be one of: off, on');
|
|
29
|
+
await runBrowserStep('360 search navigation', () => page.goto(buildSearchUrl({ keyword, limit, page: pageNumber, type: kwargs.type, safe: kwargs.safe })));
|
|
30
|
+
await page.wait({ selector: '.result, .res-list', timeout: 8 }).catch(() => page.wait(2).catch(() => {})); const data = await runBrowserStep('360 search extraction', () => page.evaluate(`(${extractItems.toString()})()`));
|
|
31
|
+
if (data?.blocked) throw new CommandExecutionError('360 Search was blocked by a verification page', 'Complete the verification in the browser and retry.'); if (!data?.items?.length) throw emptySearchResults('360 Search', keyword);
|
|
32
|
+
return data.items.slice(0, limit).map((item, index) => ({ rank: (pageNumber - 1) * limit + index + 1, ...item, source: 'so', author: null, publishedAt: null, score: null }));
|
|
33
|
+
},
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
export const __test__ = { command, buildSearchUrl };
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
2
|
+
import { ArgumentError, CommandExecutionError } from '@sovovs/bycli/errors';
|
|
3
|
+
import { emptySearchResults, requireBoundedInteger, requireSearchQuery, runBrowserStep } from '../_shared/search-adapter.js';
|
|
4
|
+
|
|
5
|
+
const TYPES = ['web', 'news', 'video', 'image']; const TIMES = ['day', 'week', 'month', 'year']; const SORTS = ['relevance', 'date'];
|
|
6
|
+
|
|
7
|
+
export function buildSearchUrl({ keyword, limit = 10, page = 1, type, time, sort }) {
|
|
8
|
+
const url = new URL('https://sogou.com/web'); url.searchParams.set('query', keyword); url.searchParams.set('num', String(limit)); url.searchParams.set('page', String(page));
|
|
9
|
+
if (type && type !== 'web') url.searchParams.set('type', type); if (time) url.searchParams.set('tsn', time); if (sort === 'date') url.searchParams.set('sort', 'time'); return url.toString();
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function extractItems() {
|
|
13
|
+
const blocked = /验证码|安全验证|访问异常|请完成验证|captcha/i.test(document.body?.textContent || ''); const items = []; const seen = new Set();
|
|
14
|
+
for (const card of document.querySelectorAll('.vrwrap, .rb')) {
|
|
15
|
+
const anchor = card.querySelector('h3 a[href], h4 a[href]'); if (!anchor || !/^https?:/i.test(anchor.href) || seen.has(anchor.href)) continue;
|
|
16
|
+
if (/推广|广告/.test(card.textContent || '')) continue; seen.add(anchor.href); items.push({ title: (anchor.textContent || '').trim(), url: anchor.href, snippet: (card.querySelector('.str_info, .ft, .text-layout')?.textContent || '').trim(), displayUrl: (card.querySelector('cite, .citeurl')?.textContent || '').trim(), resultType: 'web', extra: {} });
|
|
17
|
+
}
|
|
18
|
+
return { blocked, items };
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export const command = cli({
|
|
22
|
+
site: 'sogou', name: 'search', access: 'read', description: 'Search Sogou', domain: 'sogou.com', strategy: Strategy.PUBLIC, browser: true,
|
|
23
|
+
args: [{ name: 'keyword', positional: true, required: true, help: 'Search query' }, { name: 'limit', type: 'int', default: 10, help: 'Number of results (1-50)' }, { name: 'page', type: 'int', default: 1, help: 'Result page number' }, { name: 'type', help: 'Result type: web, news, video, image' }, { name: 'time', help: 'Time range: day, week, month, year' }, { name: 'sort', help: 'Sort order: relevance or date' }],
|
|
24
|
+
columns: ['rank', 'title', 'url', 'snippet', 'displayUrl', 'source', 'resultType', 'author', 'publishedAt', 'score', 'extra'],
|
|
25
|
+
func: async (page, kwargs) => {
|
|
26
|
+
const keyword = requireSearchQuery(kwargs.keyword); const limit = requireBoundedInteger(kwargs.limit, 10, 1, 50, '--limit'); const pageNumber = requireBoundedInteger(kwargs.page, 1, 1, 100, '--page');
|
|
27
|
+
for (const [value, choices, flag] of [[kwargs.type, TYPES, 'type'], [kwargs.time, TIMES, 'time'], [kwargs.sort, SORTS, 'sort']]) if (value !== undefined && !choices.includes(String(value))) throw new ArgumentError(`--${flag} must be one of: ${choices.join(', ')}`);
|
|
28
|
+
await runBrowserStep('Sogou search navigation', () => page.goto(buildSearchUrl({ keyword, limit, page: pageNumber, type: kwargs.type, time: kwargs.time, sort: kwargs.sort })));
|
|
29
|
+
await page.wait({ selector: '.vrwrap, .rb', timeout: 8 }).catch(() => page.wait(2).catch(() => {})); const data = await runBrowserStep('Sogou search extraction', () => page.evaluate(`(${extractItems.toString()})()`));
|
|
30
|
+
if (data?.blocked) throw new CommandExecutionError('Sogou search was blocked by a verification page', 'Complete the verification in the browser and retry.'); if (!data?.items?.length) throw emptySearchResults('Sogou', keyword);
|
|
31
|
+
return data.items.slice(0, limit).map((item, index) => ({ rank: (pageNumber - 1) * limit + index + 1, ...item, source: 'sogou', author: null, publishedAt: null, score: null }));
|
|
32
|
+
},
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
export const __test__ = { command, buildSearchUrl };
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
2
|
+
import { ArgumentError, AuthRequiredError, CommandExecutionError } from '@sovovs/bycli/errors';
|
|
3
|
+
import { emptySearchResults, requireBoundedInteger, requireSearchQuery, runBrowserStep } from '../_shared/search-adapter.js';
|
|
4
|
+
|
|
5
|
+
export function buildSearchUrl({ keyword, limit = 10, page = 1, author, since, until }) {
|
|
6
|
+
const url = new URL('https://www.threads.com/search'); url.searchParams.set('q', keyword); url.searchParams.set('limit', String(limit)); url.searchParams.set('page', String(page));
|
|
7
|
+
if (author) url.searchParams.set('author', String(author).replace(/^@+/, '')); if (since) url.searchParams.set('since', since); if (until) url.searchParams.set('until', until); return url.toString();
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
function extractItems() {
|
|
11
|
+
const body = document.body?.textContent || ''; const authRequired = /log in|sign up|登录|注册/i.test(body) && !document.querySelector('article[data-pressable-container], article'); const blocked = /captcha|unusual activity|try again later|暂时无法/i.test(body); const items = []; const seen = new Set();
|
|
12
|
+
for (const card of document.querySelectorAll('article[data-pressable-container], article')) {
|
|
13
|
+
const anchor = card.querySelector('a[href*="/post/"]'); if (!anchor || !/^https?:/i.test(anchor.href) || seen.has(anchor.href)) continue; seen.add(anchor.href);
|
|
14
|
+
const authorAnchor = card.querySelector('a[href^="/@"], a[href*="/@"]'); const author = (authorAnchor?.textContent || authorAnchor?.getAttribute('href') || '').trim().replace(/^@+/, '').replace(/^\//, '').split('/')[0] || null; const time = card.querySelector('time');
|
|
15
|
+
items.push({ title: (card.querySelector('.text, [data-pressable-container] div')?.textContent || card.textContent || '').trim().slice(0, 500), url: anchor.href, snippet: (card.textContent || '').trim().slice(0, 500), displayUrl: new URL(anchor.href).hostname + new URL(anchor.href).pathname, source: 'threads', resultType: 'post', author, publishedAt: time?.getAttribute('datetime') || null, score: null, extra: {} });
|
|
16
|
+
}
|
|
17
|
+
return { authRequired, blocked, items };
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export const command = cli({
|
|
21
|
+
site: 'threads', name: 'search', access: 'read', description: 'Search Threads', domain: 'www.threads.com', strategy: Strategy.PUBLIC, browser: true,
|
|
22
|
+
args: [{ name: 'keyword', positional: true, required: true, help: 'Search query' }, { name: 'limit', type: 'int', default: 10, help: 'Number of posts (1-50)' }, { name: 'page', type: 'int', default: 1, help: 'Result page number' }, { name: 'author', help: 'Filter by author handle' }, { name: 'since', help: 'Only posts on or after YYYY-MM-DD' }, { name: 'until', help: 'Only posts on or before YYYY-MM-DD' }],
|
|
23
|
+
columns: ['rank', 'title', 'url', 'snippet', 'displayUrl', 'source', 'resultType', 'author', 'publishedAt', 'score', 'extra'],
|
|
24
|
+
func: async (page, kwargs) => {
|
|
25
|
+
const keyword = requireSearchQuery(kwargs.keyword); const limit = requireBoundedInteger(kwargs.limit, 10, 1, 50, '--limit'); const pageNumber = requireBoundedInteger(kwargs.page, 1, 1, 100, '--page');
|
|
26
|
+
for (const [value, flag] of [[kwargs.since, 'since'], [kwargs.until, 'until']]) if (value !== undefined && !/^\d{4}-\d{2}-\d{2}$/.test(String(value))) throw new ArgumentError(`--${flag} must use YYYY-MM-DD format`);
|
|
27
|
+
await runBrowserStep('Threads search navigation', () => page.goto(buildSearchUrl({ keyword, limit, page: pageNumber, author: kwargs.author, since: kwargs.since, until: kwargs.until }))); await page.wait({ selector: 'article', timeout: 8 }).catch(() => page.wait(2).catch(() => {}));
|
|
28
|
+
const data = await runBrowserStep('Threads search extraction', () => page.evaluate(`(${extractItems.toString()})()`)); if (data?.authRequired) throw new AuthRequiredError('Threads search requires login', 'Open Threads in the browser and sign in before retrying.'); if (data?.blocked) throw new CommandExecutionError('Threads search was blocked by an anti-bot page', 'Complete the browser prompt and retry.'); if (!data?.items?.length) throw emptySearchResults('Threads', keyword);
|
|
29
|
+
return data.items.slice(0, limit).map((item, index) => ({ rank: (pageNumber - 1) * limit + index + 1, ...item }));
|
|
30
|
+
},
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
export const __test__ = { command, buildSearchUrl };
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Toutiao public site search — extracts rich result cards from the rendered
|
|
3
|
+
* public search page. No authentication required.
|
|
4
|
+
*/
|
|
5
|
+
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
6
|
+
import { CommandExecutionError, EmptyResultError } from '@sovovs/bycli/errors';
|
|
7
|
+
import { parseSearchLimit, parseSearchType, parseToutiaoSearchHtml, TOUTIAO_SEARCH_URL } from './utils.js';
|
|
8
|
+
|
|
9
|
+
cli({
|
|
10
|
+
site: 'toutiao',
|
|
11
|
+
name: 'search',
|
|
12
|
+
access: 'read',
|
|
13
|
+
description: '搜索今日头条公开站内内容',
|
|
14
|
+
domain: 'www.toutiao.com',
|
|
15
|
+
strategy: Strategy.PUBLIC,
|
|
16
|
+
browser: false,
|
|
17
|
+
args: [
|
|
18
|
+
{ name: 'query', type: 'string', positional: true, help: '搜索关键词' },
|
|
19
|
+
{ name: 'type', type: 'string', default: 'synthesis', help: '搜索类型 (synthesis/information/video/atlas/user/xiaoshipin/weitoutiao/music)' },
|
|
20
|
+
{ name: 'limit', type: 'int', default: 20, help: '返回条数 (1-50)' },
|
|
21
|
+
],
|
|
22
|
+
columns: [
|
|
23
|
+
'rank', 'title', 'url', 'source', 'publish_time', 'summary', 'image_url',
|
|
24
|
+
'like_count', 'comment_count', 'share_count', 'read_count',
|
|
25
|
+
],
|
|
26
|
+
func: async (_page, kwargs) => {
|
|
27
|
+
const query = String(kwargs?.query ?? '').trim();
|
|
28
|
+
if (!query) throw new CommandExecutionError('toutiao search requires a non-empty query');
|
|
29
|
+
const type = parseSearchType(kwargs?.type, 'synthesis');
|
|
30
|
+
const limit = parseSearchLimit(kwargs?.limit, 20);
|
|
31
|
+
const url = new URL(TOUTIAO_SEARCH_URL);
|
|
32
|
+
url.searchParams.set('keyword', query);
|
|
33
|
+
url.searchParams.set('pd', type);
|
|
34
|
+
url.searchParams.set('page_num', '0');
|
|
35
|
+
|
|
36
|
+
let resp;
|
|
37
|
+
try {
|
|
38
|
+
resp = await fetch(url, {
|
|
39
|
+
headers: {
|
|
40
|
+
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
|
41
|
+
Accept: 'text/html,application/xhtml+xml',
|
|
42
|
+
Referer: 'https://www.toutiao.com/',
|
|
43
|
+
},
|
|
44
|
+
});
|
|
45
|
+
} catch (error) {
|
|
46
|
+
throw new CommandExecutionError(`toutiao search request failed: ${error?.message || error}`);
|
|
47
|
+
}
|
|
48
|
+
if (!resp.ok) {
|
|
49
|
+
throw new CommandExecutionError(`toutiao search failed: HTTP ${resp.status}`);
|
|
50
|
+
}
|
|
51
|
+
let html;
|
|
52
|
+
try {
|
|
53
|
+
html = await resp.text();
|
|
54
|
+
} catch (error) {
|
|
55
|
+
throw new CommandExecutionError(`toutiao search response read failed: ${error?.message || error}`);
|
|
56
|
+
}
|
|
57
|
+
const rows = parseToutiaoSearchHtml(html, limit);
|
|
58
|
+
if (rows.length === 0) {
|
|
59
|
+
throw new EmptyResultError('toutiao search', `未找到与「${query}」相关的公开内容。`);
|
|
60
|
+
}
|
|
61
|
+
return rows;
|
|
62
|
+
},
|
|
63
|
+
});
|
package/clis/toutiao/utils.js
CHANGED
|
@@ -7,6 +7,9 @@ const ARTICLES_MIN_PAGE = 1;
|
|
|
7
7
|
const ARTICLES_MAX_PAGE = 4;
|
|
8
8
|
const HOT_MIN_LIMIT = 1;
|
|
9
9
|
const HOT_MAX_LIMIT = 50;
|
|
10
|
+
const SEARCH_MIN_LIMIT = 1;
|
|
11
|
+
const SEARCH_MAX_LIMIT = 50;
|
|
12
|
+
const SEARCH_TYPES = ['synthesis', 'information', 'video', 'atlas', 'user', 'xiaoshipin', 'weitoutiao', 'music'];
|
|
10
13
|
|
|
11
14
|
export function parseArticlesPage(raw, fallback = 1) {
|
|
12
15
|
if (raw === undefined || raw === null || raw === '') return fallback;
|
|
@@ -32,6 +35,27 @@ export function parseHotLimit(raw, fallback = 30) {
|
|
|
32
35
|
return parsed;
|
|
33
36
|
}
|
|
34
37
|
|
|
38
|
+
export function parseSearchLimit(raw, fallback = 20) {
|
|
39
|
+
if (raw === undefined || raw === null || raw === '') return fallback;
|
|
40
|
+
const parsed = Number(raw);
|
|
41
|
+
if (!Number.isFinite(parsed) || !Number.isInteger(parsed)) {
|
|
42
|
+
throw new ArgumentError(`--limit must be an integer between ${SEARCH_MIN_LIMIT} and ${SEARCH_MAX_LIMIT}, got ${JSON.stringify(raw)}`);
|
|
43
|
+
}
|
|
44
|
+
if (parsed < SEARCH_MIN_LIMIT || parsed > SEARCH_MAX_LIMIT) {
|
|
45
|
+
throw new ArgumentError(`--limit must be between ${SEARCH_MIN_LIMIT} and ${SEARCH_MAX_LIMIT}, got ${parsed}`);
|
|
46
|
+
}
|
|
47
|
+
return parsed;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function parseSearchType(raw, fallback = 'synthesis') {
|
|
51
|
+
if (raw === undefined || raw === null || raw === '') return fallback;
|
|
52
|
+
const value = String(raw).trim();
|
|
53
|
+
if (!SEARCH_TYPES.includes(value)) {
|
|
54
|
+
throw new ArgumentError(`--type must be one of ${SEARCH_TYPES.join(', ')}, got ${JSON.stringify(raw)}`);
|
|
55
|
+
}
|
|
56
|
+
return value;
|
|
57
|
+
}
|
|
58
|
+
|
|
35
59
|
const NON_TITLE_LINES = new Set([
|
|
36
60
|
'展现', '阅读', '点赞', '评论',
|
|
37
61
|
'查看数据', '查看评论', '修改', '更多', '首发',
|
|
@@ -149,6 +173,83 @@ export function mapHotRow(item, index) {
|
|
|
149
173
|
}
|
|
150
174
|
|
|
151
175
|
export const HOT_BOARD_URL = 'https://www.toutiao.com/hot-event/hot-board/?origin=toutiao_pc';
|
|
176
|
+
export const TOUTIAO_SEARCH_URL = 'https://www.toutiao.com/search/';
|
|
177
|
+
|
|
178
|
+
function parseSearchNumber(value) {
|
|
179
|
+
if (value === undefined || value === null || value === '') return null;
|
|
180
|
+
const parsed = Number(String(value).replace(/,/g, ''));
|
|
181
|
+
return Number.isFinite(parsed) && parsed >= 0 ? Math.trunc(parsed) : null;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function absoluteSearchUrl(value) {
|
|
185
|
+
const url = trimOrNull(value);
|
|
186
|
+
if (!url) return null;
|
|
187
|
+
try {
|
|
188
|
+
return new URL(url, 'https://www.toutiao.com/').toString();
|
|
189
|
+
} catch {
|
|
190
|
+
return null;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function searchImage(item) {
|
|
195
|
+
const candidates = [
|
|
196
|
+
item?.image_url,
|
|
197
|
+
item?.large_image_url,
|
|
198
|
+
item?.other_image_url,
|
|
199
|
+
...(Array.isArray(item?.image_list) ? item.image_list.map((image) => image?.url || image) : []),
|
|
200
|
+
...(Array.isArray(item?.detail_image_list) ? item.detail_image_list.map((image) => image?.url || image) : []),
|
|
201
|
+
];
|
|
202
|
+
return candidates.map(trimOrNull).find(Boolean) || null;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function searchRowFromCard(item, index) {
|
|
206
|
+
if (!item || typeof item !== 'object') return null;
|
|
207
|
+
const title = trimOrNull(item.title);
|
|
208
|
+
const url = absoluteSearchUrl(
|
|
209
|
+
item.article_url || item.open_url || item.source_url || item.item_source_url || item?.display?.info?.url,
|
|
210
|
+
);
|
|
211
|
+
if (!title || !url) return null;
|
|
212
|
+
return {
|
|
213
|
+
rank: index + 1,
|
|
214
|
+
title,
|
|
215
|
+
url,
|
|
216
|
+
source: trimOrNull(item.source || item.media_name),
|
|
217
|
+
publish_time: trimOrNull(item.datetime || item.publish_time || item.display_time),
|
|
218
|
+
summary: trimOrNull(item.abstract || item.summary || item?.emphasized?.summary),
|
|
219
|
+
image_url: searchImage(item),
|
|
220
|
+
like_count: parseSearchNumber(item.like_count ?? item.digg_count),
|
|
221
|
+
comment_count: parseSearchNumber(item.comment_count),
|
|
222
|
+
share_count: parseSearchNumber(item.share_count ?? item.repin_count ?? item.forward_count),
|
|
223
|
+
read_count: parseSearchNumber(item.read_count),
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* Extract search result cards embedded in the public Toutiao search page.
|
|
229
|
+
* The page currently serializes each result as an application/json script.
|
|
230
|
+
*/
|
|
231
|
+
export function parseToutiaoSearchHtml(html, limit = 20) {
|
|
232
|
+
const source = String(html || '');
|
|
233
|
+
const rows = [];
|
|
234
|
+
const seenUrls = new Set();
|
|
235
|
+
const scriptPattern = /<script\b[^>]*type=["']application\/json["'][^>]*>([\s\S]*?)<\/script>/gi;
|
|
236
|
+
let match;
|
|
237
|
+
while ((match = scriptPattern.exec(source))) {
|
|
238
|
+
let payload;
|
|
239
|
+
try {
|
|
240
|
+
payload = JSON.parse(match[1]);
|
|
241
|
+
} catch {
|
|
242
|
+
continue;
|
|
243
|
+
}
|
|
244
|
+
const candidate = payload?.data;
|
|
245
|
+
const row = searchRowFromCard(candidate, rows.length);
|
|
246
|
+
if (!row || seenUrls.has(row.url)) continue;
|
|
247
|
+
seenUrls.add(row.url);
|
|
248
|
+
rows.push(row);
|
|
249
|
+
if (rows.length >= limit) break;
|
|
250
|
+
}
|
|
251
|
+
return rows.map((row, index) => ({ ...row, rank: index + 1 }));
|
|
252
|
+
}
|
|
152
253
|
|
|
153
254
|
export function looksToutiaoAuthWallText(value) {
|
|
154
255
|
const text = String(value || '').replace(/\s+/g, ' ').trim().toLowerCase();
|
|
@@ -158,4 +259,12 @@ export function looksToutiaoAuthWallText(value) {
|
|
|
158
259
|
/mp\.toutiao\.com\/profile_v4\/login/.test(text);
|
|
159
260
|
}
|
|
160
261
|
|
|
161
|
-
export const __test__ = {
|
|
262
|
+
export const __test__ = {
|
|
263
|
+
ARTICLES_MIN_PAGE,
|
|
264
|
+
ARTICLES_MAX_PAGE,
|
|
265
|
+
HOT_MIN_LIMIT,
|
|
266
|
+
HOT_MAX_LIMIT,
|
|
267
|
+
SEARCH_MIN_LIMIT,
|
|
268
|
+
SEARCH_MAX_LIMIT,
|
|
269
|
+
SEARCH_TYPES,
|
|
270
|
+
};
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
2
|
+
import { ArgumentError, CommandExecutionError } from '@sovovs/bycli/errors';
|
|
3
|
+
import { emptySearchResults, requireBoundedInteger, requireSearchQuery, runBrowserStep } from '../_shared/search-adapter.js';
|
|
4
|
+
|
|
5
|
+
const SORTS = ['relevance', 'date'];
|
|
6
|
+
|
|
7
|
+
export function buildSearchUrl({ keyword, limit = 10, page = 1, lr, lang, sort }) {
|
|
8
|
+
const url = new URL('https://yandex.com/search/');
|
|
9
|
+
url.searchParams.set('text', keyword); url.searchParams.set('numdoc', String(limit)); url.searchParams.set('p', String(page - 1));
|
|
10
|
+
if (lr) url.searchParams.set('lr', lr); if (lang) url.searchParams.set('lang', lang); if (sort === 'date') url.searchParams.set('how', 'tm');
|
|
11
|
+
return url.toString();
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function extractItems() {
|
|
15
|
+
const blocked = /captcha|verify|access denied|consent/i.test(document.body?.textContent || '') && !document.querySelector('.serp-item');
|
|
16
|
+
const items = []; const seen = new Set();
|
|
17
|
+
for (const card of document.querySelectorAll('.serp-item')) {
|
|
18
|
+
const anchor = card.querySelector('h2 a[href], .OrganicTitle a[href]'); if (!anchor || !/^https?:/i.test(anchor.href) || seen.has(anchor.href)) continue;
|
|
19
|
+
seen.add(anchor.href); items.push({ title: (anchor.textContent || '').trim(), url: anchor.href, snippet: (card.querySelector('.OrganicText, .TextContainer')?.textContent || '').trim(), displayUrl: (card.querySelector('.Path, .OrganicUrl')?.textContent || '').trim(), resultType: 'web', extra: {} });
|
|
20
|
+
}
|
|
21
|
+
return { blocked, items };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export const command = cli({
|
|
25
|
+
site: 'yandex', name: 'search', access: 'read', description: 'Search Yandex', domain: 'yandex.com', strategy: Strategy.PUBLIC, browser: true,
|
|
26
|
+
args: [
|
|
27
|
+
{ name: 'keyword', positional: true, required: true, help: 'Search query' }, { name: 'limit', type: 'int', default: 10, help: 'Number of results (1-50)' }, { name: 'page', type: 'int', default: 1, help: 'Result page number' },
|
|
28
|
+
{ name: 'lr', help: 'Yandex region code' }, { name: 'lang', help: 'Language code' }, { name: 'sort', help: 'Sort order: relevance or date' },
|
|
29
|
+
],
|
|
30
|
+
columns: ['rank', 'title', 'url', 'snippet', 'displayUrl', 'source', 'resultType', 'author', 'publishedAt', 'score', 'extra'],
|
|
31
|
+
func: async (page, kwargs) => {
|
|
32
|
+
const keyword = requireSearchQuery(kwargs.keyword); const limit = requireBoundedInteger(kwargs.limit, 10, 1, 50, '--limit'); const pageNumber = requireBoundedInteger(kwargs.page, 1, 1, 100, '--page');
|
|
33
|
+
if (kwargs.sort !== undefined && !SORTS.includes(String(kwargs.sort))) throw new ArgumentError('--sort must be one of: relevance, date');
|
|
34
|
+
await runBrowserStep('Yandex search navigation', () => page.goto(buildSearchUrl({ keyword, limit, page: pageNumber, lr: kwargs.lr, lang: kwargs.lang, sort: kwargs.sort })));
|
|
35
|
+
await page.wait({ selector: '.serp-item', timeout: 8 }).catch(() => page.wait(2).catch(() => {}));
|
|
36
|
+
const data = await runBrowserStep('Yandex search extraction', () => page.evaluate(`(${extractItems.toString()})()`));
|
|
37
|
+
if (data?.blocked) throw new CommandExecutionError('Yandex search was blocked by a consent or verification page', 'Complete the page prompt in the browser and retry.');
|
|
38
|
+
if (!data?.items?.length) throw emptySearchResults('Yandex', keyword);
|
|
39
|
+
return data.items.slice(0, limit).map((item, index) => ({ rank: (pageNumber - 1) * limit + index + 1, ...item, source: 'yandex', author: null, publishedAt: null, score: null }));
|
|
40
|
+
},
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
export const __test__ = { command, buildSearchUrl };
|