@remixmate/cli 0.9.25 → 0.9.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -4
- package/README.zh-CN.md +8 -4
- package/dist/handlers/gen-voice.d.ts +11 -1
- package/dist/handlers/gen-voice.js +17 -1
- package/dist/manifest.json +166 -10
- package/package.json +3 -1
- package/skills/gen-script/SKILL.md +1 -0
- package/skills/gen-script/scripts/gen_script.py +52 -3
- package/skills/gen-script/skill.json +5 -0
- package/skills/gen-script/version.json +1 -1
- package/skills/gen-voice/SKILL.md +5 -3
- package/skills/gen-voice/skill.json +3 -8
- package/skills/render-video/scripts/render_video.py +32 -0
- package/skills/render-video/version.json +1 -1
- package/skills/web-read/SKILL.md +146 -0
- package/skills/web-read/skill.json +131 -0
- package/skills/web-screenshot/scripts/_media_screenshot/__init__.py +2 -0
- package/skills/web-screenshot/scripts/_media_screenshot/js/extract_article.js +373 -0
- package/skills/web-screenshot/scripts/_media_screenshot/reader.py +245 -0
- package/skills/web-screenshot/scripts/_media_screenshot/urlguard.py +90 -0
- package/skills/web-screenshot/scripts/read_page.py +136 -0
|
@@ -0,0 +1,373 @@
|
|
|
1
|
+
// Extract a page's main readable content as an ordered list of typed blocks.
|
|
2
|
+
// Injected via page.evaluate(js_loader.load("extract_article"), cfg).
|
|
3
|
+
//
|
|
4
|
+
// A compact Readability-style pass: strip chrome, score paragraph density to
|
|
5
|
+
// pick the content container, then walk it in document order. It returns
|
|
6
|
+
// structured blocks rather than a formatted string so that every output format
|
|
7
|
+
// (markdown / plain text / json) is a pure function of one single extraction —
|
|
8
|
+
// the alternative, formatting in the page and re-parsing in Python, loses the
|
|
9
|
+
// block boundaries that make truncation land somewhere readable.
|
|
10
|
+
//
|
|
11
|
+
// Runs against the live DOM and mutates it (junk removal). That is safe here:
|
|
12
|
+
// the page is headless and thrown away immediately after this call.
|
|
13
|
+
(cfg) => {
|
|
14
|
+
const opts = cfg || {};
|
|
15
|
+
const wantLinks = !!opts.includeLinks;
|
|
16
|
+
const wantImages = !!opts.includeImages;
|
|
17
|
+
const inlineMarkdown = opts.inline === 'markdown';
|
|
18
|
+
|
|
19
|
+
const squash = (s) => (s || '').replace(/\s+/g, ' ').trim();
|
|
20
|
+
const rawLen = (el) => squash(el.textContent).length;
|
|
21
|
+
|
|
22
|
+
// ── metadata (read before junk removal strips the <head> siblings) ────────
|
|
23
|
+
const meta = (sel, attr) => {
|
|
24
|
+
const el = document.querySelector(sel);
|
|
25
|
+
if (!el) return '';
|
|
26
|
+
return squash(attr ? el.getAttribute(attr) : el.textContent);
|
|
27
|
+
};
|
|
28
|
+
const metadata = {
|
|
29
|
+
title:
|
|
30
|
+
meta('meta[property="og:title"]', 'content') ||
|
|
31
|
+
meta('meta[name="twitter:title"]', 'content') ||
|
|
32
|
+
squash(document.title) ||
|
|
33
|
+
meta('h1'),
|
|
34
|
+
byline:
|
|
35
|
+
meta('meta[name="author"]', 'content') ||
|
|
36
|
+
meta('meta[property="article:author"]', 'content') ||
|
|
37
|
+
meta('[rel="author"]'),
|
|
38
|
+
siteName: meta('meta[property="og:site_name"]', 'content'),
|
|
39
|
+
publishedTime:
|
|
40
|
+
meta('meta[property="article:published_time"]', 'content') ||
|
|
41
|
+
meta('time[datetime]', 'datetime'),
|
|
42
|
+
description:
|
|
43
|
+
meta('meta[name="description"]', 'content') ||
|
|
44
|
+
meta('meta[property="og:description"]', 'content'),
|
|
45
|
+
lang: squash(document.documentElement.getAttribute('lang') || ''),
|
|
46
|
+
url: location.href,
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
const bodyCharCount = rawLen(document.body);
|
|
50
|
+
|
|
51
|
+
// ── 1. strip boilerplate (structural only, see below) ────────────────────
|
|
52
|
+
const JUNK_TAGS = [
|
|
53
|
+
'script', 'style', 'noscript', 'template', 'svg', 'canvas', 'iframe',
|
|
54
|
+
'form', 'button', 'select', 'textarea', 'nav', 'aside', 'footer',
|
|
55
|
+
'[aria-hidden="true"]', '[hidden]', '[role="navigation"]', '[role="banner"]',
|
|
56
|
+
'[role="complementary"]', '[role="search"]', '[role="dialog"]',
|
|
57
|
+
].join(',');
|
|
58
|
+
|
|
59
|
+
// Matched against class + id, on "words" delimited by -, _ or space.
|
|
60
|
+
const JUNK_WORDS =
|
|
61
|
+
/(^|[-_\s])(comments?|disqus|sidebar|side-?bar|related|recommend(ed|ations?)?|advert(isement)?|ads?|adsbygoogle|banner|breadcrumbs?|share|sharing|social|subscribe|newsletter|popup|modal|overlay|cookie|toolbar|pagination|pager|footer|masthead|menu|promo|sponsor|widget|skip-link|back-to-top)($|[-_\s])/i;
|
|
62
|
+
|
|
63
|
+
const isJunkClass = (el) => {
|
|
64
|
+
const tokens = `${el.className || ''} ${el.id || ''}`;
|
|
65
|
+
return typeof tokens === 'string' && JUNK_WORDS.test(tokens);
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
document.querySelectorAll(JUNK_TAGS).forEach((el) => {
|
|
69
|
+
try { el.remove(); } catch (e) { /* already detached */ }
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
// Class-based junk is NOT removed here — only scored down in step 2 and
|
|
73
|
+
// pruned inside the winner in step 3. Removing it up front looks tidier and
|
|
74
|
+
// is how this first went wrong: Tencent Cloud's whole page wrapper carries
|
|
75
|
+
// `show-ad`, the `ads?` rule matched it, and the extractor deleted the entire
|
|
76
|
+
// article before it had a chance to score anything. A class name is a hint
|
|
77
|
+
// about a subtree's role, never grounds for deleting an ancestor of the text
|
|
78
|
+
// we came for.
|
|
79
|
+
|
|
80
|
+
// ── 2. pick the content container ────────────────────────────────────────
|
|
81
|
+
const linkDensity = (el) => {
|
|
82
|
+
const total = rawLen(el);
|
|
83
|
+
if (!total) return 1;
|
|
84
|
+
let linked = 0;
|
|
85
|
+
el.querySelectorAll('a').forEach((a) => { linked += rawLen(a); });
|
|
86
|
+
return Math.min(1, linked / total);
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
const tagBonus = (el) => {
|
|
90
|
+
const tag = el.tagName;
|
|
91
|
+
if (isJunkClass(el)) return 0.2;
|
|
92
|
+
if (tag === 'ARTICLE' || tag === 'MAIN') return 1.5;
|
|
93
|
+
const tokens = `${el.className || ''} ${el.id || ''}`;
|
|
94
|
+
if (typeof tokens === 'string' &&
|
|
95
|
+
/(^|[-_\s])(article|post|content|entry|markdown-body|rich_media|story|body)($|[-_\s])/i.test(tokens)) {
|
|
96
|
+
return 1.25;
|
|
97
|
+
}
|
|
98
|
+
return 1;
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
let container = null;
|
|
102
|
+
let containerReason = '';
|
|
103
|
+
|
|
104
|
+
if (opts.selector) {
|
|
105
|
+
container = document.querySelector(opts.selector);
|
|
106
|
+
if (!container) {
|
|
107
|
+
return { error: 'selector-not-found', selector: opts.selector, metadata };
|
|
108
|
+
}
|
|
109
|
+
containerReason = `selector:${opts.selector}`;
|
|
110
|
+
} else {
|
|
111
|
+
// Readability's scoring:每个够长的段落给自己父/祖父/曾祖父记分(1/2、1/3 衰减),
|
|
112
|
+
// 所以赢家是"段落最密集的那层",而不是 <body>——<body> 拿不到直接段落的分。
|
|
113
|
+
const scores = new Map();
|
|
114
|
+
const add = (el, amount) => {
|
|
115
|
+
if (!el || el.nodeType !== 1 || el === document.body ||
|
|
116
|
+
el === document.documentElement) return;
|
|
117
|
+
scores.set(el, (scores.get(el) || 0) + amount);
|
|
118
|
+
};
|
|
119
|
+
document.querySelectorAll('p, pre, blockquote, li, td, h2, h3').forEach((node) => {
|
|
120
|
+
const text = squash(node.textContent);
|
|
121
|
+
if (text.length < 25) return;
|
|
122
|
+
const commas = (text.match(/[,,、.。;;]/g) || []).length;
|
|
123
|
+
let s = 1 + Math.min(text.length / 100, 3) + Math.min(commas, 3);
|
|
124
|
+
// 列表项 / 单元格常常是导航或目录,给一半权重。
|
|
125
|
+
if (node.tagName === 'LI' || node.tagName === 'TD') s *= 0.5;
|
|
126
|
+
const p = node.parentElement;
|
|
127
|
+
const gp = p && p.parentElement;
|
|
128
|
+
const ggp = gp && gp.parentElement;
|
|
129
|
+
add(p, s);
|
|
130
|
+
add(gp, s / 2);
|
|
131
|
+
add(ggp, s / 3);
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
let bestScore = 0;
|
|
135
|
+
scores.forEach((s, el) => {
|
|
136
|
+
const adjusted = s * (1 - linkDensity(el)) * tagBonus(el);
|
|
137
|
+
if (adjusted > bestScore) { bestScore = adjusted; container = el; }
|
|
138
|
+
});
|
|
139
|
+
containerReason = container ? `scored:${bestScore.toFixed(1)}` : 'fallback:body';
|
|
140
|
+
|
|
141
|
+
// 赢家常常只是正文里**最密的那一层**,而不是正文本身:docs.python.org 上
|
|
142
|
+
// 打分赢家是 `section#text`(第 3.1.2 节),它上面还有三节同级内容。所以向上
|
|
143
|
+
// 合并兄弟小节,直到父层开始引入噪音(体量暴涨或链接占比变高)为止。
|
|
144
|
+
let guard = 0;
|
|
145
|
+
while (container && container.parentElement &&
|
|
146
|
+
container.parentElement !== document.body &&
|
|
147
|
+
container.parentElement !== document.documentElement &&
|
|
148
|
+
guard++ < 8) {
|
|
149
|
+
const parent = container.parentElement;
|
|
150
|
+
if (isJunkClass(parent)) break;
|
|
151
|
+
const tooSmall = rawLen(container) < 200;
|
|
152
|
+
const growth = rawLen(parent) / Math.max(rawLen(container), 1);
|
|
153
|
+
if (!tooSmall && (growth > 3 || linkDensity(parent) > 0.3)) break;
|
|
154
|
+
container = parent;
|
|
155
|
+
containerReason += '+up';
|
|
156
|
+
}
|
|
157
|
+
if (!container) container = document.body;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// ── 3. prune junk *inside* the winner, then walk it into blocks ──────────
|
|
161
|
+
// Safe by construction: a descendant can never be the container, and the
|
|
162
|
+
// 40% rule keeps a mislabelled wrapper (the `show-ad` case, one level down)
|
|
163
|
+
// from taking the article with it.
|
|
164
|
+
const containerChars = rawLen(container) || 1;
|
|
165
|
+
Array.prototype.slice.call(container.querySelectorAll('div,section,ul,ol,span,header,p,figure'))
|
|
166
|
+
.forEach((el) => {
|
|
167
|
+
if (!container.contains(el) || el === container) return;
|
|
168
|
+
if (!isJunkClass(el)) return;
|
|
169
|
+
if (rawLen(el) / containerChars > 0.4) return;
|
|
170
|
+
try { el.remove(); } catch (e) { /* already detached */ }
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
const describe = (el) => {
|
|
174
|
+
if (!el || !el.tagName) return '';
|
|
175
|
+
const id = el.id ? `#${el.id}` : '';
|
|
176
|
+
const cls = typeof el.className === 'string' && el.className.trim()
|
|
177
|
+
? `.${el.className.trim().split(/\s+/).slice(0, 2).join('.')}`
|
|
178
|
+
: '';
|
|
179
|
+
return `${el.tagName.toLowerCase()}${id}${cls}`;
|
|
180
|
+
};
|
|
181
|
+
|
|
182
|
+
// Inline serializer — the only place that knows about markdown syntax.
|
|
183
|
+
const inline = (node) => {
|
|
184
|
+
let out = '';
|
|
185
|
+
node.childNodes.forEach((child) => {
|
|
186
|
+
if (child.nodeType === 3) { out += child.nodeValue; return; }
|
|
187
|
+
if (child.nodeType !== 1) return;
|
|
188
|
+
const tag = child.tagName;
|
|
189
|
+
if (tag === 'BR') { out += ' '; return; }
|
|
190
|
+
if (tag === 'IMG') {
|
|
191
|
+
if (wantImages) {
|
|
192
|
+
const alt = squash(child.getAttribute('alt'));
|
|
193
|
+
const src = child.currentSrc || child.src ||
|
|
194
|
+
child.getAttribute('data-src') || '';
|
|
195
|
+
if (src) out += ``;
|
|
196
|
+
}
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
const inner = inline(child);
|
|
200
|
+
if (!inner.trim()) return;
|
|
201
|
+
if (tag === 'A' && wantLinks) {
|
|
202
|
+
const href = child.href || child.getAttribute('href') || '';
|
|
203
|
+
out += href ? `[${inner}](${href})` : inner;
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
if (!inlineMarkdown) { out += inner; return; }
|
|
207
|
+
if (tag === 'CODE' || tag === 'KBD' || tag === 'SAMP') out += `\`${inner}\``;
|
|
208
|
+
else if (tag === 'STRONG' || tag === 'B') out += `**${inner}**`;
|
|
209
|
+
else if (tag === 'EM' || tag === 'I') out += `*${inner}*`;
|
|
210
|
+
else if (tag === 'DEL' || tag === 'S') out += `~~${inner}~~`;
|
|
211
|
+
else out += inner;
|
|
212
|
+
});
|
|
213
|
+
return out;
|
|
214
|
+
};
|
|
215
|
+
|
|
216
|
+
const text = (node) => squash(inline(node));
|
|
217
|
+
|
|
218
|
+
const listItems = (el, depth) => {
|
|
219
|
+
const items = [];
|
|
220
|
+
Array.prototype.forEach.call(el.children, (li) => {
|
|
221
|
+
if (li.tagName !== 'LI') return;
|
|
222
|
+
// Nested lists are flattened with an indent prefix rather than nested
|
|
223
|
+
// structures — every consumer here renders to flat text anyway.
|
|
224
|
+
const nested = li.querySelector(':scope > ul, :scope > ol');
|
|
225
|
+
const own = squash(
|
|
226
|
+
Array.prototype.filter
|
|
227
|
+
.call(li.childNodes, (n) => n !== nested)
|
|
228
|
+
.map((n) => (n.nodeType === 1 ? inline(n) : n.nodeValue || ''))
|
|
229
|
+
.join(''),
|
|
230
|
+
);
|
|
231
|
+
if (own) items.push({ depth, text: own });
|
|
232
|
+
if (nested && depth < 3) items.push(...listItems(nested, depth + 1));
|
|
233
|
+
});
|
|
234
|
+
return items;
|
|
235
|
+
};
|
|
236
|
+
|
|
237
|
+
const EMIT = new Set([
|
|
238
|
+
'H1', 'H2', 'H3', 'H4', 'H5', 'H6', 'P', 'PRE', 'BLOCKQUOTE',
|
|
239
|
+
'UL', 'OL', 'TABLE', 'FIGURE', 'IMG', 'HR', 'DL',
|
|
240
|
+
]);
|
|
241
|
+
const SKIP_DESCEND = new Set(['PRE', 'TABLE', 'UL', 'OL', 'FIGURE', 'DL']);
|
|
242
|
+
|
|
243
|
+
const blocks = [];
|
|
244
|
+
const push = (block) => {
|
|
245
|
+
if (!block) return;
|
|
246
|
+
const prev = blocks[blocks.length - 1];
|
|
247
|
+
// 同一段文字在页面里出现两次(懒加载占位 + 真身)时只留一份。
|
|
248
|
+
if (prev && prev.type === block.type && prev.text && prev.text === block.text) return;
|
|
249
|
+
blocks.push(block);
|
|
250
|
+
};
|
|
251
|
+
|
|
252
|
+
const emit = (el) => {
|
|
253
|
+
const tag = el.tagName;
|
|
254
|
+
if (/^H[1-6]$/.test(tag)) {
|
|
255
|
+
const t = text(el);
|
|
256
|
+
if (t) push({ type: 'heading', level: Number(tag[1]), text: t });
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
if (tag === 'P') {
|
|
260
|
+
const t = text(el);
|
|
261
|
+
if (t) push({ type: 'paragraph', text: t });
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
if (tag === 'PRE') {
|
|
265
|
+
const code = el.querySelector('code') || el;
|
|
266
|
+
const cls = `${el.className || ''} ${code.className || ''}`;
|
|
267
|
+
const m = typeof cls === 'string' ? cls.match(/(?:language|lang|highlight)[-_](\w+)/i) : null;
|
|
268
|
+
const body = (code.innerText || code.textContent || '').replace(/\s+$/, '');
|
|
269
|
+
if (body.trim()) push({ type: 'code', lang: m ? m[1].toLowerCase() : '', text: body });
|
|
270
|
+
return;
|
|
271
|
+
}
|
|
272
|
+
if (tag === 'BLOCKQUOTE') {
|
|
273
|
+
const t = text(el);
|
|
274
|
+
if (t) push({ type: 'quote', text: t });
|
|
275
|
+
return;
|
|
276
|
+
}
|
|
277
|
+
if (tag === 'UL' || tag === 'OL') {
|
|
278
|
+
const items = listItems(el, 0);
|
|
279
|
+
if (items.length) push({ type: 'list', ordered: tag === 'OL', items });
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
282
|
+
if (tag === 'DL') {
|
|
283
|
+
const items = [];
|
|
284
|
+
Array.prototype.forEach.call(el.children, (child) => {
|
|
285
|
+
const t = text(child);
|
|
286
|
+
if (!t) return;
|
|
287
|
+
items.push({ depth: child.tagName === 'DD' ? 1 : 0, text: t });
|
|
288
|
+
});
|
|
289
|
+
if (items.length) push({ type: 'list', ordered: false, items });
|
|
290
|
+
return;
|
|
291
|
+
}
|
|
292
|
+
if (tag === 'TABLE') {
|
|
293
|
+
const rows = [];
|
|
294
|
+
el.querySelectorAll('tr').forEach((tr) => {
|
|
295
|
+
const cells = [];
|
|
296
|
+
tr.querySelectorAll('th,td').forEach((cell) => cells.push(text(cell)));
|
|
297
|
+
if (cells.some((c) => c)) rows.push(cells);
|
|
298
|
+
});
|
|
299
|
+
if (rows.length) push({ type: 'table', rows });
|
|
300
|
+
return;
|
|
301
|
+
}
|
|
302
|
+
if (tag === 'FIGURE') {
|
|
303
|
+
const img = el.querySelector('img');
|
|
304
|
+
const cap = el.querySelector('figcaption');
|
|
305
|
+
const caption = cap ? text(cap) : '';
|
|
306
|
+
if (wantImages && img) {
|
|
307
|
+
const src = img.currentSrc || img.src || img.getAttribute('data-src') || '';
|
|
308
|
+
if (src) push({ type: 'image', src, alt: squash(img.getAttribute('alt')), caption });
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
311
|
+
if (caption) push({ type: 'paragraph', text: caption });
|
|
312
|
+
return;
|
|
313
|
+
}
|
|
314
|
+
if (tag === 'IMG') {
|
|
315
|
+
if (!wantImages) return;
|
|
316
|
+
const src = el.currentSrc || el.src || el.getAttribute('data-src') || '';
|
|
317
|
+
if (src) push({ type: 'image', src, alt: squash(el.getAttribute('alt')), caption: '' });
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
320
|
+
if (tag === 'HR') push({ type: 'rule' });
|
|
321
|
+
};
|
|
322
|
+
|
|
323
|
+
// 直接挂在容器上、没有被 <p> 包起来的裸文本(老网页很常见)也要收;
|
|
324
|
+
// 用 buffer 在遇到下一个块级元素或本层结束时冲刷成一个段落。
|
|
325
|
+
const walk = (el, depth) => {
|
|
326
|
+
if (depth > 24) return;
|
|
327
|
+
let buffer = '';
|
|
328
|
+
const flush = () => {
|
|
329
|
+
const t = squash(buffer);
|
|
330
|
+
buffer = '';
|
|
331
|
+
if (t.length >= 2) push({ type: 'paragraph', text: t });
|
|
332
|
+
};
|
|
333
|
+
el.childNodes.forEach((child) => {
|
|
334
|
+
if (child.nodeType === 3) { buffer += child.nodeValue; return; }
|
|
335
|
+
if (child.nodeType !== 1) return;
|
|
336
|
+
const tag = child.tagName;
|
|
337
|
+
if (EMIT.has(tag)) {
|
|
338
|
+
flush();
|
|
339
|
+
emit(child);
|
|
340
|
+
return;
|
|
341
|
+
}
|
|
342
|
+
if (SKIP_DESCEND.has(tag)) return;
|
|
343
|
+
if (tag === 'BR') { buffer += ' '; return; }
|
|
344
|
+
// Inline-level element sitting between block siblings: keep its text in
|
|
345
|
+
// the running buffer instead of dropping it on the floor.
|
|
346
|
+
if (['A', 'SPAN', 'CODE', 'STRONG', 'B', 'EM', 'I', 'SMALL', 'LABEL', 'TIME',
|
|
347
|
+
'SUP', 'SUB', 'MARK', 'ABBR', 'CITE', 'Q', 'DEL', 'S', 'U'].includes(tag)) {
|
|
348
|
+
buffer += inline(child);
|
|
349
|
+
return;
|
|
350
|
+
}
|
|
351
|
+
flush();
|
|
352
|
+
walk(child, depth + 1);
|
|
353
|
+
});
|
|
354
|
+
flush();
|
|
355
|
+
};
|
|
356
|
+
|
|
357
|
+
walk(container, 0);
|
|
358
|
+
|
|
359
|
+
const charCount = blocks.reduce((n, b) => {
|
|
360
|
+
if (b.type === 'list') return n + b.items.reduce((m, i) => m + i.text.length, 0);
|
|
361
|
+
if (b.type === 'table') return n + b.rows.reduce((m, r) => m + r.join('').length, 0);
|
|
362
|
+
return n + (b.text || '').length;
|
|
363
|
+
}, 0);
|
|
364
|
+
|
|
365
|
+
return {
|
|
366
|
+
metadata,
|
|
367
|
+
blocks,
|
|
368
|
+
container: describe(container),
|
|
369
|
+
containerReason,
|
|
370
|
+
charCount,
|
|
371
|
+
bodyCharCount,
|
|
372
|
+
};
|
|
373
|
+
}
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
"""Read operation: load a page and extract its main content as text.
|
|
2
|
+
|
|
3
|
+
The DOM work lives in ``js/extract_article.js`` (one injected pass returning
|
|
4
|
+
typed blocks); this module drives the browser, renders those blocks into the
|
|
5
|
+
requested format, and enforces the character budget.
|
|
6
|
+
|
|
7
|
+
Truncation is the part that matters for the caller: a 200k-character page
|
|
8
|
+
pasted into a model's context is worse than useless. ``max_chars`` cuts on a
|
|
9
|
+
block boundary and says so in-band, and ``--output`` keeps the full text on
|
|
10
|
+
disk so nothing is actually lost.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from . import js_loader
|
|
18
|
+
from .bootstrap import ensure_runtime
|
|
19
|
+
from .browser import apply_pre_action_waits, build_context_options, launch_with_browser_install
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def extract(cfg: dict) -> dict:
|
|
23
|
+
"""Load cfg['url'] and return the raw extraction dict from the page."""
|
|
24
|
+
ensure_runtime()
|
|
25
|
+
from playwright.sync_api import sync_playwright
|
|
26
|
+
|
|
27
|
+
js_cfg = {
|
|
28
|
+
"selector": cfg.get("selector") or "",
|
|
29
|
+
"includeLinks": bool(cfg.get("includeLinks")),
|
|
30
|
+
"includeImages": bool(cfg.get("includeImages")),
|
|
31
|
+
"inline": "markdown" if cfg.get("format", "markdown") == "markdown" else "plain",
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
with sync_playwright() as p:
|
|
35
|
+
browser = launch_with_browser_install(p, cfg.get("browser"))
|
|
36
|
+
try:
|
|
37
|
+
context = browser.new_context(**build_context_options(p, cfg, None))
|
|
38
|
+
if cfg.get("timeout"):
|
|
39
|
+
context.set_default_timeout(cfg["timeout"])
|
|
40
|
+
page = context.new_page()
|
|
41
|
+
try:
|
|
42
|
+
try:
|
|
43
|
+
response = page.goto(cfg["url"], wait_until="domcontentloaded")
|
|
44
|
+
except Exception as e:
|
|
45
|
+
# 一条模型能据以行动的错误,胜过 30 行 Playwright traceback。
|
|
46
|
+
# 导航失败是这个 skill 最常见的失败形态(打不开 / 超时 / DNS),
|
|
47
|
+
# 让它读起来像"这个网址打不开",而不是像程序崩了。
|
|
48
|
+
raise SystemExit(
|
|
49
|
+
f"打开页面失败:{cfg['url']}\n"
|
|
50
|
+
f"{type(e).__name__}: {str(e).splitlines()[0]}\n"
|
|
51
|
+
"可能是网络不可达、站点屏蔽了无头浏览器,或加载超过了超时时间。"
|
|
52
|
+
"可尝试加大 --timeout、换 --user-agent,或确认该 URL 在本机能打开。"
|
|
53
|
+
)
|
|
54
|
+
status = response.status if response else None
|
|
55
|
+
if status is not None and status >= 400:
|
|
56
|
+
# Keep going: many sites serve real content under a 403/404
|
|
57
|
+
# (paywalls, soft 404s). The status rides along in the output
|
|
58
|
+
# so the caller can tell "empty page" from "blocked".
|
|
59
|
+
pass
|
|
60
|
+
apply_pre_action_waits(page, cfg)
|
|
61
|
+
# SPA 正文往往在 domcontentloaded 之后才注水。networkidle 等不到就
|
|
62
|
+
# 算了——静态页本来就不会再有请求,硬等只是白白花掉 20 秒。
|
|
63
|
+
try:
|
|
64
|
+
page.wait_for_load_state("networkidle", timeout=cfg.get("networkIdleMs", 8000))
|
|
65
|
+
except Exception:
|
|
66
|
+
pass
|
|
67
|
+
settle = cfg.get("settleMs")
|
|
68
|
+
if settle:
|
|
69
|
+
page.wait_for_timeout(settle)
|
|
70
|
+
|
|
71
|
+
data = page.evaluate(js_loader.load("extract_article"), js_cfg)
|
|
72
|
+
data["status"] = status
|
|
73
|
+
data["finalUrl"] = page.url
|
|
74
|
+
return data
|
|
75
|
+
finally:
|
|
76
|
+
context.close()
|
|
77
|
+
finally:
|
|
78
|
+
browser.close()
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
# ── rendering ────────────────────────────────────────────────────────────────
|
|
82
|
+
|
|
83
|
+
def _render_blocks(blocks: list, fmt: str) -> list[str]:
|
|
84
|
+
"""One string per block, ready to be joined with a blank line."""
|
|
85
|
+
md = fmt == "markdown"
|
|
86
|
+
out: list[str] = []
|
|
87
|
+
for block in blocks:
|
|
88
|
+
kind = block.get("type")
|
|
89
|
+
if kind == "heading":
|
|
90
|
+
level = int(block.get("level") or 2)
|
|
91
|
+
out.append(f"{'#' * min(level, 6)} {block['text']}" if md else block["text"].upper())
|
|
92
|
+
elif kind == "paragraph":
|
|
93
|
+
out.append(block["text"])
|
|
94
|
+
elif kind == "quote":
|
|
95
|
+
out.append(f"> {block['text']}" if md else f'"{block["text"]}"')
|
|
96
|
+
elif kind == "code":
|
|
97
|
+
body = block.get("text", "")
|
|
98
|
+
out.append(f"```{block.get('lang') or ''}\n{body}\n```" if md else body)
|
|
99
|
+
elif kind == "list":
|
|
100
|
+
ordered = bool(block.get("ordered"))
|
|
101
|
+
lines = []
|
|
102
|
+
counter = 1
|
|
103
|
+
for item in block.get("items") or []:
|
|
104
|
+
indent = " " * int(item.get("depth") or 0)
|
|
105
|
+
if ordered and not item.get("depth"):
|
|
106
|
+
lines.append(f"{indent}{counter}. {item['text']}")
|
|
107
|
+
counter += 1
|
|
108
|
+
else:
|
|
109
|
+
lines.append(f"{indent}- {item['text']}")
|
|
110
|
+
out.append("\n".join(lines))
|
|
111
|
+
elif kind == "table":
|
|
112
|
+
rows = block.get("rows") or []
|
|
113
|
+
if not rows:
|
|
114
|
+
continue
|
|
115
|
+
if md:
|
|
116
|
+
width = max(len(r) for r in rows)
|
|
117
|
+
padded = [r + [""] * (width - len(r)) for r in rows]
|
|
118
|
+
lines = ["| " + " | ".join(padded[0]) + " |",
|
|
119
|
+
"| " + " | ".join(["---"] * width) + " |"]
|
|
120
|
+
lines += ["| " + " | ".join(r) + " |" for r in padded[1:]]
|
|
121
|
+
out.append("\n".join(lines))
|
|
122
|
+
else:
|
|
123
|
+
out.append("\n".join("\t".join(r) for r in rows))
|
|
124
|
+
elif kind == "image":
|
|
125
|
+
src = block.get("src") or ""
|
|
126
|
+
alt = block.get("alt") or ""
|
|
127
|
+
caption = block.get("caption") or ""
|
|
128
|
+
if md:
|
|
129
|
+
line = f""
|
|
130
|
+
out.append(f"{line}\n{caption}" if caption else line)
|
|
131
|
+
else:
|
|
132
|
+
out.append(f"[image: {alt or caption or src}]")
|
|
133
|
+
elif kind == "rule":
|
|
134
|
+
out.append("---" if md else "—")
|
|
135
|
+
return [s for s in out if s.strip()]
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _header(data: dict, fmt: str) -> str:
|
|
139
|
+
meta = data.get("metadata") or {}
|
|
140
|
+
lines = []
|
|
141
|
+
title = meta.get("title")
|
|
142
|
+
if title:
|
|
143
|
+
lines.append(f"# {title}" if fmt == "markdown" else title)
|
|
144
|
+
trailer = []
|
|
145
|
+
if meta.get("siteName"):
|
|
146
|
+
trailer.append(meta["siteName"])
|
|
147
|
+
if meta.get("byline"):
|
|
148
|
+
trailer.append(meta["byline"])
|
|
149
|
+
if meta.get("publishedTime"):
|
|
150
|
+
trailer.append(meta["publishedTime"])
|
|
151
|
+
trailer.append(data.get("finalUrl") or meta.get("url") or "")
|
|
152
|
+
lines.append(" · ".join(t for t in trailer if t))
|
|
153
|
+
return "\n".join(lines)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def render(data: dict, cfg: dict) -> tuple[str, str, bool]:
|
|
157
|
+
"""Return (stdout_text, full_text, truncated).
|
|
158
|
+
|
|
159
|
+
``full_text`` is always the complete document; ``stdout_text`` is what the
|
|
160
|
+
caller should print, i.e. the same thing cut to ``maxChars``.
|
|
161
|
+
"""
|
|
162
|
+
fmt = cfg.get("format", "markdown")
|
|
163
|
+
blocks = list(data.get("blocks") or [])
|
|
164
|
+
# 标题已经在 header 里印过一次;正文第一个块又是同一句时,去掉重复的那个。
|
|
165
|
+
title = ((data.get("metadata") or {}).get("title") or "").strip()
|
|
166
|
+
if blocks and title and blocks[0].get("type") == "heading":
|
|
167
|
+
first = (blocks[0].get("text") or "").strip().rstrip("¶").strip()
|
|
168
|
+
if first and (first == title or title.startswith(first)):
|
|
169
|
+
blocks = blocks[1:]
|
|
170
|
+
|
|
171
|
+
body_parts = _render_blocks(blocks, fmt)
|
|
172
|
+
body = "\n\n".join(body_parts)
|
|
173
|
+
|
|
174
|
+
if fmt == "json":
|
|
175
|
+
payload = {
|
|
176
|
+
"url": data.get("finalUrl"),
|
|
177
|
+
"status": data.get("status"),
|
|
178
|
+
"metadata": data.get("metadata"),
|
|
179
|
+
"container": data.get("container"),
|
|
180
|
+
"charCount": data.get("charCount"),
|
|
181
|
+
"blocks": data.get("blocks"),
|
|
182
|
+
}
|
|
183
|
+
full = json.dumps(payload, ensure_ascii=False, indent=2)
|
|
184
|
+
# JSON 不做块级截断:切一半的 JSON 不是 JSON。超预算时只报告,
|
|
185
|
+
# 让调用方自己决定是改格式还是配 --output。
|
|
186
|
+
return full, full, False
|
|
187
|
+
|
|
188
|
+
full = f"{_header(data, fmt)}\n\n{body}".strip()
|
|
189
|
+
|
|
190
|
+
max_chars = int(cfg.get("maxChars") or 0)
|
|
191
|
+
if max_chars <= 0 or len(full) <= max_chars:
|
|
192
|
+
return full, full, False
|
|
193
|
+
|
|
194
|
+
# Cut on a block boundary so the tail is a whole paragraph, not half a word.
|
|
195
|
+
head = f"{_header(data, fmt)}\n\n"
|
|
196
|
+
kept: list[str] = []
|
|
197
|
+
used = len(head)
|
|
198
|
+
for part in body_parts:
|
|
199
|
+
if used + len(part) + 2 > max_chars:
|
|
200
|
+
break
|
|
201
|
+
kept.append(part)
|
|
202
|
+
used += len(part) + 2
|
|
203
|
+
if not kept:
|
|
204
|
+
# Single oversized block (one giant <pre>, say) — fall back to a hard cut.
|
|
205
|
+
kept = [body[: max(0, max_chars - len(head))]]
|
|
206
|
+
shown = (head + "\n\n".join(kept)).strip()
|
|
207
|
+
notice = (
|
|
208
|
+
f"\n\n---\n[truncated] 已显示 {len(shown)} / {len(full)} 字符。"
|
|
209
|
+
"需要全文时加大 --max-chars(0 = 不限),或用 --output 把全文写到文件再按需读取。"
|
|
210
|
+
)
|
|
211
|
+
return shown + notice, full, True
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def do_read(cfg: dict) -> dict:
|
|
215
|
+
"""Extract, render, optionally persist. Returns a small result summary."""
|
|
216
|
+
data = extract(cfg)
|
|
217
|
+
|
|
218
|
+
if data.get("error") == "selector-not-found":
|
|
219
|
+
raise SystemExit(
|
|
220
|
+
f'selector "{data.get("selector")}" 在页面上未找到({cfg["url"]})。'
|
|
221
|
+
"请换一个选择器,或去掉 --selector 让正文自动识别。"
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
stdout_text, full_text, truncated = render(data, cfg)
|
|
225
|
+
|
|
226
|
+
out_path = None
|
|
227
|
+
if cfg.get("output"):
|
|
228
|
+
out_path = Path(cfg["output"]).resolve()
|
|
229
|
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
230
|
+
out_path.write_text(full_text, encoding="utf-8")
|
|
231
|
+
|
|
232
|
+
return {
|
|
233
|
+
"text": stdout_text,
|
|
234
|
+
"fullText": full_text,
|
|
235
|
+
"truncated": truncated,
|
|
236
|
+
"outputPath": str(out_path) if out_path else None,
|
|
237
|
+
"charCount": data.get("charCount") or 0,
|
|
238
|
+
"bodyCharCount": data.get("bodyCharCount") or 0,
|
|
239
|
+
"blockCount": len(data.get("blocks") or []),
|
|
240
|
+
"container": data.get("container"),
|
|
241
|
+
"containerReason": data.get("containerReason"),
|
|
242
|
+
"status": data.get("status"),
|
|
243
|
+
"finalUrl": data.get("finalUrl"),
|
|
244
|
+
"metadata": data.get("metadata") or {},
|
|
245
|
+
}
|