dsh-web-fetch-playwright 0.2.19 → 0.2.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +16 -16
- package/lib/index.js +57160 -379
- package/lib/types/block-model.d.ts +2 -0
- package/lib/types/blocks.d.ts +67 -0
- package/lib/types/candidates.d.ts +157 -0
- package/lib/types/index.d.ts +4 -2
- package/lib/types/markdown.d.ts +50 -24
- package/lib/types/selector-model.d.ts +2 -0
- package/lib/types/selector.d.ts +58 -0
- package/lib/types/trees.d.ts +25 -0
- package/package.json +1 -1
- package/src/block-model.ts +5 -0
- package/src/blocks.ts +223 -0
- package/src/candidates.ts +639 -0
- package/src/index.ts +4 -2
- package/src/markdown.ts +136 -413
- package/src/selector-model.ts +6 -0
- package/src/selector.ts +277 -0
- package/src/trees.ts +43 -0
package/src/selector.ts
ADDED
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Learned candidate selector: page-level + candidate-level features → an
|
|
3
|
+
* estimated word-F1 per candidate (gradient-boosted regression trees trained
|
|
4
|
+
* offline by `benchmark/train-selector.py`); the pipeline returns the
|
|
5
|
+
* candidate with the highest estimate.
|
|
6
|
+
*
|
|
7
|
+
* Page features double as a page-type signal (generator meta, JSON-LD
|
|
8
|
+
* `@type`, `og:type`, URL shape, DOM statistics), so the trees learn
|
|
9
|
+
* type-conditional choices — "prune on articles, keep the link directory on
|
|
10
|
+
* docs", "unwrap noscript on forums only" — that no single global threshold
|
|
11
|
+
* could express.
|
|
12
|
+
*
|
|
13
|
+
* @module dsh-web-fetch-playwright/selector
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { CANDIDATE_KINDS, type CandidateKind, type GenerationFacts, NOISE_SUBTREE_TAGS } from './candidates.ts'
|
|
17
|
+
import { SELECTOR_MODEL } from './selector-model.ts'
|
|
18
|
+
import { blockKey, segmentBlocks, WORD } from './blocks.ts'
|
|
19
|
+
import { predictTrees, type TreeEnsemble } from './trees.ts'
|
|
20
|
+
|
|
21
|
+
/** Text statistics of one sanitized candidate. */
|
|
22
|
+
export interface CandidateStats {
|
|
23
|
+
kind: CandidateKind
|
|
24
|
+
legacy: boolean
|
|
25
|
+
words: number
|
|
26
|
+
linkWords: number
|
|
27
|
+
paragraphs: number
|
|
28
|
+
headings: number
|
|
29
|
+
listItems: number
|
|
30
|
+
tableCells: number
|
|
31
|
+
pres: number
|
|
32
|
+
images: number
|
|
33
|
+
blocks: number
|
|
34
|
+
tokens: Set<string>
|
|
35
|
+
consentHits: number
|
|
36
|
+
/** Text blocks as (key, words, linked words), for block-level diffs. */
|
|
37
|
+
blockList: { key: string, words: number, linkWords: number }[]
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const CONSENT_WORDS = /\b(cookies?|consent|datenschutz|privacy|einwilligung|akzeptieren|accept)\b/gi
|
|
41
|
+
|
|
42
|
+
/** Measure one sanitized candidate body. */
|
|
43
|
+
export function measureCandidate(kind: CandidateKind, legacy: boolean, root: Element): CandidateStats {
|
|
44
|
+
let words = 0
|
|
45
|
+
let linkWords = 0
|
|
46
|
+
const tokens = new Set<string>()
|
|
47
|
+
let head = ''
|
|
48
|
+
const visit = (node: Node, inAnchor: boolean): void => {
|
|
49
|
+
if (node.nodeType === 3) {
|
|
50
|
+
const text = node.nodeValue ?? ''
|
|
51
|
+
const matches = text.toLowerCase().match(WORD)
|
|
52
|
+
if (matches !== null) {
|
|
53
|
+
words += matches.length
|
|
54
|
+
if (inAnchor) linkWords += matches.length
|
|
55
|
+
if (tokens.size < 20_000) for (const token of matches) tokens.add(token)
|
|
56
|
+
}
|
|
57
|
+
if (head.length < 600) head += ` ${text}`
|
|
58
|
+
return
|
|
59
|
+
}
|
|
60
|
+
if (node.nodeType !== 1 || NOISE_SUBTREE_TAGS.has(node.nodeName.toLowerCase())) return
|
|
61
|
+
const anchor = inAnchor || node.nodeName === 'A'
|
|
62
|
+
for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child, anchor)
|
|
63
|
+
}
|
|
64
|
+
visit(root, false)
|
|
65
|
+
const count = (selector: string): number => root.querySelectorAll(selector).length
|
|
66
|
+
const paragraphs = count('p')
|
|
67
|
+
const listItems = count('li')
|
|
68
|
+
const headings = count('h1,h2,h3,h4,h5,h6')
|
|
69
|
+
const tableCells = count('td,th')
|
|
70
|
+
return {
|
|
71
|
+
kind,
|
|
72
|
+
legacy,
|
|
73
|
+
words,
|
|
74
|
+
linkWords,
|
|
75
|
+
paragraphs,
|
|
76
|
+
headings,
|
|
77
|
+
listItems,
|
|
78
|
+
tableCells,
|
|
79
|
+
pres: count('pre'),
|
|
80
|
+
images: count('img'),
|
|
81
|
+
blocks: paragraphs + listItems + headings + tableCells + count('div,section,article,blockquote,dd,dt'),
|
|
82
|
+
tokens,
|
|
83
|
+
consentHits: (head.match(CONSENT_WORDS) ?? []).length,
|
|
84
|
+
blockList: segmentBlocks(root).map(block => ({ key: blockKey(block.text), words: block.words, linkWords: block.linkWords })),
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Page-shape regexes over the URL. */
|
|
89
|
+
const URL_SHAPES: [string, RegExp][] = [
|
|
90
|
+
['product', /\/(products?|p|dp|item|itm|artikel|produkt)\/|[?&](pid|sku|product)=/i],
|
|
91
|
+
['collection', /\/(collections?|categor(y|ies)|shop|c|catalog|kategorie|brands?|department)\b/i],
|
|
92
|
+
['docs', /\/(docs?|documentation|api|reference|guides?|manual|tutorials?|learn|wiki)\b|^https?:\/\/(docs|developer|developers|wiki)\./i],
|
|
93
|
+
['forum', /\/(t|threads?|forums?|topics?|discussions?|questions|community|comments)\b|^https?:\/\/(community|forum|forums|discuss|answers)\./i],
|
|
94
|
+
['article', /\/(20\d\d|19\d\d)\/|\/(blog|news|articles?|story|stories|posts?|press)\b/i],
|
|
95
|
+
['search', /\/search\b|[?&](q|query|s|keyword)=/i],
|
|
96
|
+
]
|
|
97
|
+
|
|
98
|
+
const GENERATORS: [string, RegExp][] = [
|
|
99
|
+
['discourse', /discourse/i],
|
|
100
|
+
['wordpress', /wordpress/i],
|
|
101
|
+
['shop', /shopify|woocommerce|magento|prestashop|shopware/i],
|
|
102
|
+
['forumsw', /flarum|xenforo|vbulletin|phpbb|discuz|invision|mybb|nodebb/i],
|
|
103
|
+
['docsgen', /sphinx|docusaurus|mkdocs|hugo|gitbook|jekyll|vitepress|mediawiki|readthedocs/i],
|
|
104
|
+
['cms', /drupal|joomla|ghost|wix|squarespace|hubspot|webflow/i],
|
|
105
|
+
]
|
|
106
|
+
|
|
107
|
+
const JSONLD_TYPES: [string, RegExp][] = [
|
|
108
|
+
['product', /"@type"\s*:\s*"?(Product|Offer|AggregateOffer|ProductGroup)/],
|
|
109
|
+
['article', /"@type"\s*:\s*"?(Article|NewsArticle|BlogPosting|Report|ScholarlyArticle|TechArticle)/],
|
|
110
|
+
['forum', /"@type"\s*:\s*"?(DiscussionForumPosting|QAPage|Question|SocialMediaPosting|Comment)/],
|
|
111
|
+
['list', /"@type"\s*:\s*"?(ItemList|CollectionPage|SearchResultsPage|OfferCatalog)/],
|
|
112
|
+
['faq', /"@type"\s*:\s*"?(FAQPage|HowTo)/],
|
|
113
|
+
['org', /"@type"\s*:\s*"?(Service|LocalBusiness|Organization|Corporation|EducationalOrganization|Course|Event)/],
|
|
114
|
+
['breadcrumb', /"@type"\s*:\s*"?BreadcrumbList/],
|
|
115
|
+
]
|
|
116
|
+
|
|
117
|
+
const PRICE = /(?:[$€£¥₹]\s?\d|\d[\d.,]*\s?(?:€|EUR|USD|zł|kr|CHF))/g
|
|
118
|
+
|
|
119
|
+
/** Page-level feature names, in vector order. */
|
|
120
|
+
export const PAGE_FEATURES = [
|
|
121
|
+
'body_words_log', 'noscript_share', 'body_link_density', 'readability_null', 'gate_pass', 'prune_changed',
|
|
122
|
+
'has_main', 'has_preloaded', 'record_items_log', 'record_share',
|
|
123
|
+
...URL_SHAPES.map(([name]) => `url_${name}`), 'url_depth', 'url_root',
|
|
124
|
+
...GENERATORS.map(([name]) => `gen_${name}`),
|
|
125
|
+
...JSONLD_TYPES.map(([name]) => `ld_${name}`),
|
|
126
|
+
'og_article', 'og_product', 'og_website',
|
|
127
|
+
'n_article_log', 'n_h1', 'n_h2_log', 'n_pre_log', 'n_time_log', 'n_form_log', 'n_price_log',
|
|
128
|
+
'n_itemprop_price', 'n_product_itemtype', 'n_post_class_log', 'n_table_log', 'n_code_log', 'n_card_class_log',
|
|
129
|
+
] as const
|
|
130
|
+
|
|
131
|
+
/** Candidate-level feature names, in vector order (one-hot kinds appended). */
|
|
132
|
+
export const CANDIDATE_FEATURES = [
|
|
133
|
+
'words_log', 'words_vs_doc', 'words_vs_max', 'words_vs_art_log', 'link_density', 'paragraphs_log', 'headings_log',
|
|
134
|
+
'li_share', 'td_log', 'pre_log', 'img_log', 'words_per_block', 'precision_vs_art', 'recall_vs_art',
|
|
135
|
+
'precision_vs_doc', 'consent_hits', 'legacy',
|
|
136
|
+
'extra_share', 'extra_link_density', 'extra_long_log', 'extra_words_per_block', 'missing_share',
|
|
137
|
+
...CANDIDATE_KINDS.map(kind => `kind_${kind}`),
|
|
138
|
+
] as const
|
|
139
|
+
|
|
140
|
+
export const FEATURE_NAMES = [...PAGE_FEATURES, ...CANDIDATE_FEATURES]
|
|
141
|
+
|
|
142
|
+
const log1p = Math.log1p
|
|
143
|
+
|
|
144
|
+
/** Compute the page feature vector. */
|
|
145
|
+
export function pageFeatures(document: Document, url: string, facts: GenerationFacts, doc: CandidateStats | undefined, bodyText: number): number[] {
|
|
146
|
+
const bodyWords = doc?.words ?? 0
|
|
147
|
+
const count = (selector: string): number => document.querySelectorAll(selector).length
|
|
148
|
+
const generator = Array.from(document.querySelectorAll('meta[name="generator" i]'), meta => meta.getAttribute('content') ?? '').join(' ')
|
|
149
|
+
+ (document.querySelector('#data-preloaded, meta[name="discourse_theme_id"], .ember-application') !== null ? ' discourse' : '')
|
|
150
|
+
const jsonld = Array.from(document.querySelectorAll('script[type="application/ld+json"]'), script => script.textContent ?? '').join('\n')
|
|
151
|
+
const ogType = (document.querySelector('meta[property="og:type"]')?.getAttribute('content') ?? '').toLowerCase()
|
|
152
|
+
let path = ''
|
|
153
|
+
try { path = new URL(url).pathname } catch { /* keep empty */ }
|
|
154
|
+
const bodyTextContent = document.body?.textContent ?? ''
|
|
155
|
+
const prices = bodyTextContent.length > 2_000_000 ? 0 : (bodyTextContent.match(PRICE) ?? []).length
|
|
156
|
+
return [
|
|
157
|
+
log1p(bodyWords),
|
|
158
|
+
facts.noscriptText / Math.max(1, bodyText + facts.noscriptText),
|
|
159
|
+
doc === undefined || doc.words === 0 ? 0 : doc.linkWords / doc.words,
|
|
160
|
+
facts.readabilityNull ? 1 : 0,
|
|
161
|
+
facts.gatePass ? 1 : 0,
|
|
162
|
+
facts.pruneChanged ? 1 : 0,
|
|
163
|
+
facts.hasMain ? 1 : 0,
|
|
164
|
+
facts.hasPreloaded ? 1 : 0,
|
|
165
|
+
log1p(facts.record?.items.length ?? 0),
|
|
166
|
+
(facts.record?.text ?? 0) / Math.max(1, bodyText),
|
|
167
|
+
...URL_SHAPES.map(([, pattern]) => pattern.test(url) ? 1 : 0),
|
|
168
|
+
path.split('/').filter(Boolean).length,
|
|
169
|
+
path === '/' || path === '' ? 1 : 0,
|
|
170
|
+
...GENERATORS.map(([, pattern]) => pattern.test(generator) ? 1 : 0),
|
|
171
|
+
...JSONLD_TYPES.map(([, pattern]) => pattern.test(jsonld) ? 1 : 0),
|
|
172
|
+
ogType.includes('article') ? 1 : 0,
|
|
173
|
+
ogType.includes('product') ? 1 : 0,
|
|
174
|
+
ogType === 'website' ? 1 : 0,
|
|
175
|
+
log1p(count('article')),
|
|
176
|
+
count('h1'),
|
|
177
|
+
log1p(count('h2')),
|
|
178
|
+
log1p(count('pre')),
|
|
179
|
+
log1p(count('time')),
|
|
180
|
+
log1p(count('form')),
|
|
181
|
+
log1p(prices),
|
|
182
|
+
count('[itemprop="price"]') > 0 ? 1 : 0,
|
|
183
|
+
count('[itemtype*="Product" i]') > 0 ? 1 : 0,
|
|
184
|
+
log1p(count('[class*="post" i], [class*="comment" i], [class*="reply" i]')),
|
|
185
|
+
log1p(count('table')),
|
|
186
|
+
log1p(count('code')),
|
|
187
|
+
log1p(count('[class*="card" i], [class*="product" i], [class*="tile" i]')),
|
|
188
|
+
]
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function overlap(tokens: Set<string>, reference: Set<string> | undefined): number {
|
|
192
|
+
if (reference === undefined || tokens.size === 0) return 0
|
|
193
|
+
let hit = 0
|
|
194
|
+
for (const token of tokens) if (reference.has(token)) hit++
|
|
195
|
+
return hit / tokens.size
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* How a candidate differs from Readability's pick, block by block: the
|
|
200
|
+
* blocks it adds (long prose → content Readability missed; short link-dense
|
|
201
|
+
* rows → chrome) and the share of Readability's blocks it drops.
|
|
202
|
+
*/
|
|
203
|
+
function blockDiff(stats: CandidateStats, art: CandidateStats | undefined): number[] {
|
|
204
|
+
if (art === undefined) return [0, 0, 0, 0, 0]
|
|
205
|
+
const artKeys = new Set(art.blockList.map(block => block.key))
|
|
206
|
+
const ownKeys = new Set(stats.blockList.map(block => block.key))
|
|
207
|
+
let total = 0
|
|
208
|
+
let extraWords = 0
|
|
209
|
+
let extraLinks = 0
|
|
210
|
+
let extraLong = 0
|
|
211
|
+
let extraBlocks = 0
|
|
212
|
+
for (const block of stats.blockList) {
|
|
213
|
+
total += block.words
|
|
214
|
+
if (artKeys.has(block.key)) continue
|
|
215
|
+
extraWords += block.words
|
|
216
|
+
extraLinks += block.linkWords
|
|
217
|
+
extraBlocks++
|
|
218
|
+
if (block.words >= 30) extraLong++
|
|
219
|
+
}
|
|
220
|
+
let artTotal = 0
|
|
221
|
+
let missing = 0
|
|
222
|
+
for (const block of art.blockList) {
|
|
223
|
+
artTotal += block.words
|
|
224
|
+
if (!ownKeys.has(block.key)) missing += block.words
|
|
225
|
+
}
|
|
226
|
+
return [
|
|
227
|
+
extraWords / Math.max(1, total),
|
|
228
|
+
extraWords === 0 ? 0 : extraLinks / extraWords,
|
|
229
|
+
Math.log1p(extraLong),
|
|
230
|
+
extraBlocks === 0 ? 0 : extraWords / extraBlocks,
|
|
231
|
+
missing / Math.max(1, artTotal),
|
|
232
|
+
]
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** Compute one candidate's feature vector (page vector excluded). */
|
|
236
|
+
export function candidateFeatures(stats: CandidateStats, all: CandidateStats[]): number[] {
|
|
237
|
+
const doc = all.find(c => c.kind === 'doc')
|
|
238
|
+
const art = all.find(c => c.kind === 'art')
|
|
239
|
+
const maxWords = Math.max(1, ...all.map(c => c.words))
|
|
240
|
+
return [
|
|
241
|
+
log1p(stats.words),
|
|
242
|
+
stats.words / Math.max(1, doc?.words ?? 0),
|
|
243
|
+
stats.words / maxWords,
|
|
244
|
+
art === undefined ? 0 : Math.log((stats.words + 1) / (art.words + 1)),
|
|
245
|
+
stats.words === 0 ? 0 : stats.linkWords / stats.words,
|
|
246
|
+
log1p(stats.paragraphs),
|
|
247
|
+
log1p(stats.headings),
|
|
248
|
+
stats.listItems / Math.max(1, stats.blocks),
|
|
249
|
+
log1p(stats.tableCells),
|
|
250
|
+
log1p(stats.pres),
|
|
251
|
+
log1p(stats.images),
|
|
252
|
+
stats.words / Math.max(1, stats.blocks),
|
|
253
|
+
overlap(stats.tokens, art?.tokens),
|
|
254
|
+
art === undefined ? 0 : overlap(art.tokens, stats.tokens),
|
|
255
|
+
overlap(stats.tokens, doc?.tokens),
|
|
256
|
+
stats.consentHits,
|
|
257
|
+
stats.legacy ? 1 : 0,
|
|
258
|
+
...blockDiff(stats, art),
|
|
259
|
+
...CANDIDATE_KINDS.map(kind => kind === stats.kind ? 1 : 0),
|
|
260
|
+
]
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/** A trained gradient-boosted regressor. */
|
|
264
|
+
export interface SelectorModel extends TreeEnsemble {
|
|
265
|
+
version: string
|
|
266
|
+
features: readonly string[]
|
|
267
|
+
/** Estimate lead another candidate needs over the legacy pick to replace it. */
|
|
268
|
+
margin?: number
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/** Estimated quality of one candidate; falls back to the legacy choice without a model. */
|
|
272
|
+
export function scoreCandidate(vector: number[], model: SelectorModel = SELECTOR_MODEL): number {
|
|
273
|
+
if (model.trees.length === 0) {
|
|
274
|
+
return vector[FEATURE_NAMES.indexOf('legacy')] ?? 0
|
|
275
|
+
}
|
|
276
|
+
return predictTrees(model, vector) + (vector[FEATURE_NAMES.indexOf('legacy')] === 1 ? model.margin ?? 0 : 0)
|
|
277
|
+
}
|
package/src/trees.ts
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Flat gradient-boosted tree inference shared by the candidate selector and
|
|
3
|
+
* the block classifier (models exported from sklearn by the benchmark
|
|
4
|
+
* training scripts).
|
|
5
|
+
*
|
|
6
|
+
* @module dsh-web-fetch-playwright/trees
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/** One tree in flat array form (sklearn layout: leaf when left === -1). */
|
|
10
|
+
export interface FlatTree {
|
|
11
|
+
feature: number[]
|
|
12
|
+
threshold: number[]
|
|
13
|
+
left: number[]
|
|
14
|
+
right: number[]
|
|
15
|
+
value: number[]
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** Any additive tree ensemble: `init + learningRate * Σ tree(x)`. */
|
|
19
|
+
export interface TreeEnsemble {
|
|
20
|
+
init: number
|
|
21
|
+
learningRate: number
|
|
22
|
+
trees: FlatTree[]
|
|
23
|
+
/** Compare in float64 (HistGradientBoosting) instead of sklearn trees' float32. */
|
|
24
|
+
float64?: boolean
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function predictTree(tree: FlatTree, x: number[], float64: boolean): number {
|
|
28
|
+
let node = 0
|
|
29
|
+
while (tree.left[node] !== -1) {
|
|
30
|
+
const feature = tree.feature[node] ?? 0
|
|
31
|
+
// sklearn compares features as float32; mirror that so split boundaries agree.
|
|
32
|
+
const value = x[feature] ?? 0
|
|
33
|
+
node = (float64 ? value : Math.fround(value)) <= (tree.threshold[node] ?? 0) ? tree.left[node] ?? 0 : tree.right[node] ?? 0
|
|
34
|
+
}
|
|
35
|
+
return tree.value[node] ?? 0
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Raw ensemble output for one feature vector. */
|
|
39
|
+
export function predictTrees(model: TreeEnsemble, x: number[]): number {
|
|
40
|
+
let sum = model.init
|
|
41
|
+
for (const tree of model.trees) sum += model.learningRate * predictTree(tree, x, model.float64 === true)
|
|
42
|
+
return sum
|
|
43
|
+
}
|