dsh-web-fetch-playwright 0.2.19 → 0.2.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,288 @@
1
+ /**
2
+ * Learned candidate selector: page-level + candidate-level features → an
3
+ * estimated word-F1 per candidate (gradient-boosted regression trees trained
4
+ * offline by `benchmark/train-selector.py`); the pipeline returns the
5
+ * candidate with the highest estimate.
6
+ *
7
+ * Page features double as a page-type signal (generator meta, JSON-LD
8
+ * `@type`, `og:type`, URL shape, DOM statistics), so the trees learn
9
+ * type-conditional choices — "prune on articles, keep the link directory on
10
+ * docs", "unwrap noscript on forums only" — that no single global threshold
11
+ * could express.
12
+ *
13
+ * @module dsh-web-fetch-playwright/selector
14
+ */
15
+
16
+ import { CANDIDATE_KINDS, type CandidateKind, CONSENT_TERMS, type GenerationFacts, NOISE_SUBTREE_TAGS } from './candidates.ts'
17
+ import { SELECTOR_MODEL } from './selector-model.ts'
18
+ import { blockKey, segmentBlocks, WORD } from './blocks.ts'
19
+ import { predictTrees, type TreeEnsemble } from './trees.ts'
20
+
21
+ /** Text statistics of one sanitized candidate. */
22
+ export interface CandidateStats {
23
+ kind: CandidateKind
24
+ legacy: boolean
25
+ words: number
26
+ linkWords: number
27
+ paragraphs: number
28
+ headings: number
29
+ listItems: number
30
+ tableCells: number
31
+ pres: number
32
+ images: number
33
+ blocks: number
34
+ tokens: Set<string>
35
+ consentHits: number
36
+ /** Text blocks as (key, words, linked words), for block-level diffs. */
37
+ blockList: { key: string, words: number, linkWords: number }[]
38
+ }
39
+
40
+ const CONSENT_WORDS = new RegExp(CONSENT_TERMS, 'gi')
41
+
42
+ /** Measure one sanitized candidate body. */
43
+ export function measureCandidate(kind: CandidateKind, legacy: boolean, root: Element): CandidateStats {
44
+ let words = 0
45
+ let linkWords = 0
46
+ const tokens = new Set<string>()
47
+ let head = ''
48
+ const visit = (node: Node, inAnchor: boolean): void => {
49
+ if (node.nodeType === 3) {
50
+ const text = node.nodeValue ?? ''
51
+ const matches = text.toLowerCase().match(WORD)
52
+ if (matches !== null) {
53
+ words += matches.length
54
+ if (inAnchor) linkWords += matches.length
55
+ if (tokens.size < 20_000) for (const token of matches) tokens.add(token)
56
+ }
57
+ if (head.length < 600) head += ` ${text}`
58
+ return
59
+ }
60
+ if (node.nodeType !== 1 || NOISE_SUBTREE_TAGS.has(node.nodeName.toLowerCase())) return
61
+ const anchor = inAnchor || node.nodeName === 'A'
62
+ for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child, anchor)
63
+ }
64
+ visit(root, false)
65
+ const count = (selector: string): number => root.querySelectorAll(selector).length
66
+ const paragraphs = count('p')
67
+ const listItems = count('li')
68
+ const headings = count('h1,h2,h3,h4,h5,h6')
69
+ const tableCells = count('td,th')
70
+ return {
71
+ kind,
72
+ legacy,
73
+ words,
74
+ linkWords,
75
+ paragraphs,
76
+ headings,
77
+ listItems,
78
+ tableCells,
79
+ pres: count('pre'),
80
+ images: count('img'),
81
+ blocks: paragraphs + listItems + headings + tableCells + count('div,section,article,blockquote,dd,dt'),
82
+ tokens,
83
+ consentHits: (head.match(CONSENT_WORDS) ?? []).length,
84
+ blockList: segmentBlocks(root).map(block => ({ key: blockKey(block.text), words: block.words, linkWords: block.linkWords })),
85
+ }
86
+ }
87
+
88
+ /**
89
+ * Page-shape regexes over the URL: whole words a site of ANY platform would
90
+ * choose for that page type. Single-letter and store-specific path codes
91
+ * (`/p/`, `/c/`, `/t/`, `/dp/`, `/itm/`) are deliberately absent — each is
92
+ * one platform's convention that collides with others (`/p/` is a product
93
+ * on one shop and every post on Substack or Jianshu), so as features they
94
+ * only taught the model which sites were in the training set.
95
+ */
96
+ const URL_SHAPES: [string, RegExp][] = [
97
+ ['product', /\/(products?|item|artikel|produkt)\/|[?&](pid|sku|product)=/i],
98
+ ['collection', /\/(collections?|categor(y|ies)|shop|catalog|kategorie|brands?|department)\b/i],
99
+ ['docs', /\/(docs?|documentation|api|reference|guides?|manual|tutorials?|learn|wiki)\b|^https?:\/\/(docs|developer|developers|wiki)\./i],
100
+ ['forum', /\/(threads?|forums?|topics?|discussions?|questions|community|comments)\b|^https?:\/\/(community|forum|forums|discuss|answers)\./i],
101
+ ['article', /\/(20\d\d|19\d\d)\/|\/(blog|news|articles?|story|stories|posts?|press)\b/i],
102
+ ['search', /\/search\b|[?&](q|query|s|keyword)=/i],
103
+ ]
104
+
105
+ const GENERATORS: [string, RegExp][] = [
106
+ ['discourse', /discourse/i],
107
+ ['wordpress', /wordpress/i],
108
+ ['shop', /shopify|woocommerce|magento|prestashop|shopware/i],
109
+ ['forumsw', /flarum|xenforo|vbulletin|phpbb|discuz|invision|mybb|nodebb/i],
110
+ ['docsgen', /sphinx|docusaurus|mkdocs|hugo|gitbook|jekyll|vitepress|mediawiki|readthedocs/i],
111
+ ['cms', /drupal|joomla|ghost|wix|squarespace|hubspot|webflow/i],
112
+ ]
113
+
114
+ const JSONLD_TYPES: [string, RegExp][] = [
115
+ ['product', /"@type"\s*:\s*"?(Product|Offer|AggregateOffer|ProductGroup)/],
116
+ ['article', /"@type"\s*:\s*"?(Article|NewsArticle|BlogPosting|Report|ScholarlyArticle|TechArticle)/],
117
+ ['forum', /"@type"\s*:\s*"?(DiscussionForumPosting|QAPage|Question|SocialMediaPosting|Comment)/],
118
+ ['list', /"@type"\s*:\s*"?(ItemList|CollectionPage|SearchResultsPage|OfferCatalog)/],
119
+ ['faq', /"@type"\s*:\s*"?(FAQPage|HowTo)/],
120
+ ['org', /"@type"\s*:\s*"?(Service|LocalBusiness|Organization|Corporation|EducationalOrganization|Course|Event)/],
121
+ ['breadcrumb', /"@type"\s*:\s*"?BreadcrumbList/],
122
+ ]
123
+
124
+ const PRICE = /(?:[$€£¥₹]\s?\d|\d[\d.,]*\s?(?:€|EUR|USD|zł|kr|CHF))/g
125
+
126
+ /** Page-level feature names, in vector order. */
127
+ export const PAGE_FEATURES = [
128
+ 'body_words_log', 'noscript_share', 'body_link_density', 'readability_null', 'gate_pass', 'prune_changed',
129
+ 'has_main', 'has_preloaded', 'record_items_log', 'record_share',
130
+ ...URL_SHAPES.map(([name]) => `url_${name}`), 'url_depth', 'url_root',
131
+ ...GENERATORS.map(([name]) => `gen_${name}`),
132
+ ...JSONLD_TYPES.map(([name]) => `ld_${name}`),
133
+ 'og_article', 'og_product', 'og_website',
134
+ 'n_article_log', 'n_h1', 'n_h2_log', 'n_pre_log', 'n_time_log', 'n_form_log', 'n_price_log',
135
+ 'n_itemprop_price', 'n_product_itemtype', 'n_post_class_log', 'n_table_log', 'n_code_log', 'n_card_class_log',
136
+ ] as const
137
+
138
+ /** Candidate-level feature names, in vector order (one-hot kinds appended). */
139
+ export const CANDIDATE_FEATURES = [
140
+ 'words_log', 'words_vs_doc', 'words_vs_max', 'words_vs_art_log', 'link_density', 'paragraphs_log', 'headings_log',
141
+ 'li_share', 'td_log', 'pre_log', 'img_log', 'words_per_block', 'precision_vs_art', 'recall_vs_art',
142
+ 'precision_vs_doc', 'consent_hits', 'legacy',
143
+ 'extra_share', 'extra_link_density', 'extra_long_log', 'extra_words_per_block', 'missing_share',
144
+ ...CANDIDATE_KINDS.map(kind => `kind_${kind}`),
145
+ ] as const
146
+
147
+ export const FEATURE_NAMES = [...PAGE_FEATURES, ...CANDIDATE_FEATURES]
148
+
149
+ const log1p = Math.log1p
150
+
151
+ /** Compute the page feature vector. */
152
+ export function pageFeatures(document: Document, url: string, facts: GenerationFacts, doc: CandidateStats | undefined, bodyText: number): number[] {
153
+ const bodyWords = doc?.words ?? 0
154
+ const count = (selector: string): number => document.querySelectorAll(selector).length
155
+ // The generator meta, plus what Discourse itself declares for installs
156
+ // that drop it (its data island, its theme meta). Framework markers
157
+ // (`.ember-application`) are not generator evidence: every Ember app
158
+ // carries them.
159
+ const generator = Array.from(document.querySelectorAll('meta[name="generator" i]'), meta => meta.getAttribute('content') ?? '').join(' ')
160
+ + (document.querySelector('#data-preloaded[data-preloaded], meta[name="discourse_theme_id"]') !== null ? ' discourse' : '')
161
+ const jsonld = Array.from(document.querySelectorAll('script[type="application/ld+json"]'), script => script.textContent ?? '').join('\n')
162
+ const ogType = (document.querySelector('meta[property="og:type"]')?.getAttribute('content') ?? '').toLowerCase()
163
+ let path = ''
164
+ try { path = new URL(url).pathname } catch { /* keep empty */ }
165
+ const bodyTextContent = document.body?.textContent ?? ''
166
+ const prices = bodyTextContent.length > 2_000_000 ? 0 : (bodyTextContent.match(PRICE) ?? []).length
167
+ return [
168
+ log1p(bodyWords),
169
+ facts.noscriptText / Math.max(1, bodyText + facts.noscriptText),
170
+ doc === undefined || doc.words === 0 ? 0 : doc.linkWords / doc.words,
171
+ facts.readabilityNull ? 1 : 0,
172
+ facts.gatePass ? 1 : 0,
173
+ facts.pruneChanged ? 1 : 0,
174
+ facts.hasMain ? 1 : 0,
175
+ facts.hasPreloaded ? 1 : 0,
176
+ log1p(facts.record?.items.length ?? 0),
177
+ (facts.record?.text ?? 0) / Math.max(1, bodyText),
178
+ ...URL_SHAPES.map(([, pattern]) => pattern.test(url) ? 1 : 0),
179
+ path.split('/').filter(Boolean).length,
180
+ path === '/' || path === '' ? 1 : 0,
181
+ ...GENERATORS.map(([, pattern]) => pattern.test(generator) ? 1 : 0),
182
+ ...JSONLD_TYPES.map(([, pattern]) => pattern.test(jsonld) ? 1 : 0),
183
+ ogType.includes('article') ? 1 : 0,
184
+ ogType.includes('product') ? 1 : 0,
185
+ ogType === 'website' ? 1 : 0,
186
+ log1p(count('article')),
187
+ count('h1'),
188
+ log1p(count('h2')),
189
+ log1p(count('pre')),
190
+ log1p(count('time')),
191
+ log1p(count('form')),
192
+ log1p(prices),
193
+ count('[itemprop="price"]') > 0 ? 1 : 0,
194
+ count('[itemtype*="Product" i]') > 0 ? 1 : 0,
195
+ log1p(count('[class*="post" i], [class*="comment" i], [class*="reply" i]')),
196
+ log1p(count('table')),
197
+ log1p(count('code')),
198
+ log1p(count('[class*="card" i], [class*="product" i], [class*="tile" i]')),
199
+ ]
200
+ }
201
+
202
+ function overlap(tokens: Set<string>, reference: Set<string> | undefined): number {
203
+ if (reference === undefined || tokens.size === 0) return 0
204
+ let hit = 0
205
+ for (const token of tokens) if (reference.has(token)) hit++
206
+ return hit / tokens.size
207
+ }
208
+
209
+ /**
210
+ * How a candidate differs from Readability's pick, block by block: the
211
+ * blocks it adds (long prose → content Readability missed; short link-dense
212
+ * rows → chrome) and the share of Readability's blocks it drops.
213
+ */
214
+ function blockDiff(stats: CandidateStats, art: CandidateStats | undefined): number[] {
215
+ if (art === undefined) return [0, 0, 0, 0, 0]
216
+ const artKeys = new Set(art.blockList.map(block => block.key))
217
+ const ownKeys = new Set(stats.blockList.map(block => block.key))
218
+ let total = 0
219
+ let extraWords = 0
220
+ let extraLinks = 0
221
+ let extraLong = 0
222
+ let extraBlocks = 0
223
+ for (const block of stats.blockList) {
224
+ total += block.words
225
+ if (artKeys.has(block.key)) continue
226
+ extraWords += block.words
227
+ extraLinks += block.linkWords
228
+ extraBlocks++
229
+ if (block.words >= 30) extraLong++
230
+ }
231
+ let artTotal = 0
232
+ let missing = 0
233
+ for (const block of art.blockList) {
234
+ artTotal += block.words
235
+ if (!ownKeys.has(block.key)) missing += block.words
236
+ }
237
+ return [
238
+ extraWords / Math.max(1, total),
239
+ extraWords === 0 ? 0 : extraLinks / extraWords,
240
+ Math.log1p(extraLong),
241
+ extraBlocks === 0 ? 0 : extraWords / extraBlocks,
242
+ missing / Math.max(1, artTotal),
243
+ ]
244
+ }
245
+
246
+ /** Compute one candidate's feature vector (page vector excluded). */
247
+ export function candidateFeatures(stats: CandidateStats, all: CandidateStats[]): number[] {
248
+ const doc = all.find(c => c.kind === 'doc')
249
+ const art = all.find(c => c.kind === 'art')
250
+ const maxWords = Math.max(1, ...all.map(c => c.words))
251
+ return [
252
+ log1p(stats.words),
253
+ stats.words / Math.max(1, doc?.words ?? 0),
254
+ stats.words / maxWords,
255
+ art === undefined ? 0 : Math.log((stats.words + 1) / (art.words + 1)),
256
+ stats.words === 0 ? 0 : stats.linkWords / stats.words,
257
+ log1p(stats.paragraphs),
258
+ log1p(stats.headings),
259
+ stats.listItems / Math.max(1, stats.blocks),
260
+ log1p(stats.tableCells),
261
+ log1p(stats.pres),
262
+ log1p(stats.images),
263
+ stats.words / Math.max(1, stats.blocks),
264
+ overlap(stats.tokens, art?.tokens),
265
+ art === undefined ? 0 : overlap(art.tokens, stats.tokens),
266
+ overlap(stats.tokens, doc?.tokens),
267
+ stats.consentHits,
268
+ stats.legacy ? 1 : 0,
269
+ ...blockDiff(stats, art),
270
+ ...CANDIDATE_KINDS.map(kind => kind === stats.kind ? 1 : 0),
271
+ ]
272
+ }
273
+
274
+ /** A trained gradient-boosted regressor. */
275
+ export interface SelectorModel extends TreeEnsemble {
276
+ version: string
277
+ features: readonly string[]
278
+ /** Estimate lead another candidate needs over the legacy pick to replace it. */
279
+ margin?: number
280
+ }
281
+
282
+ /** Estimated quality of one candidate; falls back to the legacy choice without a model. */
283
+ export function scoreCandidate(vector: number[], model: SelectorModel = SELECTOR_MODEL): number {
284
+ if (model.trees.length === 0) {
285
+ return vector[FEATURE_NAMES.indexOf('legacy')] ?? 0
286
+ }
287
+ return predictTrees(model, vector) + (vector[FEATURE_NAMES.indexOf('legacy')] === 1 ? model.margin ?? 0 : 0)
288
+ }
package/src/trees.ts ADDED
@@ -0,0 +1,43 @@
1
+ /**
2
+ * Flat gradient-boosted tree inference shared by the candidate selector and
3
+ * the block classifier (models exported from sklearn by the benchmark
4
+ * training scripts).
5
+ *
6
+ * @module dsh-web-fetch-playwright/trees
7
+ */
8
+
9
+ /** One tree in flat array form (sklearn layout: leaf when left === -1). */
10
+ export interface FlatTree {
11
+ feature: number[]
12
+ threshold: number[]
13
+ left: number[]
14
+ right: number[]
15
+ value: number[]
16
+ }
17
+
18
+ /** Any additive tree ensemble: `init + learningRate * Σ tree(x)`. */
19
+ export interface TreeEnsemble {
20
+ init: number
21
+ learningRate: number
22
+ trees: FlatTree[]
23
+ /** Compare in float64 (HistGradientBoosting) instead of sklearn trees' float32. */
24
+ float64?: boolean
25
+ }
26
+
27
+ function predictTree(tree: FlatTree, x: number[], float64: boolean): number {
28
+ let node = 0
29
+ while (tree.left[node] !== -1) {
30
+ const feature = tree.feature[node] ?? 0
31
+ // sklearn compares features as float32; mirror that so split boundaries agree.
32
+ const value = x[feature] ?? 0
33
+ node = (float64 ? value : Math.fround(value)) <= (tree.threshold[node] ?? 0) ? tree.left[node] ?? 0 : tree.right[node] ?? 0
34
+ }
35
+ return tree.value[node] ?? 0
36
+ }
37
+
38
+ /** Raw ensemble output for one feature vector. */
39
+ export function predictTrees(model: TreeEnsemble, x: number[]): number {
40
+ let sum = model.init
41
+ for (const tree of model.trees) sum += model.learningRate * predictTree(tree, x, model.float64 === true)
42
+ return sum
43
+ }
package/src/types.ts CHANGED
@@ -36,15 +36,15 @@ export interface PlaywrightDownload {
36
36
  }
37
37
 
38
38
  /**
39
- * The page's `APIRequestContext` surface the download recovery refetches
40
- * through — plain HTTP from the Playwright client, the context's cookie jar,
41
- * and none of Chromium's render-or-download semantics. Reuses
42
- * {@link PlaywrightResponse} because that is structurally what an
43
- * `APIResponse` is for the members this plugin reads. Absent on fakes that
44
- * do not exercise the recovery.
39
+ * The page's `APIRequestContext` surface the download recovery and the
40
+ * challenge dead-end fallback refetch through — plain HTTP from the
41
+ * Playwright client, the context's cookie jar, and none of Chromium's
42
+ * render-or-download semantics. Reuses {@link PlaywrightResponse} because
43
+ * that is structurally what an `APIResponse` is for the members this
44
+ * plugin reads. Absent on fakes that do not exercise the recovery.
45
45
  */
46
46
  export interface PlaywrightRequestContext {
47
- get(url: string, options?: { timeout?: number; signal?: AbortSignal }): Promise<PlaywrightResponse>
47
+ get(url: string, options?: { timeout?: number; signal?: AbortSignal; headers?: Record<string, string> }): Promise<PlaywrightResponse>
48
48
  }
49
49
 
50
50
  /** The request side of a response, for main-frame filtering. */