dsh-web-fetch-playwright 0.2.18 → 0.2.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,277 @@
1
+ /**
2
+ * Learned candidate selector: page-level + candidate-level features → an
3
+ * estimated word-F1 per candidate (gradient-boosted regression trees trained
4
+ * offline by `benchmark/train-selector.py`); the pipeline returns the
5
+ * candidate with the highest estimate.
6
+ *
7
+ * Page features double as a page-type signal (generator meta, JSON-LD
8
+ * `@type`, `og:type`, URL shape, DOM statistics), so the trees learn
9
+ * type-conditional choices — "prune on articles, keep the link directory on
10
+ * docs", "unwrap noscript on forums only" — that no single global threshold
11
+ * could express.
12
+ *
13
+ * @module dsh-web-fetch-playwright/selector
14
+ */
15
+
16
+ import { CANDIDATE_KINDS, type CandidateKind, type GenerationFacts, NOISE_SUBTREE_TAGS } from './candidates.ts'
17
+ import { SELECTOR_MODEL } from './selector-model.ts'
18
+ import { blockKey, segmentBlocks, WORD } from './blocks.ts'
19
+ import { predictTrees, type TreeEnsemble } from './trees.ts'
20
+
21
+ /** Text statistics of one sanitized candidate. */
22
+ export interface CandidateStats {
23
+ kind: CandidateKind
24
+ legacy: boolean
25
+ words: number
26
+ linkWords: number
27
+ paragraphs: number
28
+ headings: number
29
+ listItems: number
30
+ tableCells: number
31
+ pres: number
32
+ images: number
33
+ blocks: number
34
+ tokens: Set<string>
35
+ consentHits: number
36
+ /** Text blocks as (key, words, linked words), for block-level diffs. */
37
+ blockList: { key: string, words: number, linkWords: number }[]
38
+ }
39
+
40
+ const CONSENT_WORDS = /\b(cookies?|consent|datenschutz|privacy|einwilligung|akzeptieren|accept)\b/gi
41
+
42
+ /** Measure one sanitized candidate body. */
43
+ export function measureCandidate(kind: CandidateKind, legacy: boolean, root: Element): CandidateStats {
44
+ let words = 0
45
+ let linkWords = 0
46
+ const tokens = new Set<string>()
47
+ let head = ''
48
+ const visit = (node: Node, inAnchor: boolean): void => {
49
+ if (node.nodeType === 3) {
50
+ const text = node.nodeValue ?? ''
51
+ const matches = text.toLowerCase().match(WORD)
52
+ if (matches !== null) {
53
+ words += matches.length
54
+ if (inAnchor) linkWords += matches.length
55
+ if (tokens.size < 20_000) for (const token of matches) tokens.add(token)
56
+ }
57
+ if (head.length < 600) head += ` ${text}`
58
+ return
59
+ }
60
+ if (node.nodeType !== 1 || NOISE_SUBTREE_TAGS.has(node.nodeName.toLowerCase())) return
61
+ const anchor = inAnchor || node.nodeName === 'A'
62
+ for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child, anchor)
63
+ }
64
+ visit(root, false)
65
+ const count = (selector: string): number => root.querySelectorAll(selector).length
66
+ const paragraphs = count('p')
67
+ const listItems = count('li')
68
+ const headings = count('h1,h2,h3,h4,h5,h6')
69
+ const tableCells = count('td,th')
70
+ return {
71
+ kind,
72
+ legacy,
73
+ words,
74
+ linkWords,
75
+ paragraphs,
76
+ headings,
77
+ listItems,
78
+ tableCells,
79
+ pres: count('pre'),
80
+ images: count('img'),
81
+ blocks: paragraphs + listItems + headings + tableCells + count('div,section,article,blockquote,dd,dt'),
82
+ tokens,
83
+ consentHits: (head.match(CONSENT_WORDS) ?? []).length,
84
+ blockList: segmentBlocks(root).map(block => ({ key: blockKey(block.text), words: block.words, linkWords: block.linkWords })),
85
+ }
86
+ }
87
+
88
+ /** Page-shape regexes over the URL. */
89
+ const URL_SHAPES: [string, RegExp][] = [
90
+ ['product', /\/(products?|p|dp|item|itm|artikel|produkt)\/|[?&](pid|sku|product)=/i],
91
+ ['collection', /\/(collections?|categor(y|ies)|shop|c|catalog|kategorie|brands?|department)\b/i],
92
+ ['docs', /\/(docs?|documentation|api|reference|guides?|manual|tutorials?|learn|wiki)\b|^https?:\/\/(docs|developer|developers|wiki)\./i],
93
+ ['forum', /\/(t|threads?|forums?|topics?|discussions?|questions|community|comments)\b|^https?:\/\/(community|forum|forums|discuss|answers)\./i],
94
+ ['article', /\/(20\d\d|19\d\d)\/|\/(blog|news|articles?|story|stories|posts?|press)\b/i],
95
+ ['search', /\/search\b|[?&](q|query|s|keyword)=/i],
96
+ ]
97
+
98
+ const GENERATORS: [string, RegExp][] = [
99
+ ['discourse', /discourse/i],
100
+ ['wordpress', /wordpress/i],
101
+ ['shop', /shopify|woocommerce|magento|prestashop|shopware/i],
102
+ ['forumsw', /flarum|xenforo|vbulletin|phpbb|discuz|invision|mybb|nodebb/i],
103
+ ['docsgen', /sphinx|docusaurus|mkdocs|hugo|gitbook|jekyll|vitepress|mediawiki|readthedocs/i],
104
+ ['cms', /drupal|joomla|ghost|wix|squarespace|hubspot|webflow/i],
105
+ ]
106
+
107
+ const JSONLD_TYPES: [string, RegExp][] = [
108
+ ['product', /"@type"\s*:\s*"?(Product|Offer|AggregateOffer|ProductGroup)/],
109
+ ['article', /"@type"\s*:\s*"?(Article|NewsArticle|BlogPosting|Report|ScholarlyArticle|TechArticle)/],
110
+ ['forum', /"@type"\s*:\s*"?(DiscussionForumPosting|QAPage|Question|SocialMediaPosting|Comment)/],
111
+ ['list', /"@type"\s*:\s*"?(ItemList|CollectionPage|SearchResultsPage|OfferCatalog)/],
112
+ ['faq', /"@type"\s*:\s*"?(FAQPage|HowTo)/],
113
+ ['org', /"@type"\s*:\s*"?(Service|LocalBusiness|Organization|Corporation|EducationalOrganization|Course|Event)/],
114
+ ['breadcrumb', /"@type"\s*:\s*"?BreadcrumbList/],
115
+ ]
116
+
117
+ const PRICE = /(?:[$€£¥₹]\s?\d|\d[\d.,]*\s?(?:€|EUR|USD|zł|kr|CHF))/g
118
+
119
+ /** Page-level feature names, in vector order. */
120
+ export const PAGE_FEATURES = [
121
+ 'body_words_log', 'noscript_share', 'body_link_density', 'readability_null', 'gate_pass', 'prune_changed',
122
+ 'has_main', 'has_preloaded', 'record_items_log', 'record_share',
123
+ ...URL_SHAPES.map(([name]) => `url_${name}`), 'url_depth', 'url_root',
124
+ ...GENERATORS.map(([name]) => `gen_${name}`),
125
+ ...JSONLD_TYPES.map(([name]) => `ld_${name}`),
126
+ 'og_article', 'og_product', 'og_website',
127
+ 'n_article_log', 'n_h1', 'n_h2_log', 'n_pre_log', 'n_time_log', 'n_form_log', 'n_price_log',
128
+ 'n_itemprop_price', 'n_product_itemtype', 'n_post_class_log', 'n_table_log', 'n_code_log', 'n_card_class_log',
129
+ ] as const
130
+
131
+ /** Candidate-level feature names, in vector order (one-hot kinds appended). */
132
+ export const CANDIDATE_FEATURES = [
133
+ 'words_log', 'words_vs_doc', 'words_vs_max', 'words_vs_art_log', 'link_density', 'paragraphs_log', 'headings_log',
134
+ 'li_share', 'td_log', 'pre_log', 'img_log', 'words_per_block', 'precision_vs_art', 'recall_vs_art',
135
+ 'precision_vs_doc', 'consent_hits', 'legacy',
136
+ 'extra_share', 'extra_link_density', 'extra_long_log', 'extra_words_per_block', 'missing_share',
137
+ ...CANDIDATE_KINDS.map(kind => `kind_${kind}`),
138
+ ] as const
139
+
140
+ export const FEATURE_NAMES = [...PAGE_FEATURES, ...CANDIDATE_FEATURES]
141
+
142
+ const log1p = Math.log1p
143
+
144
+ /** Compute the page feature vector. */
145
+ export function pageFeatures(document: Document, url: string, facts: GenerationFacts, doc: CandidateStats | undefined, bodyText: number): number[] {
146
+ const bodyWords = doc?.words ?? 0
147
+ const count = (selector: string): number => document.querySelectorAll(selector).length
148
+ const generator = Array.from(document.querySelectorAll('meta[name="generator" i]'), meta => meta.getAttribute('content') ?? '').join(' ')
149
+ + (document.querySelector('#data-preloaded, meta[name="discourse_theme_id"], .ember-application') !== null ? ' discourse' : '')
150
+ const jsonld = Array.from(document.querySelectorAll('script[type="application/ld+json"]'), script => script.textContent ?? '').join('\n')
151
+ const ogType = (document.querySelector('meta[property="og:type"]')?.getAttribute('content') ?? '').toLowerCase()
152
+ let path = ''
153
+ try { path = new URL(url).pathname } catch { /* keep empty */ }
154
+ const bodyTextContent = document.body?.textContent ?? ''
155
+ const prices = bodyTextContent.length > 2_000_000 ? 0 : (bodyTextContent.match(PRICE) ?? []).length
156
+ return [
157
+ log1p(bodyWords),
158
+ facts.noscriptText / Math.max(1, bodyText + facts.noscriptText),
159
+ doc === undefined || doc.words === 0 ? 0 : doc.linkWords / doc.words,
160
+ facts.readabilityNull ? 1 : 0,
161
+ facts.gatePass ? 1 : 0,
162
+ facts.pruneChanged ? 1 : 0,
163
+ facts.hasMain ? 1 : 0,
164
+ facts.hasPreloaded ? 1 : 0,
165
+ log1p(facts.record?.items.length ?? 0),
166
+ (facts.record?.text ?? 0) / Math.max(1, bodyText),
167
+ ...URL_SHAPES.map(([, pattern]) => pattern.test(url) ? 1 : 0),
168
+ path.split('/').filter(Boolean).length,
169
+ path === '/' || path === '' ? 1 : 0,
170
+ ...GENERATORS.map(([, pattern]) => pattern.test(generator) ? 1 : 0),
171
+ ...JSONLD_TYPES.map(([, pattern]) => pattern.test(jsonld) ? 1 : 0),
172
+ ogType.includes('article') ? 1 : 0,
173
+ ogType.includes('product') ? 1 : 0,
174
+ ogType === 'website' ? 1 : 0,
175
+ log1p(count('article')),
176
+ count('h1'),
177
+ log1p(count('h2')),
178
+ log1p(count('pre')),
179
+ log1p(count('time')),
180
+ log1p(count('form')),
181
+ log1p(prices),
182
+ count('[itemprop="price"]') > 0 ? 1 : 0,
183
+ count('[itemtype*="Product" i]') > 0 ? 1 : 0,
184
+ log1p(count('[class*="post" i], [class*="comment" i], [class*="reply" i]')),
185
+ log1p(count('table')),
186
+ log1p(count('code')),
187
+ log1p(count('[class*="card" i], [class*="product" i], [class*="tile" i]')),
188
+ ]
189
+ }
190
+
191
+ function overlap(tokens: Set<string>, reference: Set<string> | undefined): number {
192
+ if (reference === undefined || tokens.size === 0) return 0
193
+ let hit = 0
194
+ for (const token of tokens) if (reference.has(token)) hit++
195
+ return hit / tokens.size
196
+ }
197
+
198
+ /**
199
+ * How a candidate differs from Readability's pick, block by block: the
200
+ * blocks it adds (long prose → content Readability missed; short link-dense
201
+ * rows → chrome) and the share of Readability's blocks it drops.
202
+ */
203
+ function blockDiff(stats: CandidateStats, art: CandidateStats | undefined): number[] {
204
+ if (art === undefined) return [0, 0, 0, 0, 0]
205
+ const artKeys = new Set(art.blockList.map(block => block.key))
206
+ const ownKeys = new Set(stats.blockList.map(block => block.key))
207
+ let total = 0
208
+ let extraWords = 0
209
+ let extraLinks = 0
210
+ let extraLong = 0
211
+ let extraBlocks = 0
212
+ for (const block of stats.blockList) {
213
+ total += block.words
214
+ if (artKeys.has(block.key)) continue
215
+ extraWords += block.words
216
+ extraLinks += block.linkWords
217
+ extraBlocks++
218
+ if (block.words >= 30) extraLong++
219
+ }
220
+ let artTotal = 0
221
+ let missing = 0
222
+ for (const block of art.blockList) {
223
+ artTotal += block.words
224
+ if (!ownKeys.has(block.key)) missing += block.words
225
+ }
226
+ return [
227
+ extraWords / Math.max(1, total),
228
+ extraWords === 0 ? 0 : extraLinks / extraWords,
229
+ Math.log1p(extraLong),
230
+ extraBlocks === 0 ? 0 : extraWords / extraBlocks,
231
+ missing / Math.max(1, artTotal),
232
+ ]
233
+ }
234
+
235
+ /** Compute one candidate's feature vector (page vector excluded). */
236
+ export function candidateFeatures(stats: CandidateStats, all: CandidateStats[]): number[] {
237
+ const doc = all.find(c => c.kind === 'doc')
238
+ const art = all.find(c => c.kind === 'art')
239
+ const maxWords = Math.max(1, ...all.map(c => c.words))
240
+ return [
241
+ log1p(stats.words),
242
+ stats.words / Math.max(1, doc?.words ?? 0),
243
+ stats.words / maxWords,
244
+ art === undefined ? 0 : Math.log((stats.words + 1) / (art.words + 1)),
245
+ stats.words === 0 ? 0 : stats.linkWords / stats.words,
246
+ log1p(stats.paragraphs),
247
+ log1p(stats.headings),
248
+ stats.listItems / Math.max(1, stats.blocks),
249
+ log1p(stats.tableCells),
250
+ log1p(stats.pres),
251
+ log1p(stats.images),
252
+ stats.words / Math.max(1, stats.blocks),
253
+ overlap(stats.tokens, art?.tokens),
254
+ art === undefined ? 0 : overlap(art.tokens, stats.tokens),
255
+ overlap(stats.tokens, doc?.tokens),
256
+ stats.consentHits,
257
+ stats.legacy ? 1 : 0,
258
+ ...blockDiff(stats, art),
259
+ ...CANDIDATE_KINDS.map(kind => kind === stats.kind ? 1 : 0),
260
+ ]
261
+ }
262
+
263
+ /** A trained gradient-boosted regressor. */
264
+ export interface SelectorModel extends TreeEnsemble {
265
+ version: string
266
+ features: readonly string[]
267
+ /** Estimate lead another candidate needs over the legacy pick to replace it. */
268
+ margin?: number
269
+ }
270
+
271
+ /** Estimated quality of one candidate; falls back to the legacy choice without a model. */
272
+ export function scoreCandidate(vector: number[], model: SelectorModel = SELECTOR_MODEL): number {
273
+ if (model.trees.length === 0) {
274
+ return vector[FEATURE_NAMES.indexOf('legacy')] ?? 0
275
+ }
276
+ return predictTrees(model, vector) + (vector[FEATURE_NAMES.indexOf('legacy')] === 1 ? model.margin ?? 0 : 0)
277
+ }
@@ -18,6 +18,10 @@ interface TargetIdentity { targetId: string; browserContextId: string }
18
18
  const ownership = new WeakMap<PlaywrightBrowser, Map<string, WeakRef<PlaywrightContext>>>()
19
19
  // Never adopt an unconfirmed late tab after timing out, nor close sibling leases.
20
20
  const unavailable = new WeakSet<PlaywrightContext>()
21
+ // Identities confirmed when each session tab was created. Re-identifying a busy
22
+ // sibling would attach to it, and detaching from a tab mid-navigation can stall
23
+ // until its response arrives; a target id and its context never change.
24
+ const identities = new WeakMap<PlaywrightPage, TargetIdentity>()
21
25
  const CLEANUP_GRACE_MS = 100
22
26
 
23
27
  function claim(browser: PlaywrightBrowser, context: PlaywrightContext, keys: string[]): void {
@@ -112,6 +116,8 @@ export async function createSessionPage(
112
116
  return session
113
117
  }
114
118
  const identify = async (page: PlaywrightPage): Promise<TargetIdentity> => {
119
+ const known = identities.get(page)
120
+ if (known) return known
115
121
  const session = await newSession(() => ctx.newCDPSession(page))
116
122
  let identificationFailed = false
117
123
  try {
@@ -164,6 +170,7 @@ export async function createSessionPage(
164
170
  throw new Error('Chromium did not place the session tab in its dedicated window')
165
171
  }
166
172
  claim(browser, context, [`context:${actual.browserContextId}`, `window:${expectedWindow}`])
173
+ identities.set(created, actual)
167
174
  succeeded = true
168
175
  return created
169
176
  }
@@ -209,6 +216,7 @@ export async function createSessionPage(
209
216
  const explicitWindow = await windowOf(explicitTargetId)
210
217
  if (explicitWindow === bootstrapWindow) throw new Error('Chromium ignored the dedicated session window request')
211
218
  claim(browser, context, [`context:${identity.browserContextId}`, `window:${explicitWindow}`])
219
+ identities.set(created, { targetId: explicitTargetId, browserContextId: identity.browserContextId })
212
220
  await deadline.run(() => bootstrap!.close())
213
221
  bootstrap = undefined
214
222
  succeeded = true
package/src/trees.ts ADDED
@@ -0,0 +1,43 @@
1
+ /**
2
+ * Flat gradient-boosted tree inference shared by the candidate selector and
3
+ * the block classifier (models exported from sklearn by the benchmark
4
+ * training scripts).
5
+ *
6
+ * @module dsh-web-fetch-playwright/trees
7
+ */
8
+
9
+ /** One tree in flat array form (sklearn layout: leaf when left === -1). */
10
+ export interface FlatTree {
11
+ feature: number[]
12
+ threshold: number[]
13
+ left: number[]
14
+ right: number[]
15
+ value: number[]
16
+ }
17
+
18
+ /** Any additive tree ensemble: `init + learningRate * Σ tree(x)`. */
19
+ export interface TreeEnsemble {
20
+ init: number
21
+ learningRate: number
22
+ trees: FlatTree[]
23
+ /** Compare in float64 (HistGradientBoosting) instead of sklearn trees' float32. */
24
+ float64?: boolean
25
+ }
26
+
27
+ function predictTree(tree: FlatTree, x: number[], float64: boolean): number {
28
+ let node = 0
29
+ while (tree.left[node] !== -1) {
30
+ const feature = tree.feature[node] ?? 0
31
+ // sklearn compares features as float32; mirror that so split boundaries agree.
32
+ const value = x[feature] ?? 0
33
+ node = (float64 ? value : Math.fround(value)) <= (tree.threshold[node] ?? 0) ? tree.left[node] ?? 0 : tree.right[node] ?? 0
34
+ }
35
+ return tree.value[node] ?? 0
36
+ }
37
+
38
+ /** Raw ensemble output for one feature vector. */
39
+ export function predictTrees(model: TreeEnsemble, x: number[]): number {
40
+ let sum = model.init
41
+ for (const tree of model.trees) sum += model.learningRate * predictTree(tree, x, model.float64 === true)
42
+ return sum
43
+ }
package/src/types.ts CHANGED
@@ -13,6 +13,11 @@ export interface PlaywrightResponse {
13
13
  status(): number
14
14
  headers(): Record<string, string>
15
15
  text(): Promise<string>
16
+ /**
17
+ * The raw body bytes — the PDF path reads binary bodies through it (a
18
+ * Node `Buffer` is a `Uint8Array`). Absent on minimal fakes.
19
+ */
20
+ body?(): Promise<Uint8Array>
16
21
  /** This response's URL; absent on minimal fakes. */
17
22
  url?(): string
18
23
  /**