dsh-web-fetch-playwright 0.2.18 → 0.2.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -1
- package/README.zh-CN.md +17 -1
- package/lib/client.js +18 -18
- package/lib/index.js +58203 -423
- package/lib/types/block-model.d.ts +2 -0
- package/lib/types/blocks.d.ts +67 -0
- package/lib/types/candidates.d.ts +157 -0
- package/lib/types/index.d.ts +6 -2
- package/lib/types/markdown.d.ts +50 -24
- package/lib/types/pdf.d.ts +143 -0
- package/lib/types/provider.d.ts +9 -0
- package/lib/types/selector-model.d.ts +2 -0
- package/lib/types/selector.d.ts +58 -0
- package/lib/types/trees.d.ts +25 -0
- package/lib/types/types.d.ts +5 -0
- package/package.json +3 -2
- package/src/block-model.ts +5 -0
- package/src/blocks.ts +223 -0
- package/src/candidates.ts +639 -0
- package/src/index.ts +6 -2
- package/src/markdown.ts +136 -413
- package/src/pdf.ts +1114 -0
- package/src/provider.ts +117 -8
- package/src/selector-model.ts +6 -0
- package/src/selector.ts +277 -0
- package/src/session-window.ts +8 -0
- package/src/trees.ts +43 -0
- package/src/types.ts +5 -0
package/src/selector.ts
ADDED
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Learned candidate selector: page-level + candidate-level features → an
|
|
3
|
+
* estimated word-F1 per candidate (gradient-boosted regression trees trained
|
|
4
|
+
* offline by `benchmark/train-selector.py`); the pipeline returns the
|
|
5
|
+
* candidate with the highest estimate.
|
|
6
|
+
*
|
|
7
|
+
* Page features double as a page-type signal (generator meta, JSON-LD
|
|
8
|
+
* `@type`, `og:type`, URL shape, DOM statistics), so the trees learn
|
|
9
|
+
* type-conditional choices — "prune on articles, keep the link directory on
|
|
10
|
+
* docs", "unwrap noscript on forums only" — that no single global threshold
|
|
11
|
+
* could express.
|
|
12
|
+
*
|
|
13
|
+
* @module dsh-web-fetch-playwright/selector
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { CANDIDATE_KINDS, type CandidateKind, type GenerationFacts, NOISE_SUBTREE_TAGS } from './candidates.ts'
|
|
17
|
+
import { SELECTOR_MODEL } from './selector-model.ts'
|
|
18
|
+
import { blockKey, segmentBlocks, WORD } from './blocks.ts'
|
|
19
|
+
import { predictTrees, type TreeEnsemble } from './trees.ts'
|
|
20
|
+
|
|
21
|
+
/** Text statistics of one sanitized candidate. */
|
|
22
|
+
export interface CandidateStats {
|
|
23
|
+
kind: CandidateKind
|
|
24
|
+
legacy: boolean
|
|
25
|
+
words: number
|
|
26
|
+
linkWords: number
|
|
27
|
+
paragraphs: number
|
|
28
|
+
headings: number
|
|
29
|
+
listItems: number
|
|
30
|
+
tableCells: number
|
|
31
|
+
pres: number
|
|
32
|
+
images: number
|
|
33
|
+
blocks: number
|
|
34
|
+
tokens: Set<string>
|
|
35
|
+
consentHits: number
|
|
36
|
+
/** Text blocks as (key, words, linked words), for block-level diffs. */
|
|
37
|
+
blockList: { key: string, words: number, linkWords: number }[]
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const CONSENT_WORDS = /\b(cookies?|consent|datenschutz|privacy|einwilligung|akzeptieren|accept)\b/gi
|
|
41
|
+
|
|
42
|
+
/** Measure one sanitized candidate body. */
|
|
43
|
+
export function measureCandidate(kind: CandidateKind, legacy: boolean, root: Element): CandidateStats {
|
|
44
|
+
let words = 0
|
|
45
|
+
let linkWords = 0
|
|
46
|
+
const tokens = new Set<string>()
|
|
47
|
+
let head = ''
|
|
48
|
+
const visit = (node: Node, inAnchor: boolean): void => {
|
|
49
|
+
if (node.nodeType === 3) {
|
|
50
|
+
const text = node.nodeValue ?? ''
|
|
51
|
+
const matches = text.toLowerCase().match(WORD)
|
|
52
|
+
if (matches !== null) {
|
|
53
|
+
words += matches.length
|
|
54
|
+
if (inAnchor) linkWords += matches.length
|
|
55
|
+
if (tokens.size < 20_000) for (const token of matches) tokens.add(token)
|
|
56
|
+
}
|
|
57
|
+
if (head.length < 600) head += ` ${text}`
|
|
58
|
+
return
|
|
59
|
+
}
|
|
60
|
+
if (node.nodeType !== 1 || NOISE_SUBTREE_TAGS.has(node.nodeName.toLowerCase())) return
|
|
61
|
+
const anchor = inAnchor || node.nodeName === 'A'
|
|
62
|
+
for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child, anchor)
|
|
63
|
+
}
|
|
64
|
+
visit(root, false)
|
|
65
|
+
const count = (selector: string): number => root.querySelectorAll(selector).length
|
|
66
|
+
const paragraphs = count('p')
|
|
67
|
+
const listItems = count('li')
|
|
68
|
+
const headings = count('h1,h2,h3,h4,h5,h6')
|
|
69
|
+
const tableCells = count('td,th')
|
|
70
|
+
return {
|
|
71
|
+
kind,
|
|
72
|
+
legacy,
|
|
73
|
+
words,
|
|
74
|
+
linkWords,
|
|
75
|
+
paragraphs,
|
|
76
|
+
headings,
|
|
77
|
+
listItems,
|
|
78
|
+
tableCells,
|
|
79
|
+
pres: count('pre'),
|
|
80
|
+
images: count('img'),
|
|
81
|
+
blocks: paragraphs + listItems + headings + tableCells + count('div,section,article,blockquote,dd,dt'),
|
|
82
|
+
tokens,
|
|
83
|
+
consentHits: (head.match(CONSENT_WORDS) ?? []).length,
|
|
84
|
+
blockList: segmentBlocks(root).map(block => ({ key: blockKey(block.text), words: block.words, linkWords: block.linkWords })),
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Page-shape regexes over the URL. */
|
|
89
|
+
const URL_SHAPES: [string, RegExp][] = [
|
|
90
|
+
['product', /\/(products?|p|dp|item|itm|artikel|produkt)\/|[?&](pid|sku|product)=/i],
|
|
91
|
+
['collection', /\/(collections?|categor(y|ies)|shop|c|catalog|kategorie|brands?|department)\b/i],
|
|
92
|
+
['docs', /\/(docs?|documentation|api|reference|guides?|manual|tutorials?|learn|wiki)\b|^https?:\/\/(docs|developer|developers|wiki)\./i],
|
|
93
|
+
['forum', /\/(t|threads?|forums?|topics?|discussions?|questions|community|comments)\b|^https?:\/\/(community|forum|forums|discuss|answers)\./i],
|
|
94
|
+
['article', /\/(20\d\d|19\d\d)\/|\/(blog|news|articles?|story|stories|posts?|press)\b/i],
|
|
95
|
+
['search', /\/search\b|[?&](q|query|s|keyword)=/i],
|
|
96
|
+
]
|
|
97
|
+
|
|
98
|
+
const GENERATORS: [string, RegExp][] = [
|
|
99
|
+
['discourse', /discourse/i],
|
|
100
|
+
['wordpress', /wordpress/i],
|
|
101
|
+
['shop', /shopify|woocommerce|magento|prestashop|shopware/i],
|
|
102
|
+
['forumsw', /flarum|xenforo|vbulletin|phpbb|discuz|invision|mybb|nodebb/i],
|
|
103
|
+
['docsgen', /sphinx|docusaurus|mkdocs|hugo|gitbook|jekyll|vitepress|mediawiki|readthedocs/i],
|
|
104
|
+
['cms', /drupal|joomla|ghost|wix|squarespace|hubspot|webflow/i],
|
|
105
|
+
]
|
|
106
|
+
|
|
107
|
+
const JSONLD_TYPES: [string, RegExp][] = [
|
|
108
|
+
['product', /"@type"\s*:\s*"?(Product|Offer|AggregateOffer|ProductGroup)/],
|
|
109
|
+
['article', /"@type"\s*:\s*"?(Article|NewsArticle|BlogPosting|Report|ScholarlyArticle|TechArticle)/],
|
|
110
|
+
['forum', /"@type"\s*:\s*"?(DiscussionForumPosting|QAPage|Question|SocialMediaPosting|Comment)/],
|
|
111
|
+
['list', /"@type"\s*:\s*"?(ItemList|CollectionPage|SearchResultsPage|OfferCatalog)/],
|
|
112
|
+
['faq', /"@type"\s*:\s*"?(FAQPage|HowTo)/],
|
|
113
|
+
['org', /"@type"\s*:\s*"?(Service|LocalBusiness|Organization|Corporation|EducationalOrganization|Course|Event)/],
|
|
114
|
+
['breadcrumb', /"@type"\s*:\s*"?BreadcrumbList/],
|
|
115
|
+
]
|
|
116
|
+
|
|
117
|
+
const PRICE = /(?:[$€£¥₹]\s?\d|\d[\d.,]*\s?(?:€|EUR|USD|zł|kr|CHF))/g
|
|
118
|
+
|
|
119
|
+
/** Page-level feature names, in vector order. */
|
|
120
|
+
export const PAGE_FEATURES = [
|
|
121
|
+
'body_words_log', 'noscript_share', 'body_link_density', 'readability_null', 'gate_pass', 'prune_changed',
|
|
122
|
+
'has_main', 'has_preloaded', 'record_items_log', 'record_share',
|
|
123
|
+
...URL_SHAPES.map(([name]) => `url_${name}`), 'url_depth', 'url_root',
|
|
124
|
+
...GENERATORS.map(([name]) => `gen_${name}`),
|
|
125
|
+
...JSONLD_TYPES.map(([name]) => `ld_${name}`),
|
|
126
|
+
'og_article', 'og_product', 'og_website',
|
|
127
|
+
'n_article_log', 'n_h1', 'n_h2_log', 'n_pre_log', 'n_time_log', 'n_form_log', 'n_price_log',
|
|
128
|
+
'n_itemprop_price', 'n_product_itemtype', 'n_post_class_log', 'n_table_log', 'n_code_log', 'n_card_class_log',
|
|
129
|
+
] as const
|
|
130
|
+
|
|
131
|
+
/** Candidate-level feature names, in vector order (one-hot kinds appended). */
|
|
132
|
+
export const CANDIDATE_FEATURES = [
|
|
133
|
+
'words_log', 'words_vs_doc', 'words_vs_max', 'words_vs_art_log', 'link_density', 'paragraphs_log', 'headings_log',
|
|
134
|
+
'li_share', 'td_log', 'pre_log', 'img_log', 'words_per_block', 'precision_vs_art', 'recall_vs_art',
|
|
135
|
+
'precision_vs_doc', 'consent_hits', 'legacy',
|
|
136
|
+
'extra_share', 'extra_link_density', 'extra_long_log', 'extra_words_per_block', 'missing_share',
|
|
137
|
+
...CANDIDATE_KINDS.map(kind => `kind_${kind}`),
|
|
138
|
+
] as const
|
|
139
|
+
|
|
140
|
+
export const FEATURE_NAMES = [...PAGE_FEATURES, ...CANDIDATE_FEATURES]
|
|
141
|
+
|
|
142
|
+
const log1p = Math.log1p
|
|
143
|
+
|
|
144
|
+
/** Compute the page feature vector. */
|
|
145
|
+
export function pageFeatures(document: Document, url: string, facts: GenerationFacts, doc: CandidateStats | undefined, bodyText: number): number[] {
|
|
146
|
+
const bodyWords = doc?.words ?? 0
|
|
147
|
+
const count = (selector: string): number => document.querySelectorAll(selector).length
|
|
148
|
+
const generator = Array.from(document.querySelectorAll('meta[name="generator" i]'), meta => meta.getAttribute('content') ?? '').join(' ')
|
|
149
|
+
+ (document.querySelector('#data-preloaded, meta[name="discourse_theme_id"], .ember-application') !== null ? ' discourse' : '')
|
|
150
|
+
const jsonld = Array.from(document.querySelectorAll('script[type="application/ld+json"]'), script => script.textContent ?? '').join('\n')
|
|
151
|
+
const ogType = (document.querySelector('meta[property="og:type"]')?.getAttribute('content') ?? '').toLowerCase()
|
|
152
|
+
let path = ''
|
|
153
|
+
try { path = new URL(url).pathname } catch { /* keep empty */ }
|
|
154
|
+
const bodyTextContent = document.body?.textContent ?? ''
|
|
155
|
+
const prices = bodyTextContent.length > 2_000_000 ? 0 : (bodyTextContent.match(PRICE) ?? []).length
|
|
156
|
+
return [
|
|
157
|
+
log1p(bodyWords),
|
|
158
|
+
facts.noscriptText / Math.max(1, bodyText + facts.noscriptText),
|
|
159
|
+
doc === undefined || doc.words === 0 ? 0 : doc.linkWords / doc.words,
|
|
160
|
+
facts.readabilityNull ? 1 : 0,
|
|
161
|
+
facts.gatePass ? 1 : 0,
|
|
162
|
+
facts.pruneChanged ? 1 : 0,
|
|
163
|
+
facts.hasMain ? 1 : 0,
|
|
164
|
+
facts.hasPreloaded ? 1 : 0,
|
|
165
|
+
log1p(facts.record?.items.length ?? 0),
|
|
166
|
+
(facts.record?.text ?? 0) / Math.max(1, bodyText),
|
|
167
|
+
...URL_SHAPES.map(([, pattern]) => pattern.test(url) ? 1 : 0),
|
|
168
|
+
path.split('/').filter(Boolean).length,
|
|
169
|
+
path === '/' || path === '' ? 1 : 0,
|
|
170
|
+
...GENERATORS.map(([, pattern]) => pattern.test(generator) ? 1 : 0),
|
|
171
|
+
...JSONLD_TYPES.map(([, pattern]) => pattern.test(jsonld) ? 1 : 0),
|
|
172
|
+
ogType.includes('article') ? 1 : 0,
|
|
173
|
+
ogType.includes('product') ? 1 : 0,
|
|
174
|
+
ogType === 'website' ? 1 : 0,
|
|
175
|
+
log1p(count('article')),
|
|
176
|
+
count('h1'),
|
|
177
|
+
log1p(count('h2')),
|
|
178
|
+
log1p(count('pre')),
|
|
179
|
+
log1p(count('time')),
|
|
180
|
+
log1p(count('form')),
|
|
181
|
+
log1p(prices),
|
|
182
|
+
count('[itemprop="price"]') > 0 ? 1 : 0,
|
|
183
|
+
count('[itemtype*="Product" i]') > 0 ? 1 : 0,
|
|
184
|
+
log1p(count('[class*="post" i], [class*="comment" i], [class*="reply" i]')),
|
|
185
|
+
log1p(count('table')),
|
|
186
|
+
log1p(count('code')),
|
|
187
|
+
log1p(count('[class*="card" i], [class*="product" i], [class*="tile" i]')),
|
|
188
|
+
]
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function overlap(tokens: Set<string>, reference: Set<string> | undefined): number {
|
|
192
|
+
if (reference === undefined || tokens.size === 0) return 0
|
|
193
|
+
let hit = 0
|
|
194
|
+
for (const token of tokens) if (reference.has(token)) hit++
|
|
195
|
+
return hit / tokens.size
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* How a candidate differs from Readability's pick, block by block: the
|
|
200
|
+
* blocks it adds (long prose → content Readability missed; short link-dense
|
|
201
|
+
* rows → chrome) and the share of Readability's blocks it drops.
|
|
202
|
+
*/
|
|
203
|
+
function blockDiff(stats: CandidateStats, art: CandidateStats | undefined): number[] {
|
|
204
|
+
if (art === undefined) return [0, 0, 0, 0, 0]
|
|
205
|
+
const artKeys = new Set(art.blockList.map(block => block.key))
|
|
206
|
+
const ownKeys = new Set(stats.blockList.map(block => block.key))
|
|
207
|
+
let total = 0
|
|
208
|
+
let extraWords = 0
|
|
209
|
+
let extraLinks = 0
|
|
210
|
+
let extraLong = 0
|
|
211
|
+
let extraBlocks = 0
|
|
212
|
+
for (const block of stats.blockList) {
|
|
213
|
+
total += block.words
|
|
214
|
+
if (artKeys.has(block.key)) continue
|
|
215
|
+
extraWords += block.words
|
|
216
|
+
extraLinks += block.linkWords
|
|
217
|
+
extraBlocks++
|
|
218
|
+
if (block.words >= 30) extraLong++
|
|
219
|
+
}
|
|
220
|
+
let artTotal = 0
|
|
221
|
+
let missing = 0
|
|
222
|
+
for (const block of art.blockList) {
|
|
223
|
+
artTotal += block.words
|
|
224
|
+
if (!ownKeys.has(block.key)) missing += block.words
|
|
225
|
+
}
|
|
226
|
+
return [
|
|
227
|
+
extraWords / Math.max(1, total),
|
|
228
|
+
extraWords === 0 ? 0 : extraLinks / extraWords,
|
|
229
|
+
Math.log1p(extraLong),
|
|
230
|
+
extraBlocks === 0 ? 0 : extraWords / extraBlocks,
|
|
231
|
+
missing / Math.max(1, artTotal),
|
|
232
|
+
]
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** Compute one candidate's feature vector (page vector excluded). */
|
|
236
|
+
export function candidateFeatures(stats: CandidateStats, all: CandidateStats[]): number[] {
|
|
237
|
+
const doc = all.find(c => c.kind === 'doc')
|
|
238
|
+
const art = all.find(c => c.kind === 'art')
|
|
239
|
+
const maxWords = Math.max(1, ...all.map(c => c.words))
|
|
240
|
+
return [
|
|
241
|
+
log1p(stats.words),
|
|
242
|
+
stats.words / Math.max(1, doc?.words ?? 0),
|
|
243
|
+
stats.words / maxWords,
|
|
244
|
+
art === undefined ? 0 : Math.log((stats.words + 1) / (art.words + 1)),
|
|
245
|
+
stats.words === 0 ? 0 : stats.linkWords / stats.words,
|
|
246
|
+
log1p(stats.paragraphs),
|
|
247
|
+
log1p(stats.headings),
|
|
248
|
+
stats.listItems / Math.max(1, stats.blocks),
|
|
249
|
+
log1p(stats.tableCells),
|
|
250
|
+
log1p(stats.pres),
|
|
251
|
+
log1p(stats.images),
|
|
252
|
+
stats.words / Math.max(1, stats.blocks),
|
|
253
|
+
overlap(stats.tokens, art?.tokens),
|
|
254
|
+
art === undefined ? 0 : overlap(art.tokens, stats.tokens),
|
|
255
|
+
overlap(stats.tokens, doc?.tokens),
|
|
256
|
+
stats.consentHits,
|
|
257
|
+
stats.legacy ? 1 : 0,
|
|
258
|
+
...blockDiff(stats, art),
|
|
259
|
+
...CANDIDATE_KINDS.map(kind => kind === stats.kind ? 1 : 0),
|
|
260
|
+
]
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/** A trained gradient-boosted regressor. */
|
|
264
|
+
export interface SelectorModel extends TreeEnsemble {
|
|
265
|
+
version: string
|
|
266
|
+
features: readonly string[]
|
|
267
|
+
/** Estimate lead another candidate needs over the legacy pick to replace it. */
|
|
268
|
+
margin?: number
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/** Estimated quality of one candidate; falls back to the legacy choice without a model. */
|
|
272
|
+
export function scoreCandidate(vector: number[], model: SelectorModel = SELECTOR_MODEL): number {
|
|
273
|
+
if (model.trees.length === 0) {
|
|
274
|
+
return vector[FEATURE_NAMES.indexOf('legacy')] ?? 0
|
|
275
|
+
}
|
|
276
|
+
return predictTrees(model, vector) + (vector[FEATURE_NAMES.indexOf('legacy')] === 1 ? model.margin ?? 0 : 0)
|
|
277
|
+
}
|
package/src/session-window.ts
CHANGED
|
@@ -18,6 +18,10 @@ interface TargetIdentity { targetId: string; browserContextId: string }
|
|
|
18
18
|
const ownership = new WeakMap<PlaywrightBrowser, Map<string, WeakRef<PlaywrightContext>>>()
|
|
19
19
|
// Never adopt an unconfirmed late tab after timing out, nor close sibling leases.
|
|
20
20
|
const unavailable = new WeakSet<PlaywrightContext>()
|
|
21
|
+
// Identities confirmed when each session tab was created. Re-identifying a busy
|
|
22
|
+
// sibling would attach to it, and detaching from a tab mid-navigation can stall
|
|
23
|
+
// until its response arrives; a target id and its context never change.
|
|
24
|
+
const identities = new WeakMap<PlaywrightPage, TargetIdentity>()
|
|
21
25
|
const CLEANUP_GRACE_MS = 100
|
|
22
26
|
|
|
23
27
|
function claim(browser: PlaywrightBrowser, context: PlaywrightContext, keys: string[]): void {
|
|
@@ -112,6 +116,8 @@ export async function createSessionPage(
|
|
|
112
116
|
return session
|
|
113
117
|
}
|
|
114
118
|
const identify = async (page: PlaywrightPage): Promise<TargetIdentity> => {
|
|
119
|
+
const known = identities.get(page)
|
|
120
|
+
if (known) return known
|
|
115
121
|
const session = await newSession(() => ctx.newCDPSession(page))
|
|
116
122
|
let identificationFailed = false
|
|
117
123
|
try {
|
|
@@ -164,6 +170,7 @@ export async function createSessionPage(
|
|
|
164
170
|
throw new Error('Chromium did not place the session tab in its dedicated window')
|
|
165
171
|
}
|
|
166
172
|
claim(browser, context, [`context:${actual.browserContextId}`, `window:${expectedWindow}`])
|
|
173
|
+
identities.set(created, actual)
|
|
167
174
|
succeeded = true
|
|
168
175
|
return created
|
|
169
176
|
}
|
|
@@ -209,6 +216,7 @@ export async function createSessionPage(
|
|
|
209
216
|
const explicitWindow = await windowOf(explicitTargetId)
|
|
210
217
|
if (explicitWindow === bootstrapWindow) throw new Error('Chromium ignored the dedicated session window request')
|
|
211
218
|
claim(browser, context, [`context:${identity.browserContextId}`, `window:${explicitWindow}`])
|
|
219
|
+
identities.set(created, { targetId: explicitTargetId, browserContextId: identity.browserContextId })
|
|
212
220
|
await deadline.run(() => bootstrap!.close())
|
|
213
221
|
bootstrap = undefined
|
|
214
222
|
succeeded = true
|
package/src/trees.ts
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Flat gradient-boosted tree inference shared by the candidate selector and
|
|
3
|
+
* the block classifier (models exported from sklearn by the benchmark
|
|
4
|
+
* training scripts).
|
|
5
|
+
*
|
|
6
|
+
* @module dsh-web-fetch-playwright/trees
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/** One tree in flat array form (sklearn layout: leaf when left === -1). */
|
|
10
|
+
export interface FlatTree {
|
|
11
|
+
feature: number[]
|
|
12
|
+
threshold: number[]
|
|
13
|
+
left: number[]
|
|
14
|
+
right: number[]
|
|
15
|
+
value: number[]
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** Any additive tree ensemble: `init + learningRate * Σ tree(x)`. */
|
|
19
|
+
export interface TreeEnsemble {
|
|
20
|
+
init: number
|
|
21
|
+
learningRate: number
|
|
22
|
+
trees: FlatTree[]
|
|
23
|
+
/** Compare in float64 (HistGradientBoosting) instead of sklearn trees' float32. */
|
|
24
|
+
float64?: boolean
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function predictTree(tree: FlatTree, x: number[], float64: boolean): number {
|
|
28
|
+
let node = 0
|
|
29
|
+
while (tree.left[node] !== -1) {
|
|
30
|
+
const feature = tree.feature[node] ?? 0
|
|
31
|
+
// sklearn compares features as float32; mirror that so split boundaries agree.
|
|
32
|
+
const value = x[feature] ?? 0
|
|
33
|
+
node = (float64 ? value : Math.fround(value)) <= (tree.threshold[node] ?? 0) ? tree.left[node] ?? 0 : tree.right[node] ?? 0
|
|
34
|
+
}
|
|
35
|
+
return tree.value[node] ?? 0
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Raw ensemble output for one feature vector. */
|
|
39
|
+
export function predictTrees(model: TreeEnsemble, x: number[]): number {
|
|
40
|
+
let sum = model.init
|
|
41
|
+
for (const tree of model.trees) sum += model.learningRate * predictTree(tree, x, model.float64 === true)
|
|
42
|
+
return sum
|
|
43
|
+
}
|
package/src/types.ts
CHANGED
|
@@ -13,6 +13,11 @@ export interface PlaywrightResponse {
|
|
|
13
13
|
status(): number
|
|
14
14
|
headers(): Record<string, string>
|
|
15
15
|
text(): Promise<string>
|
|
16
|
+
/**
|
|
17
|
+
* The raw body bytes — the PDF path reads binary bodies through it (a
|
|
18
|
+
* Node `Buffer` is a `Uint8Array`). Absent on minimal fakes.
|
|
19
|
+
*/
|
|
20
|
+
body?(): Promise<Uint8Array>
|
|
16
21
|
/** This response's URL; absent on minimal fakes. */
|
|
17
22
|
url?(): string
|
|
18
23
|
/**
|