dsh-web-fetch-playwright 0.2.20 → 0.2.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/README.zh-CN.md +15 -15
- package/cordis.patch.yml +0 -4
- package/lib/client.js +21 -71
- package/lib/index.js +23564 -22605
- package/lib/types/autoscroll.d.ts +46 -10
- package/lib/types/candidates.d.ts +7 -0
- package/lib/types/client/controller.d.ts +0 -14
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +54 -30
- package/lib/types/index.d.ts +1 -1
- package/lib/types/provider.d.ts +46 -9
- package/lib/types/types.d.ts +7 -6
- package/package.json +1 -1
- package/src/autoscroll.ts +459 -17
- package/src/block-model.ts +1 -1
- package/src/candidates.ts +22 -6
- package/src/client/card.tsx +0 -26
- package/src/client/controller.ts +0 -18
- package/src/client/locales.ts +0 -14
- package/src/config.ts +67 -38
- package/src/index.ts +2 -1
- package/src/provider.ts +198 -37
- package/src/selector-model.ts +1 -1
- package/src/selector.ts +18 -7
- package/src/types.ts +7 -7
package/src/selector.ts
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* @module dsh-web-fetch-playwright/selector
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
|
-
import { CANDIDATE_KINDS, type CandidateKind, type GenerationFacts, NOISE_SUBTREE_TAGS } from './candidates.ts'
|
|
16
|
+
import { CANDIDATE_KINDS, type CandidateKind, CONSENT_TERMS, type GenerationFacts, NOISE_SUBTREE_TAGS } from './candidates.ts'
|
|
17
17
|
import { SELECTOR_MODEL } from './selector-model.ts'
|
|
18
18
|
import { blockKey, segmentBlocks, WORD } from './blocks.ts'
|
|
19
19
|
import { predictTrees, type TreeEnsemble } from './trees.ts'
|
|
@@ -37,7 +37,7 @@ export interface CandidateStats {
|
|
|
37
37
|
blockList: { key: string, words: number, linkWords: number }[]
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
-
const CONSENT_WORDS =
|
|
40
|
+
const CONSENT_WORDS = new RegExp(CONSENT_TERMS, 'gi')
|
|
41
41
|
|
|
42
42
|
/** Measure one sanitized candidate body. */
|
|
43
43
|
export function measureCandidate(kind: CandidateKind, legacy: boolean, root: Element): CandidateStats {
|
|
@@ -85,12 +85,19 @@ export function measureCandidate(kind: CandidateKind, legacy: boolean, root: Ele
|
|
|
85
85
|
}
|
|
86
86
|
}
|
|
87
87
|
|
|
88
|
-
/**
|
|
88
|
+
/**
|
|
89
|
+
* Page-shape regexes over the URL: whole words a site of ANY platform would
|
|
90
|
+
* choose for that page type. Single-letter and store-specific path codes
|
|
91
|
+
* (`/p/`, `/c/`, `/t/`, `/dp/`, `/itm/`) are deliberately absent — each is
|
|
92
|
+
* one platform's convention that collides with others (`/p/` is a product
|
|
93
|
+
* on one shop and every post on Substack or Jianshu), so as features they
|
|
94
|
+
* only taught the model which sites were in the training set.
|
|
95
|
+
*/
|
|
89
96
|
const URL_SHAPES: [string, RegExp][] = [
|
|
90
|
-
['product', /\/(products?|
|
|
91
|
-
['collection', /\/(collections?|categor(y|ies)|shop|
|
|
97
|
+
['product', /\/(products?|item|artikel|produkt)\/|[?&](pid|sku|product)=/i],
|
|
98
|
+
['collection', /\/(collections?|categor(y|ies)|shop|catalog|kategorie|brands?|department)\b/i],
|
|
92
99
|
['docs', /\/(docs?|documentation|api|reference|guides?|manual|tutorials?|learn|wiki)\b|^https?:\/\/(docs|developer|developers|wiki)\./i],
|
|
93
|
-
['forum', /\/(
|
|
100
|
+
['forum', /\/(threads?|forums?|topics?|discussions?|questions|community|comments)\b|^https?:\/\/(community|forum|forums|discuss|answers)\./i],
|
|
94
101
|
['article', /\/(20\d\d|19\d\d)\/|\/(blog|news|articles?|story|stories|posts?|press)\b/i],
|
|
95
102
|
['search', /\/search\b|[?&](q|query|s|keyword)=/i],
|
|
96
103
|
]
|
|
@@ -145,8 +152,12 @@ const log1p = Math.log1p
|
|
|
145
152
|
export function pageFeatures(document: Document, url: string, facts: GenerationFacts, doc: CandidateStats | undefined, bodyText: number): number[] {
|
|
146
153
|
const bodyWords = doc?.words ?? 0
|
|
147
154
|
const count = (selector: string): number => document.querySelectorAll(selector).length
|
|
155
|
+
// The generator meta, plus what Discourse itself declares for installs
|
|
156
|
+
// that drop it (its data island, its theme meta). Framework markers
|
|
157
|
+
// (`.ember-application`) are not generator evidence: every Ember app
|
|
158
|
+
// carries them.
|
|
148
159
|
const generator = Array.from(document.querySelectorAll('meta[name="generator" i]'), meta => meta.getAttribute('content') ?? '').join(' ')
|
|
149
|
-
+ (document.querySelector('#data-preloaded, meta[name="discourse_theme_id"]
|
|
160
|
+
+ (document.querySelector('#data-preloaded[data-preloaded], meta[name="discourse_theme_id"]') !== null ? ' discourse' : '')
|
|
150
161
|
const jsonld = Array.from(document.querySelectorAll('script[type="application/ld+json"]'), script => script.textContent ?? '').join('\n')
|
|
151
162
|
const ogType = (document.querySelector('meta[property="og:type"]')?.getAttribute('content') ?? '').toLowerCase()
|
|
152
163
|
let path = ''
|
package/src/types.ts
CHANGED
|
@@ -36,15 +36,15 @@ export interface PlaywrightDownload {
|
|
|
36
36
|
}
|
|
37
37
|
|
|
38
38
|
/**
|
|
39
|
-
* The page's `APIRequestContext` surface the download recovery
|
|
40
|
-
* through — plain HTTP from the
|
|
41
|
-
* and none of Chromium's
|
|
42
|
-
* {@link PlaywrightResponse} because
|
|
43
|
-
* `APIResponse` is for the members this
|
|
44
|
-
* do not exercise the recovery.
|
|
39
|
+
* The page's `APIRequestContext` surface the download recovery and the
|
|
40
|
+
* challenge dead-end fallback refetch through — plain HTTP from the
|
|
41
|
+
* Playwright client, the context's cookie jar, and none of Chromium's
|
|
42
|
+
* render-or-download semantics. Reuses {@link PlaywrightResponse} because
|
|
43
|
+
* that is structurally what an `APIResponse` is for the members this
|
|
44
|
+
* plugin reads. Absent on fakes that do not exercise the recovery.
|
|
45
45
|
*/
|
|
46
46
|
export interface PlaywrightRequestContext {
|
|
47
|
-
get(url: string, options?: { timeout?: number; signal?: AbortSignal }): Promise<PlaywrightResponse>
|
|
47
|
+
get(url: string, options?: { timeout?: number; signal?: AbortSignal; headers?: Record<string, string> }): Promise<PlaywrightResponse>
|
|
48
48
|
}
|
|
49
49
|
|
|
50
50
|
/** The request side of a response, for main-frame filtering. */
|