dsh-web-fetch-playwright 0.2.20 → 0.2.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/selector.ts CHANGED
@@ -13,7 +13,7 @@
13
13
  * @module dsh-web-fetch-playwright/selector
14
14
  */
15
15
 
16
- import { CANDIDATE_KINDS, type CandidateKind, type GenerationFacts, NOISE_SUBTREE_TAGS } from './candidates.ts'
16
+ import { CANDIDATE_KINDS, type CandidateKind, CONSENT_TERMS, type GenerationFacts, NOISE_SUBTREE_TAGS } from './candidates.ts'
17
17
  import { SELECTOR_MODEL } from './selector-model.ts'
18
18
  import { blockKey, segmentBlocks, WORD } from './blocks.ts'
19
19
  import { predictTrees, type TreeEnsemble } from './trees.ts'
@@ -37,7 +37,7 @@ export interface CandidateStats {
37
37
  blockList: { key: string, words: number, linkWords: number }[]
38
38
  }
39
39
 
40
- const CONSENT_WORDS = /\b(cookies?|consent|datenschutz|privacy|einwilligung|akzeptieren|accept)\b/gi
40
+ const CONSENT_WORDS = new RegExp(CONSENT_TERMS, 'gi')
41
41
 
42
42
  /** Measure one sanitized candidate body. */
43
43
  export function measureCandidate(kind: CandidateKind, legacy: boolean, root: Element): CandidateStats {
@@ -85,12 +85,19 @@ export function measureCandidate(kind: CandidateKind, legacy: boolean, root: Ele
85
85
  }
86
86
  }
87
87
 
88
- /** Page-shape regexes over the URL. */
88
+ /**
89
+ * Page-shape regexes over the URL: whole words a site of ANY platform would
90
+ * choose for that page type. Single-letter and store-specific path codes
91
+ * (`/p/`, `/c/`, `/t/`, `/dp/`, `/itm/`) are deliberately absent — each is
92
+ * one platform's convention that collides with others (`/p/` is a product
93
+ * on one shop and every post on Substack or Jianshu), so as features they
94
+ * only taught the model which sites were in the training set.
95
+ */
89
96
  const URL_SHAPES: [string, RegExp][] = [
90
- ['product', /\/(products?|p|dp|item|itm|artikel|produkt)\/|[?&](pid|sku|product)=/i],
91
- ['collection', /\/(collections?|categor(y|ies)|shop|c|catalog|kategorie|brands?|department)\b/i],
97
+ ['product', /\/(products?|item|artikel|produkt)\/|[?&](pid|sku|product)=/i],
98
+ ['collection', /\/(collections?|categor(y|ies)|shop|catalog|kategorie|brands?|department)\b/i],
92
99
  ['docs', /\/(docs?|documentation|api|reference|guides?|manual|tutorials?|learn|wiki)\b|^https?:\/\/(docs|developer|developers|wiki)\./i],
93
- ['forum', /\/(t|threads?|forums?|topics?|discussions?|questions|community|comments)\b|^https?:\/\/(community|forum|forums|discuss|answers)\./i],
100
+ ['forum', /\/(threads?|forums?|topics?|discussions?|questions|community|comments)\b|^https?:\/\/(community|forum|forums|discuss|answers)\./i],
94
101
  ['article', /\/(20\d\d|19\d\d)\/|\/(blog|news|articles?|story|stories|posts?|press)\b/i],
95
102
  ['search', /\/search\b|[?&](q|query|s|keyword)=/i],
96
103
  ]
@@ -145,8 +152,12 @@ const log1p = Math.log1p
145
152
  export function pageFeatures(document: Document, url: string, facts: GenerationFacts, doc: CandidateStats | undefined, bodyText: number): number[] {
146
153
  const bodyWords = doc?.words ?? 0
147
154
  const count = (selector: string): number => document.querySelectorAll(selector).length
155
+ // The generator meta, plus what Discourse itself declares for installs
156
+ // that drop it (its data island, its theme meta). Framework markers
157
+ // (`.ember-application`) are not generator evidence: every Ember app
158
+ // carries them.
148
159
  const generator = Array.from(document.querySelectorAll('meta[name="generator" i]'), meta => meta.getAttribute('content') ?? '').join(' ')
149
- + (document.querySelector('#data-preloaded, meta[name="discourse_theme_id"], .ember-application') !== null ? ' discourse' : '')
160
+ + (document.querySelector('#data-preloaded[data-preloaded], meta[name="discourse_theme_id"]') !== null ? ' discourse' : '')
150
161
  const jsonld = Array.from(document.querySelectorAll('script[type="application/ld+json"]'), script => script.textContent ?? '').join('\n')
151
162
  const ogType = (document.querySelector('meta[property="og:type"]')?.getAttribute('content') ?? '').toLowerCase()
152
163
  let path = ''
package/src/types.ts CHANGED
@@ -36,15 +36,15 @@ export interface PlaywrightDownload {
36
36
  }
37
37
 
38
38
  /**
39
- * The page's `APIRequestContext` surface the download recovery refetches
40
- * through — plain HTTP from the Playwright client, the context's cookie jar,
41
- * and none of Chromium's render-or-download semantics. Reuses
42
- * {@link PlaywrightResponse} because that is structurally what an
43
- * `APIResponse` is for the members this plugin reads. Absent on fakes that
44
- * do not exercise the recovery.
39
+ * The page's `APIRequestContext` surface the download recovery and the
40
+ * challenge dead-end fallback refetch through — plain HTTP from the
41
+ * Playwright client, the context's cookie jar, and none of Chromium's
42
+ * render-or-download semantics. Reuses {@link PlaywrightResponse} because
43
+ * that is structurally what an `APIResponse` is for the members this
44
+ * plugin reads. Absent on fakes that do not exercise the recovery.
45
45
  */
46
46
  export interface PlaywrightRequestContext {
47
- get(url: string, options?: { timeout?: number; signal?: AbortSignal }): Promise<PlaywrightResponse>
47
+ get(url: string, options?: { timeout?: number; signal?: AbortSignal; headers?: Record<string, string> }): Promise<PlaywrightResponse>
48
48
  }
49
49
 
50
50
  /** The request side of a response, for main-frame filtering. */
@@ -57,6 +57,15 @@ export interface PlaywrightRequest {
57
57
  frame?(): unknown
58
58
  }
59
59
 
60
+ /**
61
+ * A network request notification, as `page.on('request')` and its settled
62
+ * siblings deliver it. Identity only — the settle's network-quiet tracker
63
+ * keys its open-request map on the object itself.
64
+ */
65
+ export interface PlaywrightWebRequest {
66
+ url?(): string
67
+ }
68
+
60
69
  /** A page inside a context. */
61
70
  export interface PlaywrightPage {
62
71
  goto(url: string, options?: { waitUntil?: 'load' | 'domcontentloaded' | 'networkidle' | 'commit'; timeout?: number }): Promise<PlaywrightResponse | null>
@@ -77,6 +86,17 @@ export interface PlaywrightPage {
77
86
  * document). Absent on minimal fakes (content polling covers them).
78
87
  */
79
88
  on?(event: 'response', listener: (response: PlaywrightResponse) => void): unknown
89
+ /**
90
+ * Request-start notification — the settle's network-quiet tracker. Absent
91
+ * on minimal fakes (the settle falls back to the legacy bounded
92
+ * networkidle wait).
93
+ */
94
+ on?(event: 'request', listener: (request: PlaywrightWebRequest) => void): unknown
95
+ /**
96
+ * Request-settled notification (finished or failed) — closes the tracker's
97
+ * open-request bookkeeping. Absent on minimal fakes.
98
+ */
99
+ on?(event: 'requestfinished' | 'requestfailed', listener: (request: PlaywrightWebRequest) => void): unknown
80
100
  /**
81
101
  * Download notification — a navigation the browser handed to its download
82
102
  * manager instead of committing a document (`Content-Disposition: