@duckduckgo/autoconsent 16.59.0 → 16.61.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/.agents/lib/regional-testing/harness.mjs +4 -172
  2. package/.agents/skills/oxylabs-testing/SKILL.md +4 -4
  3. package/.agents/skills/oxylabs-testing/scripts/oxylabs.mjs +17 -164
  4. package/.agents/skills/oxylabs-testing/scripts/oxylabs.test.mjs +6 -3
  5. package/.agents/skills/verify-pr/SKILL.md +1 -1
  6. package/AGENTS.md +1 -1
  7. package/CHANGELOG.md +107 -0
  8. package/Jenkinsfile +36 -1
  9. package/dist/addon-firefox/background.bundle.js +21 -0
  10. package/dist/addon-firefox/compact-rules.json +1 -1
  11. package/dist/addon-firefox/content.bundle.js +253 -93
  12. package/dist/addon-firefox/manifest.json +1 -1
  13. package/dist/addon-firefox/rule-index.json +1 -1
  14. package/dist/addon-firefox/rules.json +1 -1
  15. package/dist/addon-mv3/background.bundle.js +21 -0
  16. package/dist/addon-mv3/compact-rules.json +1 -1
  17. package/dist/addon-mv3/content.bundle.js +253 -93
  18. package/dist/addon-mv3/manifest.json +1 -1
  19. package/dist/addon-mv3/rule-index.json +1 -1
  20. package/dist/addon-mv3/rules.json +1 -1
  21. package/dist/autoconsent.cjs.js +253 -93
  22. package/dist/autoconsent.esm.js +253 -93
  23. package/dist/autoconsent.playwright.js +253 -93
  24. package/dist/autoconsent.standalone.js +254 -94
  25. package/dist/heuristics.cjs.js +29 -3
  26. package/dist/heuristics.mjs +28 -3
  27. package/dist/types/cmps/onetrust.d.ts +1 -0
  28. package/dist/types/eval-snippets.d.ts +3 -0
  29. package/dist/types/heuristic-classify.d.ts +1 -0
  30. package/dist/types/heuristics.d.ts +10 -1
  31. package/lib/cmps/onetrust.ts +15 -0
  32. package/lib/eval-snippets.ts +21 -0
  33. package/lib/heuristic-classify.ts +6 -0
  34. package/lib/heuristic-patterns.ts +26 -1
  35. package/lib/heuristics.ts +256 -104
  36. package/package.json +1 -1
  37. package/playwright/isolated-world.mjs +217 -0
  38. package/playwright/oxylabs.mjs +196 -0
  39. package/playwright/runner.ts +183 -39
  40. package/playwright.config.ts +4 -1
  41. package/rules/autoconsent/aeolservice.json +39 -0
  42. package/rules/autoconsent/airbnb.json +32 -0
  43. package/rules/autoconsent/amazon-music.json +13 -0
  44. package/rules/autoconsent/annunci69.json +19 -0
  45. package/rules/autoconsent/appconsent.json +6 -1
  46. package/rules/autoconsent/arbeidsplassen-nav-no.json +33 -0
  47. package/rules/autoconsent/att.json +36 -0
  48. package/rules/autoconsent/auctionet.json +39 -0
  49. package/rules/autoconsent/autodoc.json +18 -0
  50. package/rules/autoconsent/bbc.com-orb-cosmetic.json +34 -0
  51. package/rules/autoconsent/bbc.com.json +8 -5
  52. package/rules/autoconsent/bokadirekt.se.json +50 -0
  53. package/rules/autoconsent/bouvet-cookiepanel.json +29 -0
  54. package/rules/autoconsent/com-consent-layer.json +10 -0
  55. package/rules/autoconsent/consent-studio.json +25 -13
  56. package/rules/autoconsent/cookielegit.json +21 -0
  57. package/rules/autoconsent/cookieprompter.json +21 -0
  58. package/rules/autoconsent/cookiereports.json +10 -0
  59. package/rules/autoconsent/cookietractor.json +37 -0
  60. package/rules/autoconsent/de-volksbank.json +14 -0
  61. package/rules/autoconsent/dpgmedia-nl.json +20 -8
  62. package/rules/autoconsent/eon.pl.json +18 -0
  63. package/rules/autoconsent/eroticiracconti-it.json +13 -0
  64. package/rules/autoconsent/ersatzteile-24.json +49 -0
  65. package/rules/autoconsent/fastweb-cookie-center.json +13 -0
  66. package/rules/autoconsent/futurabolzano-it.json +35 -0
  67. package/rules/autoconsent/gobasic.json +10 -0
  68. package/rules/autoconsent/gravito.json +23 -16
  69. package/rules/autoconsent/hacon-next-gen.json +41 -0
  70. package/rules/autoconsent/ing-de.json +31 -0
  71. package/rules/autoconsent/interia.json +1 -4
  72. package/rules/autoconsent/intesasanpaolo.json +37 -0
  73. package/rules/autoconsent/jobindex.json +40 -0
  74. package/rules/autoconsent/kaspersky-cookies-notification.json +9 -0
  75. package/rules/autoconsent/lantmateriet.json +13 -0
  76. package/rules/autoconsent/livsmedelsverket.json +18 -0
  77. package/rules/autoconsent/mainichi.json +39 -0
  78. package/rules/autoconsent/mango.json +56 -0
  79. package/rules/autoconsent/nederlandseloterij.json +35 -0
  80. package/rules/autoconsent/newsbreak.json +16 -0
  81. package/rules/autoconsent/nhsapp.json +10 -0
  82. package/rules/autoconsent/nordea.json +21 -0
  83. package/rules/autoconsent/nordnet.json +33 -0
  84. package/rules/autoconsent/ok.json +4 -12
  85. package/rules/autoconsent/onlineradiobox.json +35 -0
  86. package/rules/autoconsent/pe-cookies.json +30 -0
  87. package/rules/autoconsent/pensionsmyndigheten.json +13 -0
  88. package/rules/autoconsent/pro-euclp-cookies-box.json +31 -0
  89. package/rules/autoconsent/pubfactory.json +45 -0
  90. package/rules/autoconsent/puesc-gov-pl.json +37 -0
  91. package/rules/autoconsent/rambler.json +30 -0
  92. package/rules/autoconsent/rejestr-io.json +36 -0
  93. package/rules/autoconsent/sas.json +6 -6
  94. package/rules/autoconsent/scb.se.json +13 -0
  95. package/rules/autoconsent/schumacher.com.json +53 -0
  96. package/rules/autoconsent/sitevision.json +18 -1
  97. package/rules/autoconsent/skatteverket-se.json +33 -0
  98. package/rules/autoconsent/smile-insurances.json +13 -0
  99. package/rules/autoconsent/studentapan.json +16 -0
  100. package/rules/autoconsent/svt.se.json +7 -7
  101. package/rules/autoconsent/temu.json +12 -16
  102. package/rules/autoconsent/thecode.json +21 -0
  103. package/rules/autoconsent/tv2reg-cookie-notice.json +39 -0
  104. package/rules/autoconsent/uk-parliament.json +19 -0
  105. package/rules/autoconsent/wirtualna-polska.json +13 -10
  106. package/rules/autoconsent/xenforo-cookie-consent.json +62 -0
  107. package/rules/autoconsent/zorginstituut-cookiewarning.json +33 -0
  108. package/rules/compact-rules.json +1 -1
  109. package/rules/rule-index.json +1 -1
  110. package/rules/rules.json +1 -1
  111. package/tests/aeolservice.spec.ts +5 -0
  112. package/tests/airbnb.spec.ts +3 -0
  113. package/tests/amazon-music.spec.ts +9 -0
  114. package/tests/annunci69.spec.ts +5 -0
  115. package/tests/appconsent.spec.ts +6 -0
  116. package/tests/arbeidsplassen-nav-no.spec.ts +3 -0
  117. package/tests/att.spec.ts +3 -0
  118. package/tests/auctionet.spec.ts +3 -0
  119. package/tests/autodoc.spec.ts +8 -0
  120. package/tests/bbc.com-orb-cosmetic.spec.ts +8 -0
  121. package/tests/bbc.com.spec.ts +11 -9
  122. package/tests/bokadirekt.se.spec.ts +3 -0
  123. package/tests/bouvet-cookiepanel.spec.ts +9 -0
  124. package/tests/com-consent-layer.spec.ts +6 -0
  125. package/tests/consent-studio.spec.ts +5 -0
  126. package/tests/cookiebot.spec.ts +1 -0
  127. package/tests/cookielegit.spec.ts +3 -0
  128. package/tests/cookieprompter.spec.ts +9 -0
  129. package/tests/cookiereports.spec.ts +13 -0
  130. package/tests/cookietractor.spec.ts +12 -0
  131. package/tests/de-volksbank.spec.ts +5 -0
  132. package/tests/dpgmedia-nl.spec.ts +15 -1
  133. package/tests/eon.pl.spec.ts +3 -0
  134. package/tests/eroticiracconti-it.spec.ts +3 -0
  135. package/tests/ersatzteile-24.spec.ts +6 -0
  136. package/tests/fastweb-cookie-center.spec.ts +3 -0
  137. package/tests/futurabolzano-it.spec.ts +3 -0
  138. package/tests/gobasic.spec.ts +3 -0
  139. package/tests/gravito.spec.ts +7 -1
  140. package/tests/hacon-next-gen.spec.ts +6 -0
  141. package/tests/ing-de.spec.ts +3 -0
  142. package/tests/interia.spec.ts +1 -1
  143. package/tests/intesasanpaolo.spec.ts +3 -0
  144. package/tests/jobindex.spec.ts +3 -0
  145. package/tests/kaspersky-cookies-notification.spec.ts +8 -0
  146. package/tests/lantmateriet.spec.ts +3 -0
  147. package/tests/livsmedelsverket.spec.ts +5 -0
  148. package/tests/mainichi.spec.ts +10 -0
  149. package/tests/mango.spec.ts +3 -0
  150. package/tests/nederlandseloterij.spec.ts +9 -0
  151. package/tests/newsbreak.spec.ts +3 -0
  152. package/tests/nhsapp.spec.ts +3 -0
  153. package/tests/nordea.spec.ts +6 -0
  154. package/tests/nordnet.spec.ts +8 -0
  155. package/tests/onlineradiobox.spec.ts +5 -0
  156. package/tests/pe-cookies.spec.ts +6 -0
  157. package/tests/pensionsmyndigheten.spec.ts +3 -0
  158. package/tests/pro-euclp-cookies-box.spec.ts +8 -0
  159. package/tests/pubfactory.spec.ts +10 -0
  160. package/tests/puesc-gov-pl.spec.ts +3 -0
  161. package/tests/rambler.spec.ts +3 -0
  162. package/tests/rejestr-io.spec.ts +3 -0
  163. package/tests/sas.spec.ts +1 -1
  164. package/tests/scb.se.spec.ts +4 -0
  165. package/tests/schumacher.com.spec.ts +6 -0
  166. package/tests/sitevision-cookie-consent.spec.ts +1 -0
  167. package/tests/sitevision.spec.ts +7 -0
  168. package/tests/skatteverket-se.spec.ts +3 -0
  169. package/tests/smile-insurances.spec.ts +6 -0
  170. package/tests/studentapan.spec.ts +3 -0
  171. package/tests/svt.se.spec.ts +1 -3
  172. package/tests/temu.spec.ts +3 -1
  173. package/tests/thecode.spec.ts +3 -0
  174. package/tests/tv2reg-cookie-notice.spec.ts +16 -0
  175. package/tests/uk-parliament.spec.ts +8 -0
  176. package/tests/wirtualna-polska.spec.ts +6 -1
  177. package/tests/xenforo-cookie-consent.spec.ts +6 -0
  178. package/tests/zorginstituut-cookiewarning.spec.ts +10 -0
  179. package/tests-wtr/heuristics/get-potential-popups.test.ts +330 -9
  180. package/tests-wtr/heuristics/heuristics-utils.test.ts +98 -0
  181. package/rules/autoconsent/twitter.json +0 -34
  182. package/rules/generated/auto_AU_flysas.com_r1s.json +0 -40
  183. package/rules/generated/auto_CH_airbnb.de_83z.json +0 -40
  184. package/rules/generated/auto_CH_autodoc.de_krd.json +0 -40
  185. package/rules/generated/auto_CH_svtplay.se_d39.json +0 -40
  186. package/rules/generated/auto_GB_parliamentlive.tv_r3v.json +0 -28
  187. package/rules/generated/auto_NL_asnbank.nl_e28.json +0 -40
  188. package/rules/generated/auto_NL_eurojackpot.nederlandseloterij.nl_7hr.json +0 -40
  189. package/rules/generated/auto_NL_lotto.nederlandseloterij.nl_7c8.json +0 -40
  190. package/rules/generated/auto_NL_nederlandseloterij.nl_b60.json +0 -40
  191. package/rules/generated/auto_NL_staatsloterij.nederlandseloterij.nl_197.json +0 -40
  192. package/rules/generated/auto_NO_arronet.se_vo5.json +0 -40
  193. package/rules/generated/auto_NO_liseberg.se_qy4.json +0 -40
  194. package/rules/generated/auto_NO_nordnet.se_2en.json +0 -40
  195. package/rules/generated/auto_NO_regeringen.se_uhv.json +0 -40
  196. package/rules/generated/auto_NO_svtplay.se_ujn.json +0 -40
  197. package/rules/generated/auto_NO_uu.se_eea.json +0 -40
  198. package/rules/generated/auto_US_newsbreak.com_0.json +0 -28
  199. package/tests/generated/auto_AU_flysas.com_r1s.spec.ts +0 -2
  200. package/tests/generated/auto_CH_airbnb.de_83z.spec.ts +0 -2
  201. package/tests/generated/auto_CH_autodoc.de_krd.spec.ts +0 -2
  202. package/tests/generated/auto_CH_svtplay.se_d39.spec.ts +0 -2
  203. package/tests/generated/auto_GB_parliamentlive.tv_r3v.spec.ts +0 -6
  204. package/tests/generated/auto_NL_asnbank.nl_e28.spec.ts +0 -6
  205. package/tests/generated/auto_NL_eurojackpot.nederlandseloterij.nl_7hr.spec.ts +0 -6
  206. package/tests/generated/auto_NL_lotto.nederlandseloterij.nl_7c8.spec.ts +0 -6
  207. package/tests/generated/auto_NL_nederlandseloterij.nl_b60.spec.ts +0 -6
  208. package/tests/generated/auto_NL_staatsloterij.nederlandseloterij.nl_197.spec.ts +0 -6
  209. package/tests/generated/auto_NO_arronet.se_vo5.spec.ts +0 -2
  210. package/tests/generated/auto_NO_liseberg.se_qy4.spec.ts +0 -2
  211. package/tests/generated/auto_NO_nordnet.se_2en.spec.ts +0 -2
  212. package/tests/generated/auto_NO_regeringen.se_uhv.spec.ts +0 -6
  213. package/tests/generated/auto_NO_svtplay.se_ujn.spec.ts +0 -2
  214. package/tests/generated/auto_NO_uu.se_eea.spec.ts +0 -2
  215. package/tests/generated/auto_US_newsbreak.com_0.spec.ts +0 -2
  216. package/tests/twitter.spec.ts +0 -10
package/lib/heuristics.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { checkHeuristicPatterns, classifyButtons, classifyPopup, isExcludedPopup } from './heuristic-classify';
1
+ import { classifyButtons, classifyPopup, hasHeuristicPatterns, isExcludedPopup } from './heuristic-classify';
2
2
  import { ButtonData, HeuristicLevel, PopupClassification, PopupData } from './types';
3
3
  import { isElementVisible, isTopFrame } from './utils';
4
4
 
@@ -8,69 +8,191 @@ const POPUP_SEARCH_MAX_TIME = 100;
8
8
  export function getActionablePopups(mode: HeuristicLevel = 'reject', timeout = POPUP_SEARCH_MAX_TIME): PopupData[] {
9
9
  const acceptedLevels: PopupClassification[] =
10
10
  mode === 'tier2' ? ['reject', 'tier1', 'tier2'] : mode === 'tier1' ? ['reject', 'tier1'] : ['reject'];
11
- if (acceptedLevels.length === 0) {
12
- return [];
13
- }
14
- const popups = getPotentialPopups(timeout);
15
- const result = popups.reduce((acc, popup) => {
11
+ const popups = getPotentialPopups(timeout).filter((popup) => {
16
12
  const popupText = popup.text?.trim();
17
- if (popupText) {
18
- if (isExcludedPopup(popupText)) {
19
- return acc;
20
- }
21
- const { patterns } = checkHeuristicPatterns(popupText);
22
- if (patterns.length > 0) {
23
- classifyButtons(popup.buttons);
24
- popup.regexClassification = classifyPopup(popup.buttons);
25
- acc.push({
26
- ...popup,
27
- });
28
- }
13
+ if (!popupText || isExcludedPopup(popupText) || !hasHeuristicPatterns(popupText)) {
14
+ return false;
29
15
  }
30
- return acc;
31
- }, [] as PopupData[]);
32
- // popups filtered by mode and sorted so a popup with reject will always win.
33
- return result
34
- .filter((popup) => popup.regexClassification !== undefined && acceptedLevels.includes(popup.regexClassification))
35
- .sort((a, b) => ((a.regexClassification ?? '') > (b.regexClassification ?? '') ? 1 : -1));
16
+ popup.regexClassification = classifyPopup(popup.buttons);
17
+ return popup.regexClassification !== undefined && acceptedLevels.includes(popup.regexClassification);
18
+ });
19
+ // a popup with reject will always win
20
+ return popups.sort((a, b) => ((a.regexClassification ?? '') > (b.regexClassification ?? '') ? 1 : -1));
36
21
  }
37
22
 
23
+ /**
24
+ * @param timeout time budget for the whole search, in ms
25
+ */
38
26
  export function getPotentialPopups(timeout = POPUP_SEARCH_MAX_TIME): PopupData[] {
39
- const isFramed = !isTopFrame();
40
- // do not inspect frames that are more than one level deep
41
- if (isFramed && window.parent && window.parent !== window.top) {
42
- return [];
27
+ if (!isTopFrame()) {
28
+ // do not inspect frames that are more than one level deep
29
+ if (window.parent && window.parent !== window.top) {
30
+ return [];
31
+ }
32
+ // for iframes, just take the whole document
33
+ const frameElement = getFramePopupElement(document);
34
+ const popup = frameElement && toPopupData(frameElement);
35
+ return popup ? [popup] : [];
36
+ }
37
+ const deadline = performance.now() + timeout;
38
+ // fixed, sticky and dialog-like elements, with fragments expanded to their popup root
39
+ const { elements, shadowHosts } = getPopupLikeElements(deadline);
40
+ const potentialPopups: PopupData[] = [];
41
+ const expanded: HTMLElement[] = [];
42
+ for (const el of elements) {
43
+ const popup = toPopupData(el, shadowHosts);
44
+ if (!popup) {
45
+ continue;
46
+ }
47
+ // past the deadline, keep the candidates found so far as they are
48
+ const root = performance.now() > deadline ? popup : expandPopupFragment(popup, shadowHosts);
49
+ if (potentialPopups.some((p) => p.element === root.element)) {
50
+ continue;
51
+ }
52
+ if (root !== popup) {
53
+ expanded.push(root.element);
54
+ }
55
+ potentialPopups.push(root);
56
+ }
57
+ // a fragment expanded to its popup root replaces any other candidates inside that root
58
+ return potentialPopups.filter((p) => !expanded.some((root) => root !== p.element && composedContains(root, p.element)));
59
+ }
60
+
61
+ function toPopupData(el: HTMLElement, shadowHosts?: HTMLElement[], text = getElementText(el)): PopupData | null {
62
+ if (!text) {
63
+ return null;
43
64
  }
65
+ const buttons = getButtonData(el, shadowHosts);
66
+ classifyButtons(buttons);
67
+ return { text, element: el, buttons };
68
+ }
44
69
 
45
- return collectPotentialPopups(isFramed, timeout);
70
+ // buttons beyond their labels, and at least one button the heuristics recognise (not just policy links)
71
+ function hasPopupButtons(text: string, buttons: ButtonData[]): boolean {
72
+ return !isOnlyButtonText(text, buttons) && buttons.some((b) => b.regexClassification !== 'other');
46
73
  }
47
74
 
48
- function collectPotentialPopups(isFramed: boolean, timeout = POPUP_SEARCH_MAX_TIME): PopupData[] {
49
- let elements = [];
50
- if (!isFramed) {
51
- elements = getPopupLikeElements(timeout);
52
- } else {
53
- // for iframes, just take the whole document
54
- const doc = document.body || document.documentElement;
55
- if (doc && isElementVisible(doc) && doc.innerText) {
56
- elements.push(doc);
75
+ // detect patterns include button wording, so a bare button bar matches; expand it so exclusions and crawl text see the whole popup
76
+ function isOnlyButtonText(text: string, buttons: ButtonData[]): boolean {
77
+ let rest = text;
78
+ for (const button of buttons) {
79
+ rest = rest.replace(button.text, '');
80
+ }
81
+ return rest.replace(/\s+/g, '').length < 10;
82
+ }
83
+
84
+ // Bounds how far expansion climbs. The stop at page-level containers is the main guard against false positives;
85
+ // this limit only keeps the search cheap on deep trees.
86
+ const MAX_FRAGMENT_EXPANSION_DEPTH = 6;
87
+
88
+ /**
89
+ * A positioned candidate is often only part of a popup, such as a sticky button bar or heading.
90
+ * If it does not look like a consent popup on its own, use the nearest ancestor that does.
91
+ */
92
+ function expandPopupFragment(popup: PopupData, shadowHosts?: HTMLElement[]): PopupData {
93
+ if (hasHeuristicPatterns(popup.text) && hasPopupButtons(popup.text, popup.buttons)) {
94
+ return popup;
95
+ }
96
+ let el = getComposedParent(popup.element);
97
+ for (let depth = 0; el && depth < MAX_FRAGMENT_EXPANSION_DEPTH; depth++, el = getComposedParent(el)) {
98
+ // stop before reaching page-level containers, whose text includes unrelated page copy
99
+ if (el === document.body || el === document.documentElement || el.getBoundingClientRect().height > window.innerHeight) {
100
+ break;
101
+ }
102
+ // check the text first: it is cheaper than collecting buttons
103
+ const text = getElementText(el);
104
+ if (!text || !hasHeuristicPatterns(text)) {
105
+ continue;
106
+ }
107
+ const root = toPopupData(el, shadowHosts, text);
108
+ if (root && hasPopupButtons(text, root.buttons)) {
109
+ return root;
57
110
  }
58
111
  }
112
+ return popup;
113
+ }
59
114
 
60
- const potentialPopups: PopupData[] = [];
115
+ function getComposedParent(el: HTMLElement): HTMLElement | null {
116
+ if (el.parentElement) {
117
+ return el.parentElement;
118
+ }
119
+ const parent = el.parentNode;
120
+ return parent instanceof ShadowRoot && parent.host instanceof HTMLElement ? parent.host : null;
121
+ }
61
122
 
62
- // for each potential popup, get the buttons
63
- for (const el of elements) {
64
- if (el.innerText) {
65
- potentialPopups.push({
66
- text: el.innerText,
67
- element: el,
68
- buttons: getButtonData(el),
69
- });
123
+ /**
124
+ * The whole document of a frame, if it is rendered and has text. The body can have zero height when all of its
125
+ * content is positioned, so only check visibility styles, not size.
126
+ */
127
+ export function getFramePopupElement(doc: Document): HTMLElement | null {
128
+ const el = doc.body || doc.documentElement;
129
+ const win = doc.defaultView;
130
+ if (!el || !win || win.innerWidth === 0 || win.innerHeight === 0 || !isStyleVisible(el)) {
131
+ return null;
132
+ }
133
+ return getElementText(el).trim() ? el : null;
134
+ }
135
+
136
+ // catches elements hidden by an ancestor's visibility; checkVisibility is not available in older browsers.
137
+ // Opacity is not checked: prehide hides popups with opacity: 0, and they must stay detectable.
138
+ function isStyleVisible(el: HTMLElement): boolean {
139
+ return typeof el.checkVisibility !== 'function' || el.checkVisibility({ checkVisibilityCSS: true });
140
+ }
141
+
142
+ /**
143
+ * Rendered text of an element, including the content of its own open shadow root.
144
+ */
145
+ function getElementText(el: HTMLElement): string {
146
+ const text = el.innerText || '';
147
+ if (!el.shadowRoot) {
148
+ return text;
149
+ }
150
+ // innerText of a shadow host does not include its shadow content
151
+ const shadowTexts = Array.from(el.shadowRoot.children, (child) => (isRenderedTextElement(child) ? getElementText(child) : ''));
152
+ return [text, ...shadowTexts].filter((t) => t.trim()).join('\n');
153
+ }
154
+
155
+ // innerText of an element that is not rendered falls back to textContent, which would leak hidden text
156
+ function isRenderedTextElement(el: Element): el is HTMLElement {
157
+ return (
158
+ el instanceof HTMLElement &&
159
+ !['STYLE', 'SCRIPT', 'TEMPLATE', 'NOSCRIPT', 'LINK', 'SLOT'].includes(el.tagName) &&
160
+ window.getComputedStyle(el).display !== 'none'
161
+ );
162
+ }
163
+
164
+ /**
165
+ * Like Node.contains, but also looks through shadow root boundaries.
166
+ */
167
+ function composedContains(container: Node, node: Node): boolean {
168
+ if (container.contains(node)) {
169
+ return true;
170
+ }
171
+ let root = node.getRootNode();
172
+ while (root instanceof ShadowRoot) {
173
+ if (container.contains(root.host)) {
174
+ return true;
70
175
  }
176
+ root = root.host.getRootNode();
71
177
  }
178
+ return false;
179
+ }
180
+
181
+ /**
182
+ * Stricter visibility check for popup candidates: catches panels hidden by an ancestor's visibility,
183
+ * and elements outside the viewport, which isElementVisible lets through.
184
+ */
185
+ function isPopupCandidateVisible(el: HTMLElement): boolean {
186
+ return isElementVisible(el) && isStyleVisible(el) && isOnScreen(el);
187
+ }
72
188
 
73
- return potentialPopups;
189
+ // A zero-size wrapper can still show its content: then any child that is on screen counts, unless the wrapper clips it.
190
+ function isOnScreen(el: Element): boolean {
191
+ const rect = el.getBoundingClientRect();
192
+ if (rect.width === 0 || rect.height === 0) {
193
+ return window.getComputedStyle(el).overflow === 'visible' && Array.from(el.children).some((child) => isOnScreen(child));
194
+ }
195
+ return rect.bottom > 0 && rect.right > 0 && rect.top < window.innerHeight && rect.left < window.innerWidth;
74
196
  }
75
197
 
76
198
  export function isDialogLikeElement(node: HTMLElement): boolean {
@@ -85,64 +207,99 @@ export function isDialogLikeElement(node: HTMLElement): boolean {
85
207
 
86
208
  /**
87
209
  * Heuristic to get all elements that look like "popups"
88
- * TODO: this heuristic is too strict, not all popups are actually sticky/fixed
210
+ * TODO: popups in the normal page flow are only found when part of them is fixed, sticky or dialog-like
89
211
  */
90
- function getPopupLikeElements(timeout = POPUP_SEARCH_MAX_TIME): HTMLElement[] {
91
- const startTime = performance.now();
92
- const walker = document.createTreeWalker(
93
- document.documentElement,
94
- NodeFilter.SHOW_ELEMENT, // visit only element nodes
95
- {
96
- acceptNode(node: HTMLElement) {
97
- if (node.tagName === 'BODY') {
98
- return NodeFilter.FILTER_SKIP;
99
- }
100
- if (isElementVisible(node)) {
101
- const cssPosition = window.getComputedStyle(node).position;
102
- if (cssPosition === 'fixed' || cssPosition === 'sticky') {
103
- return NodeFilter.FILTER_ACCEPT;
212
+ function getPopupLikeElements(deadline: number): { elements: HTMLElement[]; shadowHosts: HTMLElement[] } {
213
+ const found: HTMLElement[] = [];
214
+ const shadowHosts: HTMLElement[] = [];
215
+ // open shadow roots are walked after the tree that hosts them
216
+ const roots: Node[] = [document.documentElement];
217
+ for (let i = 0; i < roots.length; i++) {
218
+ if (i > 0 && performance.now() > deadline) {
219
+ break;
220
+ }
221
+ const walker = document.createTreeWalker(
222
+ roots[i],
223
+ NodeFilter.SHOW_ELEMENT, // visit only element nodes
224
+ {
225
+ acceptNode(node: HTMLElement) {
226
+ if (node.shadowRoot) {
227
+ shadowHosts.push(node);
228
+ roots.push(node.shadowRoot);
229
+ }
230
+ if (node.tagName === 'BODY') {
231
+ return NodeFilter.FILTER_SKIP;
104
232
  }
105
- if (isDialogLikeElement(node)) {
233
+ const style = window.getComputedStyle(node);
234
+ // nothing under display: none can be visible, and styling hidden subtrees is slow
235
+ if (style.display === 'none') {
236
+ return NodeFilter.FILTER_REJECT;
237
+ }
238
+ const cssPosition = style.position;
239
+ if (
240
+ (cssPosition === 'fixed' || cssPosition === 'sticky' || isDialogLikeElement(node)) &&
241
+ isPopupCandidateVisible(node)
242
+ ) {
106
243
  return NodeFilter.FILTER_ACCEPT;
107
244
  }
108
- }
109
- // start rejecting after POPUP_SEARCH_MAX_TIME to avoid blocking the main thread
110
- if (performance.now() - startTime > timeout) {
111
- return NodeFilter.FILTER_REJECT;
112
- }
113
- return NodeFilter.FILTER_SKIP;
245
+ // start rejecting after the deadline to avoid blocking the main thread
246
+ if (performance.now() > deadline) {
247
+ return NodeFilter.FILTER_REJECT;
248
+ }
249
+ return NodeFilter.FILTER_SKIP;
250
+ },
114
251
  },
115
- },
116
- );
117
- const found = [];
118
- for (let node = walker.nextNode(); node; node = walker.nextNode()) {
119
- found.push(node as HTMLElement);
252
+ );
253
+ for (let node = walker.nextNode(); node; node = walker.nextNode()) {
254
+ found.push(node as HTMLElement);
255
+ }
120
256
  }
121
257
  // Empty positioned children should not displace a contentful popup container.
122
- return excludeContainers(found.filter((element) => element.innerText?.trim()));
258
+ return { elements: excludeContainers(found.filter((element) => getElementText(element).trim())), shadowHosts };
123
259
  }
124
260
 
125
261
  /**
126
262
  * Serialize all actionable buttons on the page
263
+ * @param shadowHosts open shadow hosts on the page, if already known; otherwise they are searched for under el
127
264
  */
128
- export function getButtonData(el: HTMLElement): ButtonData[] {
129
- const actionableButtons = excludeContainers(getButtonLikeElements(el)).filter(
130
- (b) =>
131
- isElementVisible(b) &&
132
- !isDisabled(b) &&
133
- (b.innerText?.trim() ||
134
- // <input> values do not appear in innerText
135
- (b instanceof HTMLInputElement && ['submit', 'button'].includes(b.type) && b.value?.trim())),
136
- );
265
+ export function getButtonData(el: HTMLElement, shadowHosts?: HTMLElement[]): ButtonData[] {
266
+ const buttons: ButtonData[] = [];
267
+ for (const b of excludeContainers(getButtonLikeElements(el, shadowHosts))) {
268
+ if (!isElementVisible(b) || isDisabled(b)) {
269
+ continue;
270
+ }
271
+ const text =
272
+ getElementText(b).trim() ||
273
+ // <input> values do not appear in innerText
274
+ (b instanceof HTMLInputElement && ['submit', 'button'].includes(b.type) ? b.value?.trim() : '');
275
+ if (text) {
276
+ buttons.push({ text, element: b });
277
+ }
278
+ }
279
+ return buttons;
280
+ }
137
281
 
138
- return actionableButtons.map((b) => ({
139
- text: (b.innerText || b.textContent || '').trim() || (b as HTMLInputElement).value?.trim() || '',
140
- element: b,
141
- }));
282
+ function getButtonLikeElements(el: HTMLElement, shadowHosts = findShadowHosts(el)): HTMLElement[] {
283
+ const buttons = Array.from(el.querySelectorAll<HTMLElement>(BUTTON_LIKE_ELEMENT_SELECTOR));
284
+ for (const host of shadowHosts) {
285
+ if (composedContains(el, host)) {
286
+ buttons.push(...host.shadowRoot!.querySelectorAll<HTMLElement>(BUTTON_LIKE_ELEMENT_SELECTOR));
287
+ }
288
+ }
289
+ return buttons;
142
290
  }
143
291
 
144
- function getButtonLikeElements(el: HTMLElement): HTMLElement[] {
145
- return Array.from(el.querySelectorAll(BUTTON_LIKE_ELEMENT_SELECTOR));
292
+ // all open shadow hosts at or under root, including those nested in other shadow roots
293
+ function findShadowHosts(root: HTMLElement | ShadowRoot): HTMLElement[] {
294
+ const hosts: HTMLElement[] = [];
295
+ const candidates =
296
+ root instanceof HTMLElement ? [root, ...root.querySelectorAll<HTMLElement>('*')] : root.querySelectorAll<HTMLElement>('*');
297
+ for (const el of candidates) {
298
+ if (el.shadowRoot) {
299
+ hosts.push(el, ...findShadowHosts(el.shadowRoot));
300
+ }
301
+ }
302
+ return hosts;
146
303
  }
147
304
 
148
305
  export function isDisabled(el: HTMLElement): boolean {
@@ -154,20 +311,15 @@ export function isDisabled(el: HTMLElement): boolean {
154
311
  * Leave only elements that do not contain any other elements
155
312
  */
156
313
  export function excludeContainers(elements: HTMLElement[]): HTMLElement[] {
157
- const results = [];
158
- if (elements.length > 0) {
159
- for (let i = elements.length - 1; i >= 0; i--) {
160
- let container = false;
161
- for (let j = 0; j < elements.length; j++) {
162
- if (i !== j && elements[i].contains(elements[j])) {
163
- container = true;
164
- break;
165
- }
166
- }
167
- if (!container) {
168
- results.push(elements[i]);
314
+ // mark every element that is an ancestor of another one, instead of comparing all pairs
315
+ const candidates = new Set<Node>(elements);
316
+ const containers = new Set<Node>();
317
+ for (const el of elements) {
318
+ for (let node = el.parentNode; node; node = node.parentNode || (node instanceof ShadowRoot ? node.host : null)) {
319
+ if (candidates.has(node)) {
320
+ containers.add(node);
169
321
  }
170
322
  }
171
323
  }
172
- return results;
324
+ return elements.filter((el) => !containers.has(el)).reverse();
173
325
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@duckduckgo/autoconsent",
3
- "version": "16.59.0",
3
+ "version": "16.61.0",
4
4
  "description": "",
5
5
  "types": "./dist/types/web.d.ts",
6
6
  "exports": {
@@ -0,0 +1,217 @@
1
+ /**
2
+ * Runs the autoconsent content script in an isolated world of every frame of a Chromium page (via CDP),
3
+ * the way the browser extension does, and bridges its messages to Node. Shared by the Playwright E2E
4
+ * runner and the regional-testing harness used by agent skills.
5
+ */
6
+
7
+ /**
8
+ * @typedef {import('playwright').Page} Page
9
+ */
10
+
11
+ /**
12
+ * Primitives the message handler uses to talk to a specific frame's content script.
13
+ * `frameRef` is the isolated world's execution-context uniqueId the message arrived from;
14
+ * the transport resolves it to the owning CDP session and the matching page-world context.
15
+ * @typedef {Object} MessageTransport
16
+ * @property {(frameRef: string, message: object) => Promise<void>} sendToContentScript
17
+ * @property {(frameRef: string, code: string) => Promise<any>} evalInMainWorld
18
+ * @property {(frameRef: string) => boolean} isMainFrame - Whether the frame is the page's top frame.
19
+ */
20
+
21
+ /**
22
+ * @typedef {(transport: MessageTransport) => (msg: any, frameRef: string) => Promise<void>} MessageHandlerFactory
23
+ */
24
+
25
+ /**
26
+ * Isolated-world injection via CDP (Chromium only). For each frame we create a dedicated isolated
27
+ * world via `Page.createIsolatedWorld`, then inject the content script there and bridge messages
28
+ * over a per-context CDP binding.
29
+ *
30
+ * @param {Page} page
31
+ * @param {MessageHandlerFactory} createMessageHandler
32
+ * @param {string} contentScript - The autoconsent content script bundle.
33
+ * @param {string} [extraScript] - Runs in each isolated world after the content script.
34
+ */
35
+ export async function injectIntoIsolatedWorld(page, createMessageHandler, contentScript, extraScript = '') {
36
+ // Isolated world name: `<prefix><pageWorldUniqueId><separator><frameId>`. Encoding the page-world
37
+ // uniqueId lets us later run eval snippets in that frame's main world.
38
+ const WORLD_PREFIX = 'autoconsent_iw_';
39
+ const WORLD_SEPARATOR = '_frame_';
40
+ const BINDING_PREFIX = 'autoconsentSendMessage_';
41
+
42
+ /** @type {Map<string, string>} isolated-world uniqueId -> page (main) world uniqueId */
43
+ const isolated2pageWorld = new Map();
44
+ /** @type {Map<string, any>} isolated-world uniqueId -> CDP session that owns it */
45
+ const sessionByContext = new Map();
46
+ /** @type {Map<string, string>} binding name -> isolated-world uniqueId */
47
+ const contextByBinding = new Map();
48
+ /** @type {Map<string, string>} isolated-world uniqueId -> binding name */
49
+ const bindingByContext = new Map();
50
+ /** @type {Map<string, string>} isolated-world uniqueId -> CDP frame id */
51
+ const frameByContext = new Map();
52
+ /** @type {Map<string, string>} CDP frame id -> uniqueId of the frame's current page (main) world */
53
+ const pageWorldByFrame = new Map();
54
+ /** @type {any} */
55
+ let pageSession = null;
56
+ /** @type {string|null} */
57
+ let mainFrameId = null;
58
+
59
+ const handle = createMessageHandler({
60
+ sendToContentScript: async (isolatedUniqueId, message) => {
61
+ const client = sessionByContext.get(isolatedUniqueId);
62
+ if (!client) return;
63
+ try {
64
+ await client.send('Runtime.evaluate', {
65
+ expression: `autoconsentReceiveMessage(${JSON.stringify(message)})`,
66
+ uniqueContextId: isolatedUniqueId,
67
+ awaitPromise: true,
68
+ // Some pages' CSP would otherwise block evaluating in the isolated world.
69
+ allowUnsafeEvalBlockedByCSP: true,
70
+ });
71
+ } catch {
72
+ // The context may be gone if the frame navigated or detached.
73
+ }
74
+ },
75
+ evalInMainWorld: async (isolatedUniqueId, code) => {
76
+ const client = sessionByContext.get(isolatedUniqueId);
77
+ const pageWorldUniqueId = isolated2pageWorld.get(isolatedUniqueId);
78
+ if (!client || !pageWorldUniqueId) return false;
79
+ const { result, exceptionDetails } = await client.send('Runtime.evaluate', {
80
+ expression: code,
81
+ uniqueContextId: pageWorldUniqueId,
82
+ returnByValue: true,
83
+ awaitPromise: true,
84
+ // Eval snippets must run even when the page's CSP disallows eval.
85
+ allowUnsafeEvalBlockedByCSP: true,
86
+ });
87
+ return exceptionDetails ? false : (result?.value ?? false);
88
+ },
89
+ isMainFrame: (isolatedUniqueId) =>
90
+ sessionByContext.get(isolatedUniqueId) === pageSession && frameByContext.get(isolatedUniqueId) === mainFrameId,
91
+ });
92
+
93
+ async function attachToSession(/** @type {any} */ client) {
94
+ client.on('Runtime.executionContextCreated', async (/** @type {any} */ event) => {
95
+ const { context } = event;
96
+ const frameId = context.auxData?.frameId;
97
+
98
+ // Our isolated world finished initializing: wire up its binding and content script.
99
+ if (context.auxData?.type === 'isolated' && typeof context.name === 'string' && context.name.startsWith(WORLD_PREFIX)) {
100
+ const separatorIndex = context.name.indexOf(WORLD_SEPARATOR);
101
+ const pageWorldUniqueId = context.name.slice(WORLD_PREFIX.length, separatorIndex);
102
+ const intendedFrameId = context.name.slice(separatorIndex + WORLD_SEPARATOR.length);
103
+ // Chromium may create the named world in other frames too, and recreates it in the frame's
104
+ // next document; keep only the one we asked for in the current document.
105
+ if (intendedFrameId !== frameId || pageWorldByFrame.get(frameId) !== pageWorldUniqueId) return;
106
+
107
+ isolated2pageWorld.set(context.uniqueId, pageWorldUniqueId);
108
+ sessionByContext.set(context.uniqueId, client);
109
+ frameByContext.set(context.uniqueId, frameId);
110
+
111
+ const bindingName = `${BINDING_PREFIX}${context.uniqueId.replace(/\W/g, '_')}`;
112
+ contextByBinding.set(bindingName, context.uniqueId);
113
+ bindingByContext.set(context.uniqueId, bindingName);
114
+ try {
115
+ await client.send('Runtime.addBinding', { name: bindingName, executionContextName: context.name });
116
+ // CDP bindings take a single string arg, so wrap it in the shape the content script expects.
117
+ await client.send('Runtime.evaluate', {
118
+ expression: `window.autoconsentSendMessage = (m) => { window.${bindingName}(JSON.stringify(m)); return Promise.resolve(); };\n${contentScript}\n${extraScript}`,
119
+ uniqueContextId: context.uniqueId,
120
+ allowUnsafeEvalBlockedByCSP: true,
121
+ });
122
+ } catch {
123
+ // The frame may have navigated or detached before we finished wiring it up.
124
+ }
125
+ return;
126
+ }
127
+
128
+ // A regular page (main) world: request an isolated world for it, tagged with its id.
129
+ if (!frameId || context.auxData?.type !== 'default') return;
130
+ pageWorldByFrame.set(frameId, context.uniqueId);
131
+ if (!context.origin || context.origin === '://') return;
132
+ try {
133
+ await client.send('Page.createIsolatedWorld', {
134
+ frameId,
135
+ worldName: `${WORLD_PREFIX}${context.uniqueId}${WORLD_SEPARATOR}${frameId}`,
136
+ });
137
+ } catch {
138
+ // The frame may have navigated or detached.
139
+ }
140
+ });
141
+
142
+ client.on('Runtime.executionContextDestroyed', (/** @type {any} */ event) => {
143
+ const uniqueId = event.executionContextUniqueId;
144
+ if (!uniqueId) return;
145
+ isolated2pageWorld.delete(uniqueId);
146
+ sessionByContext.delete(uniqueId);
147
+ frameByContext.delete(uniqueId);
148
+ const bindingName = bindingByContext.get(uniqueId);
149
+ if (bindingName) {
150
+ bindingByContext.delete(uniqueId);
151
+ contextByBinding.delete(bindingName);
152
+ // Each document gets its own binding, so drop the old one with its context.
153
+ client.send('Runtime.removeBinding', { name: bindingName }).catch(() => {});
154
+ }
155
+ });
156
+
157
+ client.on('Runtime.bindingCalled', (/** @type {any} */ event) => {
158
+ const isolatedUniqueId = contextByBinding.get(event.name);
159
+ if (!isolatedUniqueId) return;
160
+ let msg;
161
+ try {
162
+ msg = JSON.parse(event.payload);
163
+ } catch {
164
+ return;
165
+ }
166
+ handle(msg, isolatedUniqueId);
167
+ });
168
+
169
+ // Page must be enabled before createIsolatedWorld; Runtime.enable replays existing contexts.
170
+ await client.send('Page.enable');
171
+ await client.send('Runtime.enable');
172
+ }
173
+
174
+ // The page session covers the main frame and all same-process (in-process) iframes.
175
+ pageSession = await page.context().newCDPSession(page);
176
+ const { frameTree } = await pageSession.send('Page.getFrameTree');
177
+ mainFrameId = frameTree.frame.id;
178
+ // Follow the top frame if Chromium replaces it (e.g. on prerender activation).
179
+ pageSession.on('Page.frameNavigated', (/** @type {any} */ event) => {
180
+ if (!event.frame.parentId) mainFrameId = event.frame.id;
181
+ });
182
+ await attachToSession(pageSession);
183
+
184
+ // Out-of-process iframes (OOPIFs) are separate CDP targets, so each needs its own session.
185
+ // newCDPSession throws for in-process frames, which are already handled above.
186
+ const attachedFrames = new WeakSet();
187
+ async function attachToOopif(/** @type {import('playwright').Frame} */ frame) {
188
+ if (!frame.parentFrame() || attachedFrames.has(frame)) return;
189
+ // Mark synchronously (before any await) so concurrent frameattached/framenavigated events
190
+ // can't open duplicate sessions for the same frame. Unmark on failure so a later event retries.
191
+ attachedFrames.add(frame);
192
+ let client;
193
+ try {
194
+ client = await page.context().newCDPSession(frame);
195
+ } catch {
196
+ // In-process frame (already covered by the page session) or the frame detached.
197
+ attachedFrames.delete(frame);
198
+ return;
199
+ }
200
+ try {
201
+ await attachToSession(client);
202
+ } catch {
203
+ // Wiring up the session failed (e.g. the OOPIF navigated/detached). Detach the
204
+ // half-initialized session before unmarking, otherwise a retry would open a second
205
+ // session for the same frame, leaving duplicate listeners and possible double injection.
206
+ try {
207
+ await client.detach();
208
+ } catch {
209
+ // The session may already be gone.
210
+ }
211
+ attachedFrames.delete(frame);
212
+ }
213
+ }
214
+ page.on('frameattached', attachToOopif);
215
+ page.on('framenavigated', attachToOopif);
216
+ await Promise.all(page.frames().map(attachToOopif));
217
+ }