fetch_util 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +37 -0
- data/Rakefile +5 -0
- data/lib/fetch_util/assets/extract.js +1 -1
- data/lib/fetch_util/browser/interaction_helpers/consent_helpers.rb +187 -104
- data/lib/fetch_util/browser/interaction_helpers/dom_interaction.rb +14 -2
- data/lib/fetch_util/browser/navigation/headers_and_readiness.rb +0 -7
- data/lib/fetch_util/browser/navigation/navigator_patch.rb +3 -1
- data/lib/fetch_util/browser/site_stabilization/community_and_marketplace.rb +28 -27
- data/lib/fetch_util/browser/site_stabilization/gitlab_repo.rb +32 -0
- data/lib/fetch_util/browser/site_stabilization/social_platforms.rb +25 -32
- data/lib/fetch_util/browser/site_stabilization.rb +2 -0
- data/lib/fetch_util/browser/stabilization/page_flow.rb +93 -10
- data/lib/fetch_util/browser.rb +55 -16
- data/lib/fetch_util/cli.rb +1 -0
- data/lib/fetch_util/extractor.rb +27 -3
- data/lib/fetch_util/fetcher.rb +696 -67
- data/lib/fetch_util/parallel_fetcher.rb +15 -20
- data/lib/fetch_util/raw_docs_fallback.rb +48 -24
- data/lib/fetch_util/regulatory/http_client.rb +111 -43
- data/lib/fetch_util/result.rb +100 -3
- data/lib/fetch_util/version.rb +1 -1
- data/lib/fetch_util.rb +12 -2
- metadata +3 -2
|
@@ -5,42 +5,172 @@ module FetchUtil
|
|
|
5
5
|
module InteractionHelpers
|
|
6
6
|
# rubocop:disable Metrics/ModuleLength
|
|
7
7
|
module ConsentHelpers
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
8
|
+
ConsentConfig = Struct.new(
|
|
9
|
+
:accept_labels,
|
|
10
|
+
:fallback_labels,
|
|
11
|
+
:reject_pattern,
|
|
12
|
+
:accept_pattern,
|
|
13
|
+
:close_pattern,
|
|
14
|
+
:context_pattern,
|
|
15
|
+
:container_pattern,
|
|
16
|
+
:known_cmp_selectors,
|
|
17
|
+
:quick_indicator_selectors,
|
|
18
|
+
:container_selectors,
|
|
19
|
+
:overlay_selectors,
|
|
20
|
+
:button_selectors,
|
|
21
|
+
keyword_init: true
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
# rubocop:disable Layout/LineLength, Style/RedundantPercentQ
|
|
25
|
+
DEFAULT_CONSENT_CONFIG = ConsentConfig.new(
|
|
26
|
+
accept_labels: [
|
|
27
|
+
"Accept All", "Accept all", "Accept", "Agree", "I agree", "Got it", "OK", "I understand", "Allow all", "Consent",
|
|
28
|
+
"Confirm My Choices", "Save preferences", "Customize Choices",
|
|
29
|
+
"Alle akzeptieren", "Alles akzeptieren", "Alle zulassen", "Akzeptieren", "Zustimmen",
|
|
30
|
+
"Tout accepter", "Accepter tout", "Autoriser tout", "Accepter", "J'accepte",
|
|
31
|
+
"Aceptar todo", "Aceptar todas", "Aceptar", "Estoy de acuerdo",
|
|
32
|
+
"Accetta tutto", "Accetta tutti", "Accetta", "Accetto",
|
|
33
|
+
"Aceitar tudo", "Aceitar todos", "Aceitar todos os cookies", "Aceitar", "Concordo",
|
|
34
|
+
"Alles accepteren", "Accepteer alles",
|
|
35
|
+
"Godta alle", "Aksepter alle", "Godkjenn alle",
|
|
36
|
+
"Acceptera alla", "Godkänn alla", "Tillåt alla",
|
|
37
|
+
"Accepter alle", "Tillad alle",
|
|
38
|
+
"Hyväksy kaikki", "Salli kaikki",
|
|
39
|
+
"Priimti visus", "Leisti visus",
|
|
40
|
+
"Pieņemt visus", "Atļaut visus",
|
|
41
|
+
"Přijmout vše", "Přijmout", "Souhlasím", "Povolit vše",
|
|
42
|
+
"Akceptuj wszystkie", "Zaakceptuj wszystkie", "Akceptuj", "Akceptuję", "Zgadzam się",
|
|
43
|
+
"Mindent elfogadok", "Összes elfogadása", "Elfogad", "Elfogadom", "Egyetértek",
|
|
44
|
+
"Acceptă tot", "Acceptă toate",
|
|
45
|
+
"Прифати сите", "Прихвати све",
|
|
46
|
+
"Tümünü kabul et", "Kabul et", "Kabul ediyorum",
|
|
47
|
+
"Принять все", "Принять", "Согласен",
|
|
48
|
+
"Приеми всички", "Приеми", "Съгласен съм",
|
|
49
|
+
"Terima semua", "Terima", "Setuju",
|
|
50
|
+
"Chấp nhận tất cả", "Chấp nhận", "Đồng ý",
|
|
51
|
+
"ยอมรับทั้งหมด", "ยอมรับ", "ตกลง",
|
|
52
|
+
"すべて受け入れる", "すべて許可", "許可", "同意する",
|
|
53
|
+
"모두 동의", "모두 허용", "동의", "수락",
|
|
54
|
+
"قبول الكل", "موافق", "全部接受"
|
|
55
|
+
].freeze,
|
|
56
|
+
fallback_labels: [
|
|
57
|
+
"Essential only", "Reject All", "Reject all", "Reject", "Close", "Dismiss",
|
|
58
|
+
"Alle ablehnen", "Ablehnen", "Schließen", "Tout refuser", "Refuser", "Fermer", "Rechazar todo", "Rechazar", "Cerrar",
|
|
59
|
+
"Rifiuta tutto", "Rifiuta tutti", "Rifiuta", "Chiudi", "Rejeitar tudo", "Rejeitar", "Recusar tudo", "Recusar", "Fechar", "Alles weigeren",
|
|
60
|
+
"Avvis alle", "Avvisa alla", "Afvis alle",
|
|
61
|
+
"Hylkää kaikki", "Atmesti visus", "Noraidīt visus",
|
|
62
|
+
"Odmítnout vše", "Odmítnout", "Zavřít", "Odrzuć wszystkie", "Odrzuć", "Zamknij",
|
|
63
|
+
"Összes elutasítása", "Elutasít", "Bezár", "Respinge tot",
|
|
64
|
+
"Одбиј ги сите", "Одбиј све", "Reddet", "Kapat",
|
|
65
|
+
"Отклонить все", "Отклонить", "Закрыть", "Отказ", "Затвори",
|
|
66
|
+
"Tolak", "Tutup", "Từ chối", "Đóng", "ปฏิเสธ", "ปิด", "拒否", "閉じる", "거부", "닫기", "رفض", "إغلاق"
|
|
67
|
+
].freeze,
|
|
68
|
+
reject_pattern: %q{^(reject(?: all)?|reject optional(?: cookies)?|decline|deny|essential only|necessary only|close|dismiss|alle ablehnen|ablehnen|schließen|tout refuser|refuser|fermer|rechazar(?: todo)?|cerrar|recusar(?: tudo)?|rejeitar(?: tudo)?|fechar|rifiuta(?: tutti| tutto)?|chiudi|odrzuć(?: wszystkie)?|zamknij|odmítnout(?: vše)?|zavřít|reddet|kapat|отклонить(?: все)?|закрыть|отказ|затвори|elutasít|bezár|tolak|tutup|từ chối|đóng|ปฏิเสธ|ปิด|拒否|閉じる|거부|닫기|رفض|إغلاق)$},
|
|
69
|
+
accept_pattern: %q{^(accept(?: all(?: cookies?)?)?|allow all(?: cookies)?|allow cookies|agree(?: to cookies| and continue| & continue)?|i agree|ok(?:ay)?|accept & continue|continue with cookies|consent|got it|i understand|continue|accept and close|accept recommended settings|przejdź do serwisu|zaakceptuj(?:\s+(?:wszystkie|wszystko))?|akceptuj(?: wszystkie)?|akceptuję|zgadzam się|zgoda|akzeptieren|alle akzeptieren|zustimmen|accepter (?:tout|les cookies|et continuer)|tout accepter|j'accepte|accepter|aceptar (?:todo|todas|cookies)|aceptar|estoy de acuerdo|acepto|accetta (?:tutto|tutti)|accetta|accetto|aceitar(?:\s+(?:tudo|todos|cookies|todos os cookies))?|aceitar cookies|aceitar todos os cookies|aceitar e continuar|aceitar e fechar|concordo|aceito|accetta e continua|akkoord|ga akkoord|alles accepteren|accepteer alles|すべて受け入れる|すべて許可|許可|同意する|同意して閉じる|쿠키 허용|모두 동의|모두 허용|동의하고 계속|동의|수락|接受全部|全部接受|同意并继续|接受并继续|принять все|принять|согласен|согласиться|приеми всички|приеми|съгласен съм|souhlasím|přijmout vše|přijmout(?:\s+(?:všechny|vše))?|povolit vše|souhlasit|godkänn alla|acceptera alla|tillåt alla|jag godkänner|godkänn|accepter alle|tillad alle|jeg accepterer|acceptér|godta alle|tanggap semua|setuju|terima semua|godkjenn alle|aksepter alle|aksepter|jeg godtar|godta|accepta-ho tot|accepta|accepto|d'acord|razumem|prihvatam|прихватам|прихвати све|tümünü kabul et|kabul et|kabul ediyorum|qəbul edirəm|ყველას მიღება|සියල්ල පිළිගන්න|පිළිගන්න|همه را بپذیرید|ተቀበል|ሁሉንም ተቀበል|allow all|confirm my choices|pokračovat|hyväksy(?:\s+kaikki)?|salli kaikki|salli evästeet|hyväksyn|priimti visus|sutinku|leisti visus|прифати(?:\s+(?:ги\s+)?сите)?|се согласувам|acceptă(?:\s+tot(?:ul)?)?|accept toate|acceptă toate|sunt de acord|pieņemt(?:\s+visus)?|piekrītu|atļaut(?:\s+visus)?|apstiprināt|elfogad(?:om)?|mindent elfogad(?:ok)?|összes elfogadása|elfogadom az összeset|egyetértek|hozzájárulok|chấp nhận tất cả|chấp nhận|đồng ý|ยอมรับทั้งหมด|ยอมรับ|ตกลง|قبول الكل|موافق)(?:\s+and\s+.*)?$},
|
|
70
|
+
close_pattern: %q{^(close|dismiss|ok(?:ay)?|got it|i understand|schließen|fermer|cerrar|fechar|chiudi|zamknij|zavřít|kapat|закрыть|затвори|bezár|tutup|đóng|ปิด|閉じる|닫기|إغلاق)$},
|
|
71
|
+
context_pattern: %q{(cookie|cookies|privacy|consent|gdpr|ccpa|cmp|onetrust|cookiebot|didomi|quantcast|before you continue|we use cookies and data|device identifiers|personalized ads|personalized content|trusted third party partners?|privacy preference center|your privacy settings|your privacy choices|manage privacy preferences|manage consent preferences|cookie information|cookie list|cookies details|list of partners(?: \(vendors\))?|configurações avançadas de cookies|declaração de cookies|gerenciar cookies|utilizamos cookies|dados pessoais|pliki cookie|datenschutz|données personnelles|datos personales|dati personali|wish to store|access information on your devices|preferenze cookie|クッキー|Cookieプリファレンス|Cookie設定|同意設定|쿠키|동의|개인정보|接受|隐私设置|cookie 偏好设置|файлы cookie|настройки cookie|souhlas|personalizac|soukromí|nastavení souhlasu|kakor|sekretess|samtycke|cookies og data|privatlivs|samtykke|privatliv|personvern|informasjonskapsler|informasjonskapslar|aller media|dine data|galetes|protecció de dades|política de privadesa|ግላዊነት|ኩኪ|ኩኪዎች|pro pokračování vyberte|technické cookies|jakou formou vám máme zobrazovat obsah|evästeet|evästeasetukset|tietosuoja|yksityisyys|hyväksy evästeet|slapukai|privatumas|slapukų nustatymai|kolačinji|приватност|поставки за колачиња|cookie-uri|confidențialitate|setări cookie|protecția datelor|sīkdatnes|sīkfailus|privātums|privātuma iestatījumi|sīkdatņu iestatījumi|sütiket|adatvédelem|adatvédelmi beállítások|süti beállítások|kabul|çerez|gizlilik|лични данни|поверителност|terima|setuju|privasi|ยินยอม|คุกกี้|ความเป็นส่วนตัว)},
|
|
72
|
+
container_pattern: %q{(cookie|consent|privacy|onetrust|cookiebot|usercentrics|trustarc|didomi|quantcast|sourcepoint|sp_message|uniconsent|osano|gdpr|ccpa)},
|
|
73
|
+
known_cmp_selectors: [
|
|
74
|
+
"#onetrust-banner-sdk", "#onetrust-pc-sdk", ".qc-cmp2-container", ".qc-cmp2-summary",
|
|
75
|
+
"#CybotCookiebotDialog", ".cc-window", ".cc_banner", "#cookie-banner", ".cookie-banner",
|
|
76
|
+
"#consent-banner", ".consent-banner", ".fc-consent-root", ".cmp-modal", ".gdpr-banner",
|
|
77
|
+
"#gdpr-consent", ".js-cookies", ".cookie-notice", "#cookieNotice",
|
|
78
|
+
"[id*='sourcepoint']", "[class*='sourcepoint']", "[id*='sp_message']", "[class*='sp_message']",
|
|
79
|
+
"[id*='uniconsent']", "[class*='uniconsent']", "[id*='uni-consent']", "[class*='uni-consent']",
|
|
80
|
+
"[id*='osano']", "[class*='osano']"
|
|
81
|
+
].freeze,
|
|
82
|
+
quick_indicator_selectors: [
|
|
83
|
+
"#onetrust-banner-sdk", "#onetrust-pc-sdk", ".qc-cmp2-container", ".qc-cmp2-summary",
|
|
84
|
+
"#CybotCookiebotDialog", ".cc-window", ".cc_banner", "#cookie-banner", ".cookie-banner",
|
|
85
|
+
"#consent-banner", ".consent-banner", ".fc-consent-root", ".cmp-modal", ".gdpr-banner",
|
|
86
|
+
"#gdpr-consent", ".js-cookies", ".cookie-notice", "#cookieNotice",
|
|
87
|
+
'[id*="onetrust" i]', '[class*="onetrust" i]', '[id*="cookiebot" i]', '[class*="cookiebot" i]',
|
|
88
|
+
'[id*="usercentrics" i]', '[class*="usercentrics" i]', '[id*="trustarc" i]', '[class*="trustarc" i]',
|
|
89
|
+
'[id*="didomi" i]', '[class*="didomi" i]', '[id*="quantcast" i]', '[class*="quantcast" i]',
|
|
90
|
+
'[id*="sourcepoint" i]', '[class*="sourcepoint" i]', '[id*="sp_message" i]', '[class*="sp_message" i]',
|
|
91
|
+
'[id*="uniconsent" i]', '[class*="uniconsent" i]', '[id*="uni-consent" i]', '[class*="uni-consent" i]',
|
|
92
|
+
'[id*="osano" i]', '[class*="osano" i]',
|
|
93
|
+
'[id*="cookie-consent" i]', '[class*="cookie-consent" i]', '[id*="cookie_consent" i]', '[class*="cookie_consent" i]',
|
|
94
|
+
'[id*="cookieconsent" i]', '[class*="cookieconsent" i]', '[id*="cookie-banner" i]', '[class*="cookie-banner" i]',
|
|
95
|
+
'[id*="cookie_banner" i]', '[class*="cookie_banner" i]', '[id*="cookie-notice" i]', '[class*="cookie-notice" i]',
|
|
96
|
+
'[id*="gdpr" i]', '[class*="gdpr" i]', '[id*="ccpa" i]', '[class*="ccpa" i]',
|
|
97
|
+
'[id*="privacy-banner" i]', '[class*="privacy-banner" i]', '[id*="privacy_banner" i]', '[class*="privacy_banner" i]',
|
|
98
|
+
'[id*="privacy-preference" i]', '[class*="privacy-preference" i]'
|
|
99
|
+
].freeze,
|
|
100
|
+
container_selectors: [
|
|
101
|
+
"#onetrust-banner-sdk", "#onetrust-pc-sdk", ".qc-cmp2-container", ".qc-cmp2-summary",
|
|
102
|
+
"#CybotCookiebotDialog", ".cc-window", ".cc_banner", "#cookie-banner", ".cookie-banner",
|
|
103
|
+
"#consent-banner", ".consent-banner", ".fc-consent-root", ".cmp-modal", ".gdpr-banner",
|
|
104
|
+
"#gdpr-consent", ".js-cookies", ".cookie-notice", "#cookieNotice",
|
|
105
|
+
'[id*="cookie" i]', '[class*="cookie" i]', '[id*="consent" i]', '[class*="consent" i]',
|
|
106
|
+
'[class*="cookie-banner" i]', '[data-testid*="cookie" i]', '[id*="privacy" i]', '[class*="privacy" i]',
|
|
107
|
+
'[id*="gdpr" i]', '[class*="gdpr" i]', '[id*="ccpa" i]', '[class*="ccpa" i]',
|
|
108
|
+
'[id*="sourcepoint" i]', '[class*="sourcepoint" i]', '[id*="sp_message" i]', '[class*="sp_message" i]',
|
|
109
|
+
'[id*="uniconsent" i]', '[class*="uniconsent" i]', '[id*="uni-consent" i]', '[class*="uni-consent" i]',
|
|
110
|
+
'[id*="osano" i]', '[class*="osano" i]'
|
|
111
|
+
].freeze,
|
|
112
|
+
overlay_selectors: [
|
|
113
|
+
"#onetrust-banner-sdk", "#onetrust-pc-sdk", ".qc-cmp2-container", ".qc-cmp2-summary",
|
|
114
|
+
"#CybotCookiebotDialog", ".cc-window", ".cc_banner", "#cookie-banner", ".cookie-banner",
|
|
115
|
+
"#consent-banner", ".consent-banner", ".fc-consent-root", ".cmp-modal", ".gdpr-banner",
|
|
116
|
+
"#gdpr-consent", ".js-cookies", ".cookie-notice", "#cookieNotice",
|
|
117
|
+
'[id*="cookie" i]', '[class*="cookie" i]', '[id*="consent" i]', '[class*="consent" i]',
|
|
118
|
+
'[class*="cookie-banner" i]', '[data-testid*="cookie" i]', '[id*="privacy" i]', '[class*="privacy" i]',
|
|
119
|
+
'[id*="gdpr" i]', '[class*="gdpr" i]', '[id*="ccpa" i]', '[class*="ccpa" i]',
|
|
120
|
+
'[id*="onetrust" i]', '[class*="onetrust" i]', '[id*="cookiebot" i]', '[class*="cookiebot" i]',
|
|
121
|
+
'[id*="usercentrics" i]', '[class*="usercentrics" i]', '[id*="trustarc" i]', '[class*="trustarc" i]',
|
|
122
|
+
'[id*="didomi" i]', '[class*="didomi" i]', '[id*="quantcast" i]',
|
|
123
|
+
'[id*="sourcepoint" i]', '[class*="sourcepoint" i]', '[id*="sp_message" i]', '[class*="sp_message" i]',
|
|
124
|
+
'[id*="uniconsent" i]', '[class*="uniconsent" i]', '[id*="uni-consent" i]', '[class*="uni-consent" i]',
|
|
125
|
+
'[id*="osano" i]', '[class*="osano" i]'
|
|
126
|
+
].freeze,
|
|
127
|
+
button_selectors: 'button, [role="button"], a, input[type="button"], input[type="submit"]'
|
|
128
|
+
).freeze
|
|
129
|
+
# rubocop:enable Layout/LineLength, Style/RedundantPercentQ
|
|
130
|
+
private_constant :ConsentConfig, :DEFAULT_CONSENT_CONFIG
|
|
41
131
|
|
|
42
132
|
private
|
|
43
133
|
|
|
134
|
+
def consent_config(accept_labels: nil, fallback_labels: nil, extra_accept_labels: [], extra_fallback_labels: [],
|
|
135
|
+
button_selectors: nil)
|
|
136
|
+
ConsentConfig.new(
|
|
137
|
+
**DEFAULT_CONSENT_CONFIG.to_h,
|
|
138
|
+
accept_labels: Array(accept_labels || DEFAULT_CONSENT_CONFIG.accept_labels).dup.concat(Array(extra_accept_labels)).freeze,
|
|
139
|
+
fallback_labels: Array(fallback_labels || DEFAULT_CONSENT_CONFIG.fallback_labels).dup.concat(Array(extra_fallback_labels)).freeze,
|
|
140
|
+
button_selectors: button_selectors || DEFAULT_CONSENT_CONFIG.button_selectors
|
|
141
|
+
).freeze
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def consent_accept_labels(config = DEFAULT_CONSENT_CONFIG)
|
|
145
|
+
config.accept_labels
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def consent_fallback_labels(config = DEFAULT_CONSENT_CONFIG)
|
|
149
|
+
config.fallback_labels
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def consent_button_selectors(config = DEFAULT_CONSENT_CONFIG)
|
|
153
|
+
config.button_selectors
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def consent_js_patterns(config = DEFAULT_CONSENT_CONFIG)
|
|
157
|
+
<<~JS
|
|
158
|
+
const rejectPattern = new RegExp(#{JSON.generate(config.reject_pattern)}, 'i');
|
|
159
|
+
const acceptPattern = new RegExp(#{JSON.generate(config.accept_pattern)}, 'i');
|
|
160
|
+
const closePattern = new RegExp(#{JSON.generate(config.close_pattern)}, 'i');
|
|
161
|
+
const consentPattern = new RegExp(#{JSON.generate(config.context_pattern)}, 'i');
|
|
162
|
+
const containerPattern = new RegExp(#{JSON.generate(config.container_pattern)}, 'i');
|
|
163
|
+
JS
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def consent_selector_js(selectors)
|
|
167
|
+
Array(selectors).join(", ").inspect
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def consent_known_cmp_selector_js(config = DEFAULT_CONSENT_CONFIG)
|
|
171
|
+
consent_selector_js(config.known_cmp_selectors)
|
|
172
|
+
end
|
|
173
|
+
|
|
44
174
|
def accept_cookie_consent(page)
|
|
45
175
|
safe_evaluate(page, <<~JS)
|
|
46
176
|
(() => {
|
|
@@ -58,9 +188,8 @@ module FetchUtil
|
|
|
58
188
|
bodyLooksLikeConsent;
|
|
59
189
|
if (!hasConsentDialog) return false;
|
|
60
190
|
|
|
61
|
-
|
|
62
|
-
const
|
|
63
|
-
const containerPattern = /(cookie|consent|privacy|onetrust|cookiebot|usercentrics|trustarc|didomi|quantcast|gdpr|ccpa)/i;
|
|
191
|
+
#{consent_js_patterns}
|
|
192
|
+
const knownCmpSelector = #{consent_known_cmp_selector_js};
|
|
64
193
|
#{js_dom_helpers}
|
|
65
194
|
const parentOrHost = (node) => {
|
|
66
195
|
if (!node) return null;
|
|
@@ -68,7 +197,7 @@ module FetchUtil
|
|
|
68
197
|
const root = node.getRootNode && node.getRootNode();
|
|
69
198
|
return root && root.host ? root.host : null;
|
|
70
199
|
};
|
|
71
|
-
const candidates = queryAllRoots(
|
|
200
|
+
const candidates = queryAllRoots(#{consent_button_selectors.inspect});
|
|
72
201
|
|
|
73
202
|
const consentContext = (el) => {
|
|
74
203
|
let node = el;
|
|
@@ -86,13 +215,17 @@ module FetchUtil
|
|
|
86
215
|
.filter(Boolean)
|
|
87
216
|
.join(' ');
|
|
88
217
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
218
|
+
const clickFirst = (buttons, buttonPattern) => {
|
|
219
|
+
for (const el of buttons) {
|
|
220
|
+
const text = textFor(el);
|
|
221
|
+
if (!text || text.length > 120 || !visible(el) || !buttonPattern.test(text) || !consentContext(el)) continue;
|
|
222
|
+
el.click();
|
|
223
|
+
return true;
|
|
224
|
+
}
|
|
225
|
+
return false;
|
|
226
|
+
};
|
|
227
|
+
|
|
228
|
+
let clicked = clickFirst(candidates, rejectPattern) || clickFirst(candidates, acceptPattern) || clickFirst(candidates, closePattern);
|
|
96
229
|
|
|
97
230
|
if (!clicked) {
|
|
98
231
|
const consentContainerSel = #{consent_container_selector_js};
|
|
@@ -100,7 +233,7 @@ module FetchUtil
|
|
|
100
233
|
if (!visible(container)) continue;
|
|
101
234
|
for (const el of container.querySelectorAll('div, span')) {
|
|
102
235
|
const text = textFor(el);
|
|
103
|
-
if (!text || !visible(el) || !
|
|
236
|
+
if (!text || !visible(el) || !(rejectPattern.test(text) || acceptPattern.test(text) || closePattern.test(text))) continue;
|
|
104
237
|
if (text.length > 120) continue;
|
|
105
238
|
el.click();
|
|
106
239
|
clicked = true;
|
|
@@ -121,7 +254,7 @@ module FetchUtil
|
|
|
121
254
|
for (const el of queryAllRoots(consentOverlaySelector)) {
|
|
122
255
|
if (el.matches('[role="dialog"], [aria-modal="true"], dialog')) continue;
|
|
123
256
|
const style = window.getComputedStyle(el);
|
|
124
|
-
if (!/fixed|sticky/.test(style.position)) continue;
|
|
257
|
+
if (!el.matches(knownCmpSelector) && !/fixed|sticky/.test(style.position)) continue;
|
|
125
258
|
if (!visible(el)) continue;
|
|
126
259
|
el.remove();
|
|
127
260
|
clicked = true;
|
|
@@ -137,21 +270,9 @@ module FetchUtil
|
|
|
137
270
|
end
|
|
138
271
|
|
|
139
272
|
def dismiss_privacy_preference_overlay(page)
|
|
140
|
-
overlay_present = safe_evaluate(page, <<~
|
|
273
|
+
overlay_present = safe_evaluate(page, <<~JS, default: false)
|
|
141
274
|
(() => {
|
|
142
|
-
if (!document.querySelector(
|
|
143
|
-
"[id*='onetrust' i], [class*='onetrust' i], " +
|
|
144
|
-
"[id*='cookiebot' i], [class*='cookiebot' i], " +
|
|
145
|
-
"[id*='cookie-consent' i], [class*='cookie-consent' i], " +
|
|
146
|
-
"[id*='cookie_consent' i], [class*='cookie_consent' i], " +
|
|
147
|
-
"[id*='cookieconsent' i], [class*='cookieconsent' i], " +
|
|
148
|
-
"[id*='cookie-banner' i], [class*='cookie-banner' i], " +
|
|
149
|
-
"[id*='cookie-notice' i], [class*='cookie-notice' i], " +
|
|
150
|
-
"[id*='privacy-banner' i], [class*='privacy-banner' i], " +
|
|
151
|
-
"[id*='privacy_banner' i], [class*='privacy_banner' i], " +
|
|
152
|
-
"[id*='privacy-preference' i], [class*='privacy-preference' i], " +
|
|
153
|
-
"[id*='gdpr' i], [class*='gdpr' i]"
|
|
154
|
-
)) return false;
|
|
275
|
+
if (!document.querySelector(#{consent_quick_indicator_selector_js})) return false;
|
|
155
276
|
const text = ((document.body && document.body.innerText) || '').toLowerCase()
|
|
156
277
|
if (!text) return false
|
|
157
278
|
if (!/(privacy preference center|your privacy settings|your privacy choices|manage privacy preferences|manage consent preferences|cookie information|cookie list|cookies details|list of partners(?: \(vendors\))?|personverninnstillinger|informasjonskapsler|sekretessinställningar|kakor|evästeasetukset|tietosuoja|slapukų nustatymai|privatumo nustatymai|sīkdatņu iestatījumi|privātuma iestatījumi|adatvédelmi beállítások|süti beállítások|setări cookie|nastavení souhlasu)/i.test(text)) return false
|
|
@@ -162,60 +283,22 @@ module FetchUtil
|
|
|
162
283
|
|
|
163
284
|
click_visible_button_by_text(
|
|
164
285
|
page,
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
selectors:
|
|
286
|
+
consent_accept_labels,
|
|
287
|
+
consent_fallback_labels,
|
|
288
|
+
selectors: consent_button_selectors
|
|
168
289
|
)
|
|
169
290
|
end
|
|
170
291
|
|
|
171
292
|
def consent_quick_indicator_selector_js
|
|
172
|
-
|
|
173
|
-
[id*="onetrust" i], [class*="onetrust" i],
|
|
174
|
-
[id*="cookiebot" i], [class*="cookiebot" i],
|
|
175
|
-
[id*="usercentrics" i], [class*="usercentrics" i],
|
|
176
|
-
[id*="trustarc" i], [class*="trustarc" i],
|
|
177
|
-
[id*="didomi" i], [class*="didomi" i],
|
|
178
|
-
[id*="quantcast" i], [class*="quantcast" i],
|
|
179
|
-
[id*="cookie-consent" i], [class*="cookie-consent" i],
|
|
180
|
-
[id*="cookie_consent" i], [class*="cookie_consent" i],
|
|
181
|
-
[id*="cookieconsent" i], [class*="cookieconsent" i],
|
|
182
|
-
[id*="cookie-banner" i], [class*="cookie-banner" i],
|
|
183
|
-
[id*="cookie_banner" i], [class*="cookie_banner" i],
|
|
184
|
-
[id*="cookie-notice" i], [class*="cookie-notice" i],
|
|
185
|
-
[id*="gdpr" i], [class*="gdpr" i],
|
|
186
|
-
[id*="ccpa" i], [class*="ccpa" i],
|
|
187
|
-
[id*="privacy-banner" i], [class*="privacy-banner" i],
|
|
188
|
-
[id*="privacy_banner" i], [class*="privacy_banner" i]
|
|
189
|
-
JS
|
|
190
|
-
selector.gsub(/\s+/, " ").strip.inspect
|
|
293
|
+
consent_selector_js(DEFAULT_CONSENT_CONFIG.quick_indicator_selectors)
|
|
191
294
|
end
|
|
192
295
|
|
|
193
296
|
def consent_container_selector_js
|
|
194
|
-
|
|
195
|
-
[id*="cookie" i], [class*="cookie" i],
|
|
196
|
-
[id*="consent" i], [class*="consent" i],
|
|
197
|
-
[id*="privacy" i], [class*="privacy" i],
|
|
198
|
-
[id*="gdpr" i], [class*="gdpr" i],
|
|
199
|
-
[id*="ccpa" i], [class*="ccpa" i]
|
|
200
|
-
JS
|
|
201
|
-
selector.gsub(/\s+/, " ").strip.inspect
|
|
297
|
+
consent_selector_js(DEFAULT_CONSENT_CONFIG.container_selectors)
|
|
202
298
|
end
|
|
203
299
|
|
|
204
300
|
def consent_overlay_selector_js
|
|
205
|
-
|
|
206
|
-
[id*="cookie" i], [class*="cookie" i],
|
|
207
|
-
[id*="consent" i], [class*="consent" i],
|
|
208
|
-
[id*="privacy" i], [class*="privacy" i],
|
|
209
|
-
[id*="gdpr" i], [class*="gdpr" i],
|
|
210
|
-
[id*="ccpa" i], [class*="ccpa" i],
|
|
211
|
-
[id*="onetrust" i], [class*="onetrust" i],
|
|
212
|
-
[id*="cookiebot" i], [class*="cookiebot" i],
|
|
213
|
-
[id*="usercentrics" i], [class*="usercentrics" i],
|
|
214
|
-
[id*="trustarc" i], [class*="trustarc" i],
|
|
215
|
-
[id*="didomi" i], [class*="didomi" i],
|
|
216
|
-
[id*="quantcast" i]
|
|
217
|
-
JS
|
|
218
|
-
selector.gsub(/\s+/, " ").strip.inspect
|
|
301
|
+
consent_selector_js(DEFAULT_CONSENT_CONFIG.overlay_selectors)
|
|
219
302
|
end
|
|
220
303
|
end
|
|
221
304
|
# rubocop:enable Metrics/ModuleLength
|
|
@@ -136,8 +136,20 @@ module FetchUtil
|
|
|
136
136
|
};
|
|
137
137
|
|
|
138
138
|
const restoreScroll = () => {
|
|
139
|
-
|
|
140
|
-
|
|
139
|
+
const unlock = (node) => {
|
|
140
|
+
if (!node) return;
|
|
141
|
+
node.style.overflow = '';
|
|
142
|
+
node.style.position = '';
|
|
143
|
+
node.style.top = '';
|
|
144
|
+
node.style.width = '';
|
|
145
|
+
Array.from(node.classList || []).forEach((name) => {
|
|
146
|
+
if (/\b(?:modal|dialog|scroll|overflow|no)[_-]?(?:open|lock|locked|hidden|scroll)\b/i.test(name)) {
|
|
147
|
+
node.classList.remove(name);
|
|
148
|
+
}
|
|
149
|
+
});
|
|
150
|
+
};
|
|
151
|
+
unlock(document.body);
|
|
152
|
+
unlock(document.documentElement);
|
|
141
153
|
};
|
|
142
154
|
JS
|
|
143
155
|
end
|
|
@@ -6,13 +6,6 @@ module FetchUtil
|
|
|
6
6
|
module HeadersAndReadiness
|
|
7
7
|
private
|
|
8
8
|
|
|
9
|
-
def default_headers
|
|
10
|
-
{
|
|
11
|
-
"User-Agent" => @user_agent,
|
|
12
|
-
"Accept-Language" => @accept_language
|
|
13
|
-
}
|
|
14
|
-
end
|
|
15
|
-
|
|
16
9
|
def page_loaded_enough?(page)
|
|
17
10
|
page.evaluate(<<~JS)
|
|
18
11
|
(() => !!(document && document.body && (document.body.innerText || document.body.textContent || '').trim().length > 0))()
|
|
@@ -6,7 +6,9 @@ module FetchUtil
|
|
|
6
6
|
module NavigatorPatch
|
|
7
7
|
private
|
|
8
8
|
|
|
9
|
-
def navigator_patch
|
|
9
|
+
def navigator_patch = @navigator_patch
|
|
10
|
+
|
|
11
|
+
def build_navigator_patch
|
|
10
12
|
ua_version = @user_agent[%r{Chrome/([\d.]+)}, 1] || "136.0.7103.113"
|
|
11
13
|
major = ua_version.split(".").first
|
|
12
14
|
languages_json = JSON.generate(@accept_language.split(",").map { |part| part.split(";").first.strip })
|
|
@@ -4,24 +4,24 @@ module FetchUtil
|
|
|
4
4
|
class Browser
|
|
5
5
|
module SiteStabilization
|
|
6
6
|
module CommunityAndMarketplace
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
7
|
+
COMMUNITY_MARKETPLACE_STABILIZATION_PROFILES = {
|
|
8
|
+
stabilize_reddit: {
|
|
9
|
+
host: "reddit.com",
|
|
10
|
+
strategy: :stabilize_reddit,
|
|
11
|
+
notes: "Use fast Reddit cookie dismissal/content readiness instead of full idle waits.",
|
|
12
|
+
tests: "spec/fetch_util/browser_stabilization_spec.rb"
|
|
13
|
+
},
|
|
14
|
+
stabilize_ebay_search: {
|
|
15
|
+
host: "ebay.com",
|
|
16
|
+
path_query: ->(uri) { uri.path.include?("/sch/") || uri.query.to_s.include?("_nkw=") },
|
|
17
|
+
strategy: :stabilize_ebay_search,
|
|
18
|
+
notes: "Wait for search result items or short-lived eBay browser checks.",
|
|
19
|
+
tests: "spec/fetch_util/browser_stabilization_spec.rb"
|
|
20
|
+
}
|
|
21
|
+
}.freeze
|
|
18
22
|
|
|
19
23
|
private
|
|
20
24
|
|
|
21
|
-
def reddit_url?(url)
|
|
22
|
-
host_matches?(url, "reddit.com")
|
|
23
|
-
end
|
|
24
|
-
|
|
25
25
|
def stabilize_reddit(page)
|
|
26
26
|
retry_until_timeout(capped_timeout(3.0), interval: 0.1) do
|
|
27
27
|
dismiss_reddit_cookie_dialog(page)
|
|
@@ -32,24 +32,25 @@ module FetchUtil
|
|
|
32
32
|
dismiss_reddit_cookie_dialog(page)
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
-
def ebay_search_url?(url)
|
|
36
|
-
uri = URI.parse(url)
|
|
37
|
-
host = uri.host.to_s.downcase
|
|
38
|
-
return false unless host == "ebay.com" || host.end_with?(".ebay.com")
|
|
39
|
-
|
|
40
|
-
uri.path.include?("/sch/") || uri.query.to_s.include?("_nkw=")
|
|
41
|
-
rescue URI::InvalidURIError
|
|
42
|
-
false
|
|
43
|
-
end
|
|
44
|
-
|
|
45
35
|
def stabilize_ebay_search(page)
|
|
46
36
|
accepted_cookies = false
|
|
37
|
+
cookie_config = consent_config(
|
|
38
|
+
accept_labels: [
|
|
39
|
+
"accept all",
|
|
40
|
+
"accept all cookies",
|
|
41
|
+
"accept cookies",
|
|
42
|
+
"allow all",
|
|
43
|
+
"allow cookies",
|
|
44
|
+
"agree to cookies",
|
|
45
|
+
"continue with cookies"
|
|
46
|
+
]
|
|
47
|
+
)
|
|
47
48
|
|
|
48
49
|
retry_until_timeout(capped_timeout(6.0), interval: 0.15) do
|
|
49
50
|
accepted_cookies ||= click_visible_button_by_text(
|
|
50
51
|
page,
|
|
51
|
-
|
|
52
|
-
selectors:
|
|
52
|
+
consent_accept_labels(cookie_config),
|
|
53
|
+
selectors: consent_button_selectors(cookie_config)
|
|
53
54
|
)
|
|
54
55
|
|
|
55
56
|
state = safe_evaluate(page, <<~JS, default: { "itemCount" => 0, "challengeVisible" => false })
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module GitlabRepo
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def gitlab_repo_url?(url)
|
|
10
|
+
uri = URI.parse(url)
|
|
11
|
+
host = uri.host.to_s.downcase
|
|
12
|
+
(host == "gitlab.com" || host.end_with?(".gitlab.com")) && uri.path.split("/").reject(&:empty?).length == 2
|
|
13
|
+
rescue URI::InvalidURIError
|
|
14
|
+
false
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def stabilize_gitlab_repo(page)
|
|
18
|
+
retry_until_timeout(capped_timeout(8.0), interval: 0.2) do
|
|
19
|
+
safe_evaluate(page, <<~JS, default: 0).to_i >= 300
|
|
20
|
+
(() => {
|
|
21
|
+
const readme = document.querySelector('[data-testid="blob-viewer-content"] .blob-viewer[data-path="README.md"] .file-content.md, .blob-viewer[data-path="README.md"] .file-content.md');
|
|
22
|
+
return readme ? (readme.innerText || readme.textContent || '').replace(/\s+/g, ' ').trim().length : 0;
|
|
23
|
+
})()
|
|
24
|
+
JS
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
settle_after_stabilization(0.25)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -4,32 +4,17 @@ module FetchUtil
|
|
|
4
4
|
class Browser
|
|
5
5
|
module SiteStabilization
|
|
6
6
|
module SocialPlatforms
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
"rifiuta i cookie opzionali"
|
|
16
|
-
].freeze
|
|
17
|
-
FACEBOOK_COOKIE_ACCEPT_LABELS = [
|
|
18
|
-
"allow all cookies",
|
|
19
|
-
"alle cookies erlauben",
|
|
20
|
-
"autoriser tous les cookies",
|
|
21
|
-
"permitir todas las cookies",
|
|
22
|
-
"consenti tutti i cookie"
|
|
23
|
-
].freeze
|
|
24
|
-
private_constant :COOKIE_BUTTON_SELECTORS, :INSTAGRAM_COOKIE_ACCEPT_LABELS, :INSTAGRAM_COOKIE_FALLBACK_LABELS,
|
|
25
|
-
:FACEBOOK_COOKIE_DECLINE_LABELS, :FACEBOOK_COOKIE_ACCEPT_LABELS
|
|
7
|
+
SOCIAL_PLATFORM_STABILIZATION_PROFILES = {
|
|
8
|
+
stabilize_instagram: { host: "instagram.com", strategy: :stabilize_instagram,
|
|
9
|
+
notes: "Accept Instagram cookie prompts and dismiss login modals before extraction.",
|
|
10
|
+
tests: "spec/fetch_util/browser_stabilization_spec.rb" },
|
|
11
|
+
stabilize_facebook: { host: "facebook.com", strategy: :stabilize_facebook,
|
|
12
|
+
notes: "Decline/accept Facebook cookie dialogs, then dismiss login prompts.",
|
|
13
|
+
tests: "spec/fetch_util/browser_stabilization_spec.rb" }
|
|
14
|
+
}.freeze
|
|
26
15
|
|
|
27
16
|
private
|
|
28
17
|
|
|
29
|
-
def instagram_url?(url)
|
|
30
|
-
host_matches?(url, "instagram.com")
|
|
31
|
-
end
|
|
32
|
-
|
|
33
18
|
def stabilize_instagram(page)
|
|
34
19
|
wait_for_idle_or_content(page) if @wait_for_idle
|
|
35
20
|
accept_instagram_cookie_dialog(page) || accept_cookie_consent(page)
|
|
@@ -41,11 +26,16 @@ module FetchUtil
|
|
|
41
26
|
end
|
|
42
27
|
|
|
43
28
|
def accept_instagram_cookie_dialog(page)
|
|
29
|
+
config = consent_config(
|
|
30
|
+
accept_labels: ["accept", "accept all", "accept all cookies"],
|
|
31
|
+
fallback_labels: ["allow all cookies", "allow all", "allow cookies"]
|
|
32
|
+
)
|
|
33
|
+
|
|
44
34
|
click_visible_button_by_text(
|
|
45
35
|
page,
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
selectors:
|
|
36
|
+
consent_accept_labels(config),
|
|
37
|
+
consent_fallback_labels(config),
|
|
38
|
+
selectors: consent_button_selectors(config)
|
|
49
39
|
)
|
|
50
40
|
end
|
|
51
41
|
|
|
@@ -83,10 +73,6 @@ module FetchUtil
|
|
|
83
73
|
)
|
|
84
74
|
end
|
|
85
75
|
|
|
86
|
-
def facebook_url?(url)
|
|
87
|
-
host_matches?(url, "facebook.com")
|
|
88
|
-
end
|
|
89
|
-
|
|
90
76
|
def stabilize_facebook(page)
|
|
91
77
|
wait_for_idle_or_content(page) if @wait_for_idle
|
|
92
78
|
social_login_phase_pause
|
|
@@ -97,10 +83,17 @@ module FetchUtil
|
|
|
97
83
|
end
|
|
98
84
|
|
|
99
85
|
def dismiss_facebook_cookie_dialog(page)
|
|
86
|
+
config = consent_config(
|
|
87
|
+
accept_labels: ["decline optional cookies", "optionale cookies ablehnen", "refuser les cookies optionnels",
|
|
88
|
+
"rechazar cookies opcionales", "rifiuta i cookie opzionali"],
|
|
89
|
+
fallback_labels: ["allow all cookies", "alle cookies erlauben", "autoriser tous les cookies", "permitir todas las cookies", "consenti tutti i cookie"]
|
|
90
|
+
)
|
|
91
|
+
|
|
100
92
|
click_visible_button_by_text(
|
|
101
93
|
page,
|
|
102
|
-
|
|
103
|
-
|
|
94
|
+
consent_accept_labels(config),
|
|
95
|
+
consent_fallback_labels(config),
|
|
96
|
+
selectors: consent_button_selectors(config)
|
|
104
97
|
)
|
|
105
98
|
end
|
|
106
99
|
|