fetch_util 0.5.2 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +49 -0
  3. data/README.md +3 -1
  4. data/Rakefile +0 -2
  5. data/lib/fetch_util/assets/extract.js +1 -1
  6. data/lib/fetch_util/browser/interaction_helpers/consent_helpers.rb +12 -17
  7. data/lib/fetch_util/browser/interaction_helpers/timing_helpers.rb +42 -15
  8. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_commit_details_script.rb +48 -0
  9. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_product_state_script.rb +80 -0
  10. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_request_helpers_script.rb +53 -0
  11. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_request_state_script.rb +118 -0
  12. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_stabilization.rb +54 -0
  13. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_state.rb +27 -0
  14. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_product_state.rb +55 -0
  15. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_request_state.rb +100 -0
  16. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_state.rb +59 -0
  17. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_modules.rb +26 -0
  18. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_state.rb +93 -0
  19. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resources.rb +73 -0
  20. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_thread_state.rb +92 -0
  21. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_threads.rb +71 -0
  22. data/lib/fetch_util/browser/site_stabilization/community_and_marketplace.rb +9 -10
  23. data/lib/fetch_util/browser/site_stabilization/facebook_stabilization.rb +56 -0
  24. data/lib/fetch_util/browser/site_stabilization/gerrit_change_product_state_script.rb +97 -0
  25. data/lib/fetch_util/browser/site_stabilization/gerrit_change_request_state_script.rb +75 -0
  26. data/lib/fetch_util/browser/site_stabilization/gerrit_change_stabilization.rb +37 -0
  27. data/lib/fetch_util/browser/site_stabilization/gerrit_change_state.rb +39 -0
  28. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_fetch_state_script.rb +31 -0
  29. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_product_state_script.rb +76 -0
  30. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_request_state_script.rb +111 -0
  31. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_route_state_script.rb +71 -0
  32. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_stabilization.rb +37 -0
  33. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_state.rb +42 -0
  34. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_validation_state_script.rb +49 -0
  35. data/lib/fetch_util/browser/site_stabilization/gitea_family_pull_resource_product_state.rb +88 -0
  36. data/lib/fetch_util/browser/site_stabilization/gitea_family_pull_resource_state.rb +109 -0
  37. data/lib/fetch_util/browser/site_stabilization/gitea_family_pull_resources.rb +30 -0
  38. data/lib/fetch_util/browser/site_stabilization/gitea_family_stabilization.rb +60 -0
  39. data/lib/fetch_util/browser/site_stabilization/gitea_family_thread_product_state.rb +74 -0
  40. data/lib/fetch_util/browser/site_stabilization/gitea_family_thread_timeline_state.rb +69 -0
  41. data/lib/fetch_util/browser/site_stabilization/gitea_family_threads.rb +28 -0
  42. data/lib/fetch_util/browser/site_stabilization/gitea_family_visibility_script.rb +28 -0
  43. data/lib/fetch_util/browser/site_stabilization/github_pull_resource_stabilization.rb +31 -0
  44. data/lib/fetch_util/browser/site_stabilization/github_pull_resources.rb +105 -0
  45. data/lib/fetch_util/browser/site_stabilization/github_threads.rb +82 -0
  46. data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state.rb +17 -0
  47. data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state_script.rb +98 -0
  48. data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_visibility_script.rb +43 -0
  49. data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resources.rb +28 -0
  50. data/lib/fetch_util/browser/site_stabilization/gitlab_repo.rb +3 -11
  51. data/lib/fetch_util/browser/site_stabilization/gitlab_stabilization.rb +33 -0
  52. data/lib/fetch_util/browser/site_stabilization/gitlab_threads.rb +92 -0
  53. data/lib/fetch_util/browser/site_stabilization/social_platforms.rb +19 -38
  54. data/lib/fetch_util/browser/site_stabilization/travel_and_lodging.rb +3 -3
  55. data/lib/fetch_util/browser/site_stabilization.rb +89 -0
  56. data/lib/fetch_util/browser/stabilization/anubis.rb +4 -5
  57. data/lib/fetch_util/browser/stabilization/page_flow.rb +71 -27
  58. data/lib/fetch_util/browser/stabilization/spa_hydration.rb +7 -5
  59. data/lib/fetch_util/browser.rb +88 -17
  60. data/lib/fetch_util/cli.rb +69 -23
  61. data/lib/fetch_util/extractor.rb +8 -2
  62. data/lib/fetch_util/fetcher.rb +144 -43
  63. data/lib/fetch_util/http_redirect_client.rb +161 -0
  64. data/lib/fetch_util/parallel_fetcher.rb +125 -28
  65. data/lib/fetch_util/raw_docs_fallback.rb +158 -21
  66. data/lib/fetch_util/regulatory/cache_store.rb +47 -14
  67. data/lib/fetch_util/regulatory/fetch_records.rb +10 -7
  68. data/lib/fetch_util/regulatory/headers.rb +8 -1
  69. data/lib/fetch_util/regulatory/http_client.rb +2 -97
  70. data/lib/fetch_util/regulatory/orchestration.rb +10 -6
  71. data/lib/fetch_util/regulatory/page.rb +8 -11
  72. data/lib/fetch_util/regulatory/robots.rb +2 -10
  73. data/lib/fetch_util/regulatory/source_selection.rb +13 -3
  74. data/lib/fetch_util/regulatory/tdm_page.rb +9 -4
  75. data/lib/fetch_util/regulatory/tdm_policy.rb +16 -9
  76. data/lib/fetch_util/regulatory/tdm_rep.rb +4 -3
  77. data/lib/fetch_util/regulatory/tdm_support.rb +10 -6
  78. data/lib/fetch_util/regulatory.rb +5 -1
  79. data/lib/fetch_util/request_log.rb +24 -3
  80. data/lib/fetch_util/result.rb +37 -89
  81. data/lib/fetch_util/search_transport.rb +102 -35
  82. data/lib/fetch_util/searcher.rb +15 -13
  83. data/lib/fetch_util/version.rb +1 -1
  84. data/lib/fetch_util.rb +4 -1
  85. metadata +66 -2
@@ -0,0 +1,100 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module BitbucketCloudPullDiffRequestState
7
+ private
8
+
9
+ def bitbucket_cloud_pull_diff_request_state_script
10
+ <<~'JS'
11
+ const fetchUtilBitbucketDiffFetchJson = async (url) => {
12
+ const response = await fetch(url, {
13
+ credentials: 'same-origin',
14
+ redirect: 'error',
15
+ headers: { 'X-Requested-With': 'XMLHttpRequest', Accept: 'application/json' }
16
+ });
17
+ if (response.url && new URL(response.url, location.href).origin !== location.origin) {
18
+ throw new Error('cross-origin Bitbucket API response');
19
+ }
20
+ if (!response.ok) throw new Error(`Bitbucket API returned HTTP ${response.status}`);
21
+ const payload = await response.json();
22
+ if (!payload || typeof payload !== 'object' || Array.isArray(payload)) {
23
+ throw new Error('invalid Bitbucket API payload');
24
+ }
25
+ return payload;
26
+ };
27
+
28
+ const fetchUtilBitbucketDiffProxyUrl = (value, product) => {
29
+ let parsed;
30
+ try { parsed = new URL(value, product.metadataUrl); } catch (_error) { return null; }
31
+ if (!/^https?:$/.test(parsed.protocol) || parsed.username || parsed.password) return null;
32
+ const match = parsed.pathname.match(
33
+ /^(?:\/!api)?\/2\.0\/repositories\/([^/]+)\/([^/]+)\/(pullrequests\/(\d+)\/diffstat|diffstat\/(.+))\/?$/
34
+ );
35
+ const decode = (item) => { try { return decodeURIComponent(item); } catch (_error) { return ''; } };
36
+ if (!match || decode(match[1]).toLowerCase() !== product.route.workspace.toLowerCase() ||
37
+ decode(match[2]).toLowerCase() !== product.route.repository.toLowerCase()) return null;
38
+ const pullRequestId = match[4] || parsed.searchParams.get('from_pullrequest_id');
39
+ if (pullRequestId !== product.route.number) return null;
40
+ const proxyPath = parsed.pathname.replace(/^(?:\/!api)?\/2\.0/, product.proxyPrefix);
41
+ return location.origin + proxyPath + parsed.search;
42
+ };
43
+
44
+ const fetchUtilBitbucketDiffRecord = (record) => {
45
+ if (!record || typeof record !== 'object' || Array.isArray(record)) return false;
46
+ const oldPath = record.old && typeof record.old === 'object' && record.old.path;
47
+ const newPath = record.new && typeof record.new === 'object' && record.new.path;
48
+ return !!(String(oldPath || '').trim() || String(newPath || '').trim());
49
+ };
50
+
51
+ const fetchUtilBitbucketDiffRequest = async (product) => {
52
+ const metadata = await fetchUtilBitbucketDiffFetchJson(product.metadataUrl);
53
+ const repository = ((metadata.destination || {}).repository || {});
54
+ if (String(metadata.id || '') !== product.route.number ||
55
+ String(repository.full_name || '').toLowerCase() !==
56
+ `${product.route.workspace}/${product.route.repository}`.toLowerCase()) {
57
+ throw new Error('Bitbucket API pull request identity mismatch');
58
+ }
59
+ const firstUrl = fetchUtilBitbucketDiffProxyUrl(
60
+ metadata.links && metadata.links.diffstat && metadata.links.diffstat.href,
61
+ product
62
+ );
63
+ if (!firstUrl) throw new Error('missing trusted Bitbucket diffstat route');
64
+
65
+ const records = [];
66
+ const seen = new Set();
67
+ let expectedCount = null;
68
+ let pageUrl = firstUrl;
69
+ while (pageUrl) {
70
+ if (seen.has(pageUrl)) throw new Error('repeated Bitbucket diffstat continuation');
71
+ seen.add(pageUrl);
72
+ const page = await fetchUtilBitbucketDiffFetchJson(pageUrl);
73
+ const hasSize = Object.prototype.hasOwnProperty.call(page, 'size');
74
+ if (!Array.isArray(page.values) || (hasSize && (!Number.isInteger(page.size) || page.size < 0)) ||
75
+ (hasSize && expectedCount !== null && page.size !== expectedCount)) {
76
+ throw new Error('incomplete Bitbucket diffstat payload');
77
+ }
78
+ if (hasSize) expectedCount = page.size;
79
+ if (!page.values.every(fetchUtilBitbucketDiffRecord)) {
80
+ throw new Error('invalid Bitbucket diffstat record');
81
+ }
82
+ records.push(...page.values);
83
+ if (page.next) {
84
+ pageUrl = fetchUtilBitbucketDiffProxyUrl(page.next, product);
85
+ if (!pageUrl) throw new Error('untrusted Bitbucket diffstat continuation');
86
+ } else {
87
+ pageUrl = null;
88
+ }
89
+ }
90
+ if (expectedCount !== null && records.length !== expectedCount) {
91
+ throw new Error('incomplete Bitbucket diffstat payload');
92
+ }
93
+ return { metadata, records };
94
+ };
95
+ JS
96
+ end
97
+ end
98
+ end
99
+ end
100
+ end
@@ -0,0 +1,59 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module BitbucketCloudPullDiffState
7
+ private
8
+
9
+ def bitbucket_cloud_pull_diff_state_script
10
+ <<~JS
11
+ (() => {
12
+ #{bitbucket_cloud_pull_diff_product_state_script}
13
+ #{bitbucket_cloud_pull_diff_request_state_script}
14
+ const product = fetchUtilBitbucketDiffProduct();
15
+ if (!product.product) return { product: false, ready: false, loading: false, signature: '' };
16
+ const routeKey = [product.route.workspace, product.route.repository, product.route.number].join(':');
17
+ let prepared = window.__fetchUtilBitbucketPullDiff;
18
+ if (!prepared || prepared.routeKey !== routeKey) {
19
+ prepared = {
20
+ status: product.error ? 'failed' : 'loading',
21
+ reason: product.error || '', routeKey, route: product.route, metadata: null, values: []
22
+ };
23
+ window.__fetchUtilBitbucketPullDiff = prepared;
24
+ if (!product.error) {
25
+ fetchUtilBitbucketDiffRequest(product).then((result) => {
26
+ if (window.__fetchUtilBitbucketPullDiff !== prepared || prepared.status !== 'loading') return;
27
+ Object.assign(prepared, {
28
+ status: 'ready', reason: '', metadata: result.metadata, values: result.records
29
+ });
30
+ }).catch((error) => {
31
+ if (window.__fetchUtilBitbucketPullDiff !== prepared || prepared.status !== 'loading') return;
32
+ prepared.status = 'failed';
33
+ prepared.reason = String(error && error.message || error || 'Bitbucket diffstat preparation failed');
34
+ });
35
+ }
36
+ }
37
+ const loading = prepared.status === 'loading';
38
+ const ready = prepared.status === 'ready' || prepared.status === 'failed';
39
+ const signature = [prepared.status, prepared.reason || '', prepared.values.length].join(':');
40
+ return { product: true, ready, loading, signature };
41
+ })()
42
+ JS
43
+ end
44
+
45
+ def fail_bitbucket_cloud_pull_diff_preparation(page)
46
+ safe_evaluate(page, <<~'JS', default: false)
47
+ (() => {
48
+ const prepared = window.__fetchUtilBitbucketPullDiff;
49
+ if (!prepared || prepared.status !== 'loading') return false;
50
+ prepared.status = 'failed';
51
+ prepared.reason = 'Bitbucket diffstat preparation timed out';
52
+ return true;
53
+ })()
54
+ JS
55
+ end
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ autoload :BitbucketCloudPullDiffProductState,
7
+ "fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_product_state"
8
+ autoload :BitbucketCloudPullDiffRequestState,
9
+ "fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_request_state"
10
+ autoload :BitbucketCloudPullDiffState,
11
+ "fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_state"
12
+ autoload :BitbucketCloudPullResourceState,
13
+ "fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_state"
14
+ autoload :BitbucketCloudPullResources,
15
+ "fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resources"
16
+
17
+ module BitbucketCloudPullResourceModules
18
+ include BitbucketCloudPullDiffProductState
19
+ include BitbucketCloudPullDiffRequestState
20
+ include BitbucketCloudPullDiffState
21
+ include BitbucketCloudPullResourceState
22
+ include BitbucketCloudPullResources
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,93 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module BitbucketCloudPullResourceState
7
+ private
8
+
9
+ def bitbucket_cloud_pull_resource_state_script
10
+ <<~JS
11
+ (() => {
12
+ if (location.pathname.endsWith("/diff") || location.pathname.endsWith("/diff/")) {
13
+ return (#{bitbucket_cloud_pull_diff_state_script});
14
+ }
15
+ return (#{bitbucket_cloud_pull_commit_resource_state_script});
16
+ })()
17
+ JS
18
+ end
19
+
20
+ def bitbucket_cloud_pull_commit_resource_state_script
21
+ <<~'JS'
22
+ (() => {
23
+ const match = location.pathname.match(/^\/([^/]+)\/([^/]+)\/pull-requests\/(\d+)\/commits\/?$/);
24
+ if (!match) return { product: false, ready: false, loading: false, signature: '' };
25
+ const decode = (value) => { try { return decodeURIComponent(value); } catch (_error) { return ''; } };
26
+ const workspace = decode(match[1]);
27
+ const repository = decode(match[2]);
28
+ if (!workspace || !repository || workspace.includes('/') || repository.includes('/')) {
29
+ return { product: false, ready: false, loading: false, signature: '' };
30
+ }
31
+ const current = (((window.__initial_state__ || {}).section || {}).repository || {}).currentRepository || {};
32
+ const publicRepository = current.is_private === false || current.isPrivate === false || current.visibility === 'public';
33
+ const runtime = current.type === 'repository' && publicRepository &&
34
+ String(current.full_name || '').toLowerCase() === `${workspace}/${repository}`.toLowerCase();
35
+ const application = document.querySelector('meta[name="application-name"]');
36
+ const branded = /^Bitbucket$/i.test((application && application.content || '').trim());
37
+ const asset = Array.from(document.querySelectorAll('script[src], link[href]'))
38
+ .some((node) => {
39
+ const value = node.getAttribute('src') || node.getAttribute('href') || '';
40
+ return /(?:frontbucket|bbc-frontbucket-static|bitbucket)[^/]*\/(?:assets|static)\//i.test(value) ||
41
+ /\/(?:assets|static)\/[^?#]*(?:frontbucket|bitbucket)/i.test(value);
42
+ });
43
+ const header = document.querySelector('[data-testid="pr-header"]');
44
+ const product = runtime && branded && !!(header || asset);
45
+ if (!product) return { product: false, ready: false, loading: false, signature: '' };
46
+
47
+ const panel = document.querySelector('[role="tabpanel"][id^="pull-request-tabs-"]');
48
+ const wrapper = document.querySelector('[data-qa^="commit-hash-wrapper-"]');
49
+ const root = panel || (wrapper && wrapper.closest('table, [role="grid"]'));
50
+
51
+ const visible = (node) => {
52
+ let currentNode = node;
53
+ while (currentNode && currentNode !== document) {
54
+ const style = getComputedStyle(currentNode);
55
+ if (currentNode.hidden || style.display === 'none') return false;
56
+ currentNode = currentNode.parentElement;
57
+ }
58
+ const style = getComputedStyle(node);
59
+ return style.visibility !== 'hidden' && style.visibility !== 'collapse';
60
+ };
61
+ const seen = new Set();
62
+ const rows = Array.from(root ? root.querySelectorAll('a[aria-label^="Commit: "][href*="/commits/"]') : [])
63
+ .map((link) => {
64
+ const label = (link.getAttribute('aria-label') || '').trim().match(/^Commit:\s*([0-9a-f]{7,40})$/i);
65
+ const row = link.closest('tr, [role="row"]');
66
+ try {
67
+ const url = new URL(link.href, location.href);
68
+ const path = url.pathname.match(/^\/([^/]+)\/([^/]+)\/commits\/([0-9a-f]{40})\/?$/i);
69
+ if (!label || !row || !visible(row) || url.origin !== location.origin || !path) return null;
70
+ if (decode(path[1]) !== workspace || decode(path[2]) !== repository) return null;
71
+ if (!path[3].toLowerCase().startsWith(label[1].toLowerCase())) return null;
72
+ const hash = path[3].toLowerCase();
73
+ if (seen.has(hash)) return null;
74
+ seen.add(hash);
75
+ return { row, hash, href: url.href };
76
+ } catch (_error) {
77
+ return null;
78
+ }
79
+ }).filter(Boolean);
80
+ const loading = Array.from(document.querySelectorAll(
81
+ '[aria-busy="true"], [data-testid*="loading"], [data-testid*="spinner"], .loading, .spinner'
82
+ )).some(visible);
83
+ const signature = rows.map(({ row, hash, href }) =>
84
+ `${hash}:${href}:${(row.textContent || '').replace(/\s+/g, ' ').trim()}`
85
+ ).join('|');
86
+ return { product: true, ready: !loading && rows.length > 0, loading, signature };
87
+ })()
88
+ JS
89
+ end
90
+ end
91
+ end
92
+ end
93
+ end
@@ -0,0 +1,73 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module BitbucketCloudPullResources
7
+ BITBUCKET_CLOUD_PULL_RESOURCE_STABILIZATION_PROFILE = {
8
+ host: true,
9
+ path_query: ->(uri) { uri.path.match?(%r{\A/[^/]+/[^/]+/pull-requests/\d+/(?:commits|diff)/?\z}) },
10
+ strategy: :stabilize_bitbucket_cloud_pull_resource,
11
+ fallthrough: true,
12
+ notes: "Wait for product-matched Bitbucket Cloud pull-request commit or diff resources.",
13
+ tests: "spec/fetch_util/browser_stabilization_spec.rb"
14
+ }.freeze
15
+
16
+ private
17
+
18
+ def stabilize_bitbucket_cloud_pull_resource(page, deadline: stabilization_deadline)
19
+ last_signature = nil
20
+ stable_observations = 0
21
+ incomplete_signature = nil
22
+ incomplete_observations = 0
23
+ terminal_incomplete = false
24
+ observed = retry_until_timeout(
25
+ capped_timeout(6.0, deadline: deadline),
26
+ interval: 0.1,
27
+ deadline: deadline
28
+ ) do
29
+ state = safe_evaluate(page, bitbucket_cloud_pull_resource_state_script, default: nil)
30
+ return false if state.is_a?(Hash) && state["product"] == false
31
+ next false unless state.is_a?(Hash) && state["product"]
32
+
33
+ unless state["ready"]
34
+ last_signature = nil
35
+ stable_observations = 0
36
+ if state["loading"]
37
+ incomplete_signature = nil
38
+ incomplete_observations = 0
39
+ elsif state["signature"] == incomplete_signature
40
+ incomplete_observations += 1
41
+ else
42
+ incomplete_signature = state["signature"]
43
+ incomplete_observations = 1
44
+ end
45
+ if incomplete_observations >= 10
46
+ terminal_incomplete = true
47
+ next true
48
+ end
49
+ next false
50
+ end
51
+
52
+ incomplete_signature = nil
53
+ incomplete_observations = 0
54
+ if state["signature"] == last_signature
55
+ stable_observations += 1
56
+ else
57
+ last_signature = state["signature"]
58
+ stable_observations = 1
59
+ end
60
+ stable_observations >= 3
61
+ end
62
+
63
+ return false if terminal_incomplete
64
+
65
+ fail_bitbucket_cloud_pull_diff_preparation(page) unless observed
66
+
67
+ settle_after_stabilization(0.5, deadline: deadline) if observed
68
+ !!observed
69
+ end
70
+ end
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,92 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module BitbucketCloudThreadState
7
+ private
8
+
9
+ def bitbucket_cloud_thread_state_script
10
+ <<~'JS'
11
+ (() => {
12
+ const match = location.pathname.match(/^\/([^/]+)\/([^/]+)\/pull-requests\/(\d+)(?:\/overview)?\/?$/);
13
+ if (!match) return { product: false, ready: false, loading: false, signature: '' };
14
+
15
+ const decoded = (value) => {
16
+ try { return decodeURIComponent(value); } catch (_error) { return value; }
17
+ };
18
+ const community = `${decoded(match[1])}/${decoded(match[2])}`.toLowerCase();
19
+ const state = window.__initial_state__ || {};
20
+ const repository = (((state.section || {}).repository || {}).currentRepository) || null;
21
+ const publicRepository = !!repository && (
22
+ repository.is_private === false || repository.isPrivate === false || repository.visibility === 'public'
23
+ );
24
+ const runtime = publicRepository && repository.type === 'repository' &&
25
+ String(repository.full_name || '').trim().toLowerCase() === community;
26
+ const application = document.querySelector('meta[name="application-name"]');
27
+ const branded = /^Bitbucket$/i.test((application && application.content || '').trim());
28
+ const asset = Array.from(document.querySelectorAll('script[src], link[href]')).some((node) => {
29
+ const value = node.getAttribute('src') || node.getAttribute('href') || '';
30
+ return /(?:frontbucket|bbc-frontbucket-static|bitbucket)[^/]*\/(?:assets|static)\//i.test(value) ||
31
+ /\/(?:assets|static)\/[^?#]*(?:frontbucket|bitbucket)/i.test(value);
32
+ });
33
+ const header = document.querySelector('[data-testid="pr-header"]');
34
+ const root = header && (header.closest('main') || document.querySelector('main') || header.parentElement) ||
35
+ document.querySelector('main');
36
+ const opening = root && root.querySelector(
37
+ '#pull-request-description-panel, section[aria-label="Pull request description"], section[aria-label="Description"]'
38
+ );
39
+ const product = runtime && branded && !!(header || asset);
40
+ if (!product) return { product: false, ready: false, loading: false, signature: '' };
41
+
42
+ const nodeVisible = (node) => {
43
+ let current = node;
44
+ while (current && current !== document) {
45
+ const style = getComputedStyle(current);
46
+ if (current.hidden || style.display === 'none') return false;
47
+ current = current.parentElement;
48
+ }
49
+ const visibility = getComputedStyle(node).visibility;
50
+ return visibility !== 'hidden' && visibility !== 'collapse';
51
+ };
52
+ const subtreeVisible = (node) => {
53
+ if (nodeVisible(node)) return true;
54
+ return Array.from(node.querySelectorAll('*')).some((child) => {
55
+ const material = (child.textContent || '').trim() ||
56
+ child.matches('img[src], video, pre, table, ul, ol');
57
+ return material && nodeVisible(child);
58
+ });
59
+ };
60
+ const comments = Array.from(root.querySelectorAll('[data-testid="comment"]')).filter(subtreeVisible);
61
+ const owned = (node, selector) => Array.from(node.querySelectorAll(selector)).find(
62
+ (candidate) => candidate.closest('[data-testid="comment"]') === node
63
+ );
64
+ const loading = Array.from(root.querySelectorAll(
65
+ '[aria-busy="true"], [data-testid*="loading" i], [data-testid*="spinner" i], .spinner, .loading'
66
+ )).some(subtreeVisible);
67
+ const openingReady = !!opening && subtreeVisible(opening) && !!(
68
+ (opening.textContent || '').trim() || opening.querySelector('img[src], video, pre, table, ul, ol')
69
+ );
70
+ const commentBodies = comments.filter((node) => {
71
+ const body = owned(node, '[data-testid="comment-content"], comment-content');
72
+ return body && subtreeVisible(body) && !!((body.textContent || '').trim() || body.querySelector('img[src], pre, table'));
73
+ });
74
+ const explicitZero = /\b0\s+comments?\b/i.test(root.textContent || '');
75
+ const rowSignatures = commentBodies.map((node, index) => {
76
+ const link = owned(node, 'a[href*="#comment-"]');
77
+ return node.id || node.getAttribute('data-comment-id') || (link && link.href) ||
78
+ `${index}:${(node.textContent || '').trim().slice(0, 120)}`;
79
+ });
80
+ return {
81
+ product: true,
82
+ ready: !!header && openingReady && !loading && (commentBodies.length > 0 || explicitZero),
83
+ loading: loading,
84
+ signature: [openingReady, commentBodies.length, explicitZero, loading, rowSignatures.join('|')].join(':')
85
+ };
86
+ })()
87
+ JS
88
+ end
89
+ end
90
+ end
91
+ end
92
+ end
@@ -0,0 +1,71 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module BitbucketCloudThreads
7
+ BITBUCKET_CLOUD_THREAD_STABILIZATION_PROFILE = {
8
+ host: true,
9
+ path_query: ->(uri) { uri.path.match?(%r{\A/[^/]+/[^/]+/pull-requests/\d+(?:/overview)?/?\z}) },
10
+ strategy: :stabilize_bitbucket_cloud_thread,
11
+ fallthrough: true,
12
+ notes: "Wait for product-matched Bitbucket Cloud pull-request conversations.",
13
+ tests: "spec/fetch_util/browser_stabilization_spec.rb"
14
+ }.freeze
15
+
16
+ private
17
+
18
+ def stabilize_bitbucket_cloud_thread(page, deadline: stabilization_deadline)
19
+ last_signature = nil
20
+ stable_observations = 0
21
+ incomplete_signature = nil
22
+ incomplete_observations = 0
23
+ terminal_incomplete = false
24
+ observed = retry_until_timeout(
25
+ capped_timeout(6.0, deadline: deadline),
26
+ interval: 0.1,
27
+ deadline: deadline
28
+ ) do
29
+ state = safe_evaluate(page, bitbucket_cloud_thread_state_script, default: nil)
30
+ return false if state.is_a?(Hash) && state["product"] == false
31
+ next false unless state.is_a?(Hash) && state["product"]
32
+
33
+ unless state["ready"]
34
+ last_signature = nil
35
+ stable_observations = 0
36
+ if state["loading"]
37
+ incomplete_signature = nil
38
+ incomplete_observations = 0
39
+ elsif state["signature"] == incomplete_signature
40
+ incomplete_observations += 1
41
+ else
42
+ incomplete_signature = state["signature"]
43
+ incomplete_observations = 1
44
+ end
45
+ if incomplete_observations >= 10
46
+ terminal_incomplete = true
47
+ next true
48
+ end
49
+ next false
50
+ end
51
+
52
+ incomplete_signature = nil
53
+ incomplete_observations = 0
54
+ if state["signature"] == last_signature
55
+ stable_observations += 1
56
+ else
57
+ last_signature = state["signature"]
58
+ stable_observations = 1
59
+ end
60
+ stable_observations >= 3
61
+ end
62
+
63
+ return false if terminal_incomplete
64
+
65
+ settle_after_stabilization(0.5, deadline: deadline) if observed
66
+ !!observed
67
+ end
68
+ end
69
+ end
70
+ end
71
+ end
@@ -22,17 +22,18 @@ module FetchUtil
22
22
 
23
23
  private
24
24
 
25
- def stabilize_reddit(page)
26
- retry_until_timeout(capped_timeout(3.0), interval: 0.1) do
25
+ def stabilize_reddit(page, deadline: stabilization_deadline)
26
+ ready = retry_until_timeout(capped_timeout(3.0, deadline: deadline), interval: 0.1, deadline: deadline) do
27
27
  dismiss_reddit_cookie_dialog(page)
28
28
  reddit_content_ready?(page)
29
29
  end
30
30
 
31
- settle_after_stabilization(0.25)
32
- dismiss_reddit_cookie_dialog(page)
31
+ settle_after_stabilization(0.25, deadline: deadline)
32
+ dismiss_reddit_cookie_dialog(page) if stabilization_time_remaining?(deadline)
33
+ ready
33
34
  end
34
35
 
35
- def stabilize_ebay_search(page)
36
+ def stabilize_ebay_search(page, deadline: stabilization_deadline)
36
37
  accepted_cookies = false
37
38
  cookie_config = consent_config(
38
39
  accept_labels: [
@@ -46,7 +47,7 @@ module FetchUtil
46
47
  ]
47
48
  )
48
49
 
49
- retry_until_timeout(capped_timeout(6.0), interval: 0.15) do
50
+ retry_until_timeout(capped_timeout(6.0, deadline: deadline), interval: 0.15, deadline: deadline) do
50
51
  accepted_cookies ||= click_visible_button_by_text(
51
52
  page,
52
53
  consent_accept_labels(cookie_config),
@@ -66,15 +67,13 @@ module FetchUtil
66
67
  state["itemCount"].to_i >= 4 || (state["challengeVisible"] ? 0.35 : false)
67
68
  end
68
69
 
69
- settle_after_stabilization(0.25) if accepted_cookies
70
+ settle_after_stabilization(0.25, deadline: deadline) if accepted_cookies
70
71
  end
71
72
 
72
73
  def reddit_content_ready?(page)
73
- page.evaluate(<<~JS)
74
+ safe_evaluate(page, <<~JS, default: false)
74
75
  !!document.querySelector('shreddit-post, faceplate-screen-reader-content, shreddit-comment, [data-testid="comment"]')
75
76
  JS
76
- rescue Ferrum::JavaScriptError
77
- false
78
77
  end
79
78
 
80
79
  def dismiss_reddit_cookie_dialog(page)
@@ -0,0 +1,56 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module FacebookStabilization
7
+ private
8
+
9
+ def stabilize_facebook(page, deadline: stabilization_deadline)
10
+ wait_for_idle_or_content(page, deadline: deadline) if @wait_for_idle
11
+ return false unless stabilization_time_remaining?(deadline)
12
+
13
+ social_login_phase_pause(deadline: deadline)
14
+ return false unless stabilization_time_remaining?(deadline)
15
+
16
+ dismiss_facebook_cookie_dialog(page)
17
+ return false unless stabilization_time_remaining?(deadline)
18
+
19
+ social_login_phase_pause(deadline: deadline)
20
+ return false unless stabilization_time_remaining?(deadline)
21
+
22
+ retry_until_timeout(capped_timeout(5.0, deadline: deadline), deadline: deadline) do
23
+ dismiss_facebook_login_dialog(page)
24
+ end
25
+ return false unless stabilization_time_remaining?(deadline)
26
+
27
+ social_login_phase_pause(deadline: deadline)
28
+ end
29
+
30
+ def dismiss_facebook_cookie_dialog(page)
31
+ config = consent_config(
32
+ accept_labels: ["decline optional cookies", "optionale cookies ablehnen", "refuser les cookies optionnels",
33
+ "rechazar cookies opcionales", "rifiuta i cookie opzionali"],
34
+ fallback_labels: ["allow all cookies", "alle cookies erlauben", "autoriser tous les cookies", "permitir todas las cookies", "consenti tutti i cookie"]
35
+ )
36
+
37
+ click_visible_button_by_text(
38
+ page,
39
+ consent_accept_labels(config),
40
+ consent_fallback_labels(config),
41
+ selectors: consent_button_selectors(config)
42
+ )
43
+ end
44
+
45
+ def dismiss_facebook_login_dialog(page)
46
+ dismiss_overlay_dialog(
47
+ page,
48
+ close_selectors: ['[aria-label="Close"]', '[aria-label="close"]'],
49
+ dialog_selectors: ['[role="dialog"]', '[aria-modal="true"]'],
50
+ dialog_pattern: "log in|sign up|create (?:new )?account|see more from"
51
+ )
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end