fetch_util 0.5.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +49 -0
- data/README.md +3 -1
- data/Rakefile +0 -2
- data/lib/fetch_util/assets/extract.js +1 -1
- data/lib/fetch_util/browser/interaction_helpers/consent_helpers.rb +12 -17
- data/lib/fetch_util/browser/interaction_helpers/timing_helpers.rb +42 -15
- data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_commit_details_script.rb +48 -0
- data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_product_state_script.rb +80 -0
- data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_request_helpers_script.rb +53 -0
- data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_request_state_script.rb +118 -0
- data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_stabilization.rb +54 -0
- data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_state.rb +27 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_product_state.rb +55 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_request_state.rb +100 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_state.rb +59 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_modules.rb +26 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_state.rb +93 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resources.rb +73 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_thread_state.rb +92 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_threads.rb +71 -0
- data/lib/fetch_util/browser/site_stabilization/community_and_marketplace.rb +9 -10
- data/lib/fetch_util/browser/site_stabilization/facebook_stabilization.rb +56 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_change_product_state_script.rb +97 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_change_request_state_script.rb +75 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_change_stabilization.rb +37 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_change_state.rb +39 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_fetch_state_script.rb +31 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_product_state_script.rb +76 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_request_state_script.rb +111 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_route_state_script.rb +71 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_stabilization.rb +37 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_state.rb +42 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_validation_state_script.rb +49 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_pull_resource_product_state.rb +88 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_pull_resource_state.rb +109 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_pull_resources.rb +30 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_stabilization.rb +60 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_thread_product_state.rb +74 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_thread_timeline_state.rb +69 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_threads.rb +28 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_visibility_script.rb +28 -0
- data/lib/fetch_util/browser/site_stabilization/github_pull_resource_stabilization.rb +31 -0
- data/lib/fetch_util/browser/site_stabilization/github_pull_resources.rb +105 -0
- data/lib/fetch_util/browser/site_stabilization/github_threads.rb +82 -0
- data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state.rb +17 -0
- data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state_script.rb +98 -0
- data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_visibility_script.rb +43 -0
- data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resources.rb +28 -0
- data/lib/fetch_util/browser/site_stabilization/gitlab_repo.rb +3 -11
- data/lib/fetch_util/browser/site_stabilization/gitlab_stabilization.rb +33 -0
- data/lib/fetch_util/browser/site_stabilization/gitlab_threads.rb +92 -0
- data/lib/fetch_util/browser/site_stabilization/social_platforms.rb +19 -38
- data/lib/fetch_util/browser/site_stabilization/travel_and_lodging.rb +3 -3
- data/lib/fetch_util/browser/site_stabilization.rb +89 -0
- data/lib/fetch_util/browser/stabilization/anubis.rb +4 -5
- data/lib/fetch_util/browser/stabilization/page_flow.rb +71 -27
- data/lib/fetch_util/browser/stabilization/spa_hydration.rb +7 -5
- data/lib/fetch_util/browser.rb +88 -17
- data/lib/fetch_util/cli.rb +69 -23
- data/lib/fetch_util/extractor.rb +8 -2
- data/lib/fetch_util/fetcher.rb +144 -43
- data/lib/fetch_util/http_redirect_client.rb +161 -0
- data/lib/fetch_util/parallel_fetcher.rb +125 -28
- data/lib/fetch_util/raw_docs_fallback.rb +158 -21
- data/lib/fetch_util/regulatory/cache_store.rb +47 -14
- data/lib/fetch_util/regulatory/fetch_records.rb +10 -7
- data/lib/fetch_util/regulatory/headers.rb +8 -1
- data/lib/fetch_util/regulatory/http_client.rb +2 -97
- data/lib/fetch_util/regulatory/orchestration.rb +10 -6
- data/lib/fetch_util/regulatory/page.rb +8 -11
- data/lib/fetch_util/regulatory/robots.rb +2 -10
- data/lib/fetch_util/regulatory/source_selection.rb +13 -3
- data/lib/fetch_util/regulatory/tdm_page.rb +9 -4
- data/lib/fetch_util/regulatory/tdm_policy.rb +16 -9
- data/lib/fetch_util/regulatory/tdm_rep.rb +4 -3
- data/lib/fetch_util/regulatory/tdm_support.rb +10 -6
- data/lib/fetch_util/regulatory.rb +5 -1
- data/lib/fetch_util/request_log.rb +24 -3
- data/lib/fetch_util/result.rb +37 -89
- data/lib/fetch_util/search_transport.rb +102 -35
- data/lib/fetch_util/searcher.rb +15 -13
- data/lib/fetch_util/version.rb +1 -1
- data/lib/fetch_util.rb +4 -1
- metadata +66 -2
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module BitbucketCloudPullDiffRequestState
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def bitbucket_cloud_pull_diff_request_state_script
|
|
10
|
+
<<~'JS'
|
|
11
|
+
const fetchUtilBitbucketDiffFetchJson = async (url) => {
|
|
12
|
+
const response = await fetch(url, {
|
|
13
|
+
credentials: 'same-origin',
|
|
14
|
+
redirect: 'error',
|
|
15
|
+
headers: { 'X-Requested-With': 'XMLHttpRequest', Accept: 'application/json' }
|
|
16
|
+
});
|
|
17
|
+
if (response.url && new URL(response.url, location.href).origin !== location.origin) {
|
|
18
|
+
throw new Error('cross-origin Bitbucket API response');
|
|
19
|
+
}
|
|
20
|
+
if (!response.ok) throw new Error(`Bitbucket API returned HTTP ${response.status}`);
|
|
21
|
+
const payload = await response.json();
|
|
22
|
+
if (!payload || typeof payload !== 'object' || Array.isArray(payload)) {
|
|
23
|
+
throw new Error('invalid Bitbucket API payload');
|
|
24
|
+
}
|
|
25
|
+
return payload;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
const fetchUtilBitbucketDiffProxyUrl = (value, product) => {
|
|
29
|
+
let parsed;
|
|
30
|
+
try { parsed = new URL(value, product.metadataUrl); } catch (_error) { return null; }
|
|
31
|
+
if (!/^https?:$/.test(parsed.protocol) || parsed.username || parsed.password) return null;
|
|
32
|
+
const match = parsed.pathname.match(
|
|
33
|
+
/^(?:\/!api)?\/2\.0\/repositories\/([^/]+)\/([^/]+)\/(pullrequests\/(\d+)\/diffstat|diffstat\/(.+))\/?$/
|
|
34
|
+
);
|
|
35
|
+
const decode = (item) => { try { return decodeURIComponent(item); } catch (_error) { return ''; } };
|
|
36
|
+
if (!match || decode(match[1]).toLowerCase() !== product.route.workspace.toLowerCase() ||
|
|
37
|
+
decode(match[2]).toLowerCase() !== product.route.repository.toLowerCase()) return null;
|
|
38
|
+
const pullRequestId = match[4] || parsed.searchParams.get('from_pullrequest_id');
|
|
39
|
+
if (pullRequestId !== product.route.number) return null;
|
|
40
|
+
const proxyPath = parsed.pathname.replace(/^(?:\/!api)?\/2\.0/, product.proxyPrefix);
|
|
41
|
+
return location.origin + proxyPath + parsed.search;
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
const fetchUtilBitbucketDiffRecord = (record) => {
|
|
45
|
+
if (!record || typeof record !== 'object' || Array.isArray(record)) return false;
|
|
46
|
+
const oldPath = record.old && typeof record.old === 'object' && record.old.path;
|
|
47
|
+
const newPath = record.new && typeof record.new === 'object' && record.new.path;
|
|
48
|
+
return !!(String(oldPath || '').trim() || String(newPath || '').trim());
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
const fetchUtilBitbucketDiffRequest = async (product) => {
|
|
52
|
+
const metadata = await fetchUtilBitbucketDiffFetchJson(product.metadataUrl);
|
|
53
|
+
const repository = ((metadata.destination || {}).repository || {});
|
|
54
|
+
if (String(metadata.id || '') !== product.route.number ||
|
|
55
|
+
String(repository.full_name || '').toLowerCase() !==
|
|
56
|
+
`${product.route.workspace}/${product.route.repository}`.toLowerCase()) {
|
|
57
|
+
throw new Error('Bitbucket API pull request identity mismatch');
|
|
58
|
+
}
|
|
59
|
+
const firstUrl = fetchUtilBitbucketDiffProxyUrl(
|
|
60
|
+
metadata.links && metadata.links.diffstat && metadata.links.diffstat.href,
|
|
61
|
+
product
|
|
62
|
+
);
|
|
63
|
+
if (!firstUrl) throw new Error('missing trusted Bitbucket diffstat route');
|
|
64
|
+
|
|
65
|
+
const records = [];
|
|
66
|
+
const seen = new Set();
|
|
67
|
+
let expectedCount = null;
|
|
68
|
+
let pageUrl = firstUrl;
|
|
69
|
+
while (pageUrl) {
|
|
70
|
+
if (seen.has(pageUrl)) throw new Error('repeated Bitbucket diffstat continuation');
|
|
71
|
+
seen.add(pageUrl);
|
|
72
|
+
const page = await fetchUtilBitbucketDiffFetchJson(pageUrl);
|
|
73
|
+
const hasSize = Object.prototype.hasOwnProperty.call(page, 'size');
|
|
74
|
+
if (!Array.isArray(page.values) || (hasSize && (!Number.isInteger(page.size) || page.size < 0)) ||
|
|
75
|
+
(hasSize && expectedCount !== null && page.size !== expectedCount)) {
|
|
76
|
+
throw new Error('incomplete Bitbucket diffstat payload');
|
|
77
|
+
}
|
|
78
|
+
if (hasSize) expectedCount = page.size;
|
|
79
|
+
if (!page.values.every(fetchUtilBitbucketDiffRecord)) {
|
|
80
|
+
throw new Error('invalid Bitbucket diffstat record');
|
|
81
|
+
}
|
|
82
|
+
records.push(...page.values);
|
|
83
|
+
if (page.next) {
|
|
84
|
+
pageUrl = fetchUtilBitbucketDiffProxyUrl(page.next, product);
|
|
85
|
+
if (!pageUrl) throw new Error('untrusted Bitbucket diffstat continuation');
|
|
86
|
+
} else {
|
|
87
|
+
pageUrl = null;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
if (expectedCount !== null && records.length !== expectedCount) {
|
|
91
|
+
throw new Error('incomplete Bitbucket diffstat payload');
|
|
92
|
+
}
|
|
93
|
+
return { metadata, records };
|
|
94
|
+
};
|
|
95
|
+
JS
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module BitbucketCloudPullDiffState
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def bitbucket_cloud_pull_diff_state_script
|
|
10
|
+
<<~JS
|
|
11
|
+
(() => {
|
|
12
|
+
#{bitbucket_cloud_pull_diff_product_state_script}
|
|
13
|
+
#{bitbucket_cloud_pull_diff_request_state_script}
|
|
14
|
+
const product = fetchUtilBitbucketDiffProduct();
|
|
15
|
+
if (!product.product) return { product: false, ready: false, loading: false, signature: '' };
|
|
16
|
+
const routeKey = [product.route.workspace, product.route.repository, product.route.number].join(':');
|
|
17
|
+
let prepared = window.__fetchUtilBitbucketPullDiff;
|
|
18
|
+
if (!prepared || prepared.routeKey !== routeKey) {
|
|
19
|
+
prepared = {
|
|
20
|
+
status: product.error ? 'failed' : 'loading',
|
|
21
|
+
reason: product.error || '', routeKey, route: product.route, metadata: null, values: []
|
|
22
|
+
};
|
|
23
|
+
window.__fetchUtilBitbucketPullDiff = prepared;
|
|
24
|
+
if (!product.error) {
|
|
25
|
+
fetchUtilBitbucketDiffRequest(product).then((result) => {
|
|
26
|
+
if (window.__fetchUtilBitbucketPullDiff !== prepared || prepared.status !== 'loading') return;
|
|
27
|
+
Object.assign(prepared, {
|
|
28
|
+
status: 'ready', reason: '', metadata: result.metadata, values: result.records
|
|
29
|
+
});
|
|
30
|
+
}).catch((error) => {
|
|
31
|
+
if (window.__fetchUtilBitbucketPullDiff !== prepared || prepared.status !== 'loading') return;
|
|
32
|
+
prepared.status = 'failed';
|
|
33
|
+
prepared.reason = String(error && error.message || error || 'Bitbucket diffstat preparation failed');
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
const loading = prepared.status === 'loading';
|
|
38
|
+
const ready = prepared.status === 'ready' || prepared.status === 'failed';
|
|
39
|
+
const signature = [prepared.status, prepared.reason || '', prepared.values.length].join(':');
|
|
40
|
+
return { product: true, ready, loading, signature };
|
|
41
|
+
})()
|
|
42
|
+
JS
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def fail_bitbucket_cloud_pull_diff_preparation(page)
|
|
46
|
+
safe_evaluate(page, <<~'JS', default: false)
|
|
47
|
+
(() => {
|
|
48
|
+
const prepared = window.__fetchUtilBitbucketPullDiff;
|
|
49
|
+
if (!prepared || prepared.status !== 'loading') return false;
|
|
50
|
+
prepared.status = 'failed';
|
|
51
|
+
prepared.reason = 'Bitbucket diffstat preparation timed out';
|
|
52
|
+
return true;
|
|
53
|
+
})()
|
|
54
|
+
JS
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
autoload :BitbucketCloudPullDiffProductState,
|
|
7
|
+
"fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_product_state"
|
|
8
|
+
autoload :BitbucketCloudPullDiffRequestState,
|
|
9
|
+
"fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_request_state"
|
|
10
|
+
autoload :BitbucketCloudPullDiffState,
|
|
11
|
+
"fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_state"
|
|
12
|
+
autoload :BitbucketCloudPullResourceState,
|
|
13
|
+
"fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_state"
|
|
14
|
+
autoload :BitbucketCloudPullResources,
|
|
15
|
+
"fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resources"
|
|
16
|
+
|
|
17
|
+
module BitbucketCloudPullResourceModules
|
|
18
|
+
include BitbucketCloudPullDiffProductState
|
|
19
|
+
include BitbucketCloudPullDiffRequestState
|
|
20
|
+
include BitbucketCloudPullDiffState
|
|
21
|
+
include BitbucketCloudPullResourceState
|
|
22
|
+
include BitbucketCloudPullResources
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module BitbucketCloudPullResourceState
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def bitbucket_cloud_pull_resource_state_script
|
|
10
|
+
<<~JS
|
|
11
|
+
(() => {
|
|
12
|
+
if (location.pathname.endsWith("/diff") || location.pathname.endsWith("/diff/")) {
|
|
13
|
+
return (#{bitbucket_cloud_pull_diff_state_script});
|
|
14
|
+
}
|
|
15
|
+
return (#{bitbucket_cloud_pull_commit_resource_state_script});
|
|
16
|
+
})()
|
|
17
|
+
JS
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def bitbucket_cloud_pull_commit_resource_state_script
|
|
21
|
+
<<~'JS'
|
|
22
|
+
(() => {
|
|
23
|
+
const match = location.pathname.match(/^\/([^/]+)\/([^/]+)\/pull-requests\/(\d+)\/commits\/?$/);
|
|
24
|
+
if (!match) return { product: false, ready: false, loading: false, signature: '' };
|
|
25
|
+
const decode = (value) => { try { return decodeURIComponent(value); } catch (_error) { return ''; } };
|
|
26
|
+
const workspace = decode(match[1]);
|
|
27
|
+
const repository = decode(match[2]);
|
|
28
|
+
if (!workspace || !repository || workspace.includes('/') || repository.includes('/')) {
|
|
29
|
+
return { product: false, ready: false, loading: false, signature: '' };
|
|
30
|
+
}
|
|
31
|
+
const current = (((window.__initial_state__ || {}).section || {}).repository || {}).currentRepository || {};
|
|
32
|
+
const publicRepository = current.is_private === false || current.isPrivate === false || current.visibility === 'public';
|
|
33
|
+
const runtime = current.type === 'repository' && publicRepository &&
|
|
34
|
+
String(current.full_name || '').toLowerCase() === `${workspace}/${repository}`.toLowerCase();
|
|
35
|
+
const application = document.querySelector('meta[name="application-name"]');
|
|
36
|
+
const branded = /^Bitbucket$/i.test((application && application.content || '').trim());
|
|
37
|
+
const asset = Array.from(document.querySelectorAll('script[src], link[href]'))
|
|
38
|
+
.some((node) => {
|
|
39
|
+
const value = node.getAttribute('src') || node.getAttribute('href') || '';
|
|
40
|
+
return /(?:frontbucket|bbc-frontbucket-static|bitbucket)[^/]*\/(?:assets|static)\//i.test(value) ||
|
|
41
|
+
/\/(?:assets|static)\/[^?#]*(?:frontbucket|bitbucket)/i.test(value);
|
|
42
|
+
});
|
|
43
|
+
const header = document.querySelector('[data-testid="pr-header"]');
|
|
44
|
+
const product = runtime && branded && !!(header || asset);
|
|
45
|
+
if (!product) return { product: false, ready: false, loading: false, signature: '' };
|
|
46
|
+
|
|
47
|
+
const panel = document.querySelector('[role="tabpanel"][id^="pull-request-tabs-"]');
|
|
48
|
+
const wrapper = document.querySelector('[data-qa^="commit-hash-wrapper-"]');
|
|
49
|
+
const root = panel || (wrapper && wrapper.closest('table, [role="grid"]'));
|
|
50
|
+
|
|
51
|
+
const visible = (node) => {
|
|
52
|
+
let currentNode = node;
|
|
53
|
+
while (currentNode && currentNode !== document) {
|
|
54
|
+
const style = getComputedStyle(currentNode);
|
|
55
|
+
if (currentNode.hidden || style.display === 'none') return false;
|
|
56
|
+
currentNode = currentNode.parentElement;
|
|
57
|
+
}
|
|
58
|
+
const style = getComputedStyle(node);
|
|
59
|
+
return style.visibility !== 'hidden' && style.visibility !== 'collapse';
|
|
60
|
+
};
|
|
61
|
+
const seen = new Set();
|
|
62
|
+
const rows = Array.from(root ? root.querySelectorAll('a[aria-label^="Commit: "][href*="/commits/"]') : [])
|
|
63
|
+
.map((link) => {
|
|
64
|
+
const label = (link.getAttribute('aria-label') || '').trim().match(/^Commit:\s*([0-9a-f]{7,40})$/i);
|
|
65
|
+
const row = link.closest('tr, [role="row"]');
|
|
66
|
+
try {
|
|
67
|
+
const url = new URL(link.href, location.href);
|
|
68
|
+
const path = url.pathname.match(/^\/([^/]+)\/([^/]+)\/commits\/([0-9a-f]{40})\/?$/i);
|
|
69
|
+
if (!label || !row || !visible(row) || url.origin !== location.origin || !path) return null;
|
|
70
|
+
if (decode(path[1]) !== workspace || decode(path[2]) !== repository) return null;
|
|
71
|
+
if (!path[3].toLowerCase().startsWith(label[1].toLowerCase())) return null;
|
|
72
|
+
const hash = path[3].toLowerCase();
|
|
73
|
+
if (seen.has(hash)) return null;
|
|
74
|
+
seen.add(hash);
|
|
75
|
+
return { row, hash, href: url.href };
|
|
76
|
+
} catch (_error) {
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
}).filter(Boolean);
|
|
80
|
+
const loading = Array.from(document.querySelectorAll(
|
|
81
|
+
'[aria-busy="true"], [data-testid*="loading"], [data-testid*="spinner"], .loading, .spinner'
|
|
82
|
+
)).some(visible);
|
|
83
|
+
const signature = rows.map(({ row, hash, href }) =>
|
|
84
|
+
`${hash}:${href}:${(row.textContent || '').replace(/\s+/g, ' ').trim()}`
|
|
85
|
+
).join('|');
|
|
86
|
+
return { product: true, ready: !loading && rows.length > 0, loading, signature };
|
|
87
|
+
})()
|
|
88
|
+
JS
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module BitbucketCloudPullResources
|
|
7
|
+
BITBUCKET_CLOUD_PULL_RESOURCE_STABILIZATION_PROFILE = {
|
|
8
|
+
host: true,
|
|
9
|
+
path_query: ->(uri) { uri.path.match?(%r{\A/[^/]+/[^/]+/pull-requests/\d+/(?:commits|diff)/?\z}) },
|
|
10
|
+
strategy: :stabilize_bitbucket_cloud_pull_resource,
|
|
11
|
+
fallthrough: true,
|
|
12
|
+
notes: "Wait for product-matched Bitbucket Cloud pull-request commit or diff resources.",
|
|
13
|
+
tests: "spec/fetch_util/browser_stabilization_spec.rb"
|
|
14
|
+
}.freeze
|
|
15
|
+
|
|
16
|
+
private
|
|
17
|
+
|
|
18
|
+
def stabilize_bitbucket_cloud_pull_resource(page, deadline: stabilization_deadline)
|
|
19
|
+
last_signature = nil
|
|
20
|
+
stable_observations = 0
|
|
21
|
+
incomplete_signature = nil
|
|
22
|
+
incomplete_observations = 0
|
|
23
|
+
terminal_incomplete = false
|
|
24
|
+
observed = retry_until_timeout(
|
|
25
|
+
capped_timeout(6.0, deadline: deadline),
|
|
26
|
+
interval: 0.1,
|
|
27
|
+
deadline: deadline
|
|
28
|
+
) do
|
|
29
|
+
state = safe_evaluate(page, bitbucket_cloud_pull_resource_state_script, default: nil)
|
|
30
|
+
return false if state.is_a?(Hash) && state["product"] == false
|
|
31
|
+
next false unless state.is_a?(Hash) && state["product"]
|
|
32
|
+
|
|
33
|
+
unless state["ready"]
|
|
34
|
+
last_signature = nil
|
|
35
|
+
stable_observations = 0
|
|
36
|
+
if state["loading"]
|
|
37
|
+
incomplete_signature = nil
|
|
38
|
+
incomplete_observations = 0
|
|
39
|
+
elsif state["signature"] == incomplete_signature
|
|
40
|
+
incomplete_observations += 1
|
|
41
|
+
else
|
|
42
|
+
incomplete_signature = state["signature"]
|
|
43
|
+
incomplete_observations = 1
|
|
44
|
+
end
|
|
45
|
+
if incomplete_observations >= 10
|
|
46
|
+
terminal_incomplete = true
|
|
47
|
+
next true
|
|
48
|
+
end
|
|
49
|
+
next false
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
incomplete_signature = nil
|
|
53
|
+
incomplete_observations = 0
|
|
54
|
+
if state["signature"] == last_signature
|
|
55
|
+
stable_observations += 1
|
|
56
|
+
else
|
|
57
|
+
last_signature = state["signature"]
|
|
58
|
+
stable_observations = 1
|
|
59
|
+
end
|
|
60
|
+
stable_observations >= 3
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
return false if terminal_incomplete
|
|
64
|
+
|
|
65
|
+
fail_bitbucket_cloud_pull_diff_preparation(page) unless observed
|
|
66
|
+
|
|
67
|
+
settle_after_stabilization(0.5, deadline: deadline) if observed
|
|
68
|
+
!!observed
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module BitbucketCloudThreadState
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def bitbucket_cloud_thread_state_script
|
|
10
|
+
<<~'JS'
|
|
11
|
+
(() => {
|
|
12
|
+
const match = location.pathname.match(/^\/([^/]+)\/([^/]+)\/pull-requests\/(\d+)(?:\/overview)?\/?$/);
|
|
13
|
+
if (!match) return { product: false, ready: false, loading: false, signature: '' };
|
|
14
|
+
|
|
15
|
+
const decoded = (value) => {
|
|
16
|
+
try { return decodeURIComponent(value); } catch (_error) { return value; }
|
|
17
|
+
};
|
|
18
|
+
const community = `${decoded(match[1])}/${decoded(match[2])}`.toLowerCase();
|
|
19
|
+
const state = window.__initial_state__ || {};
|
|
20
|
+
const repository = (((state.section || {}).repository || {}).currentRepository) || null;
|
|
21
|
+
const publicRepository = !!repository && (
|
|
22
|
+
repository.is_private === false || repository.isPrivate === false || repository.visibility === 'public'
|
|
23
|
+
);
|
|
24
|
+
const runtime = publicRepository && repository.type === 'repository' &&
|
|
25
|
+
String(repository.full_name || '').trim().toLowerCase() === community;
|
|
26
|
+
const application = document.querySelector('meta[name="application-name"]');
|
|
27
|
+
const branded = /^Bitbucket$/i.test((application && application.content || '').trim());
|
|
28
|
+
const asset = Array.from(document.querySelectorAll('script[src], link[href]')).some((node) => {
|
|
29
|
+
const value = node.getAttribute('src') || node.getAttribute('href') || '';
|
|
30
|
+
return /(?:frontbucket|bbc-frontbucket-static|bitbucket)[^/]*\/(?:assets|static)\//i.test(value) ||
|
|
31
|
+
/\/(?:assets|static)\/[^?#]*(?:frontbucket|bitbucket)/i.test(value);
|
|
32
|
+
});
|
|
33
|
+
const header = document.querySelector('[data-testid="pr-header"]');
|
|
34
|
+
const root = header && (header.closest('main') || document.querySelector('main') || header.parentElement) ||
|
|
35
|
+
document.querySelector('main');
|
|
36
|
+
const opening = root && root.querySelector(
|
|
37
|
+
'#pull-request-description-panel, section[aria-label="Pull request description"], section[aria-label="Description"]'
|
|
38
|
+
);
|
|
39
|
+
const product = runtime && branded && !!(header || asset);
|
|
40
|
+
if (!product) return { product: false, ready: false, loading: false, signature: '' };
|
|
41
|
+
|
|
42
|
+
const nodeVisible = (node) => {
|
|
43
|
+
let current = node;
|
|
44
|
+
while (current && current !== document) {
|
|
45
|
+
const style = getComputedStyle(current);
|
|
46
|
+
if (current.hidden || style.display === 'none') return false;
|
|
47
|
+
current = current.parentElement;
|
|
48
|
+
}
|
|
49
|
+
const visibility = getComputedStyle(node).visibility;
|
|
50
|
+
return visibility !== 'hidden' && visibility !== 'collapse';
|
|
51
|
+
};
|
|
52
|
+
const subtreeVisible = (node) => {
|
|
53
|
+
if (nodeVisible(node)) return true;
|
|
54
|
+
return Array.from(node.querySelectorAll('*')).some((child) => {
|
|
55
|
+
const material = (child.textContent || '').trim() ||
|
|
56
|
+
child.matches('img[src], video, pre, table, ul, ol');
|
|
57
|
+
return material && nodeVisible(child);
|
|
58
|
+
});
|
|
59
|
+
};
|
|
60
|
+
const comments = Array.from(root.querySelectorAll('[data-testid="comment"]')).filter(subtreeVisible);
|
|
61
|
+
const owned = (node, selector) => Array.from(node.querySelectorAll(selector)).find(
|
|
62
|
+
(candidate) => candidate.closest('[data-testid="comment"]') === node
|
|
63
|
+
);
|
|
64
|
+
const loading = Array.from(root.querySelectorAll(
|
|
65
|
+
'[aria-busy="true"], [data-testid*="loading" i], [data-testid*="spinner" i], .spinner, .loading'
|
|
66
|
+
)).some(subtreeVisible);
|
|
67
|
+
const openingReady = !!opening && subtreeVisible(opening) && !!(
|
|
68
|
+
(opening.textContent || '').trim() || opening.querySelector('img[src], video, pre, table, ul, ol')
|
|
69
|
+
);
|
|
70
|
+
const commentBodies = comments.filter((node) => {
|
|
71
|
+
const body = owned(node, '[data-testid="comment-content"], comment-content');
|
|
72
|
+
return body && subtreeVisible(body) && !!((body.textContent || '').trim() || body.querySelector('img[src], pre, table'));
|
|
73
|
+
});
|
|
74
|
+
const explicitZero = /\b0\s+comments?\b/i.test(root.textContent || '');
|
|
75
|
+
const rowSignatures = commentBodies.map((node, index) => {
|
|
76
|
+
const link = owned(node, 'a[href*="#comment-"]');
|
|
77
|
+
return node.id || node.getAttribute('data-comment-id') || (link && link.href) ||
|
|
78
|
+
`${index}:${(node.textContent || '').trim().slice(0, 120)}`;
|
|
79
|
+
});
|
|
80
|
+
return {
|
|
81
|
+
product: true,
|
|
82
|
+
ready: !!header && openingReady && !loading && (commentBodies.length > 0 || explicitZero),
|
|
83
|
+
loading: loading,
|
|
84
|
+
signature: [openingReady, commentBodies.length, explicitZero, loading, rowSignatures.join('|')].join(':')
|
|
85
|
+
};
|
|
86
|
+
})()
|
|
87
|
+
JS
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module BitbucketCloudThreads
|
|
7
|
+
BITBUCKET_CLOUD_THREAD_STABILIZATION_PROFILE = {
|
|
8
|
+
host: true,
|
|
9
|
+
path_query: ->(uri) { uri.path.match?(%r{\A/[^/]+/[^/]+/pull-requests/\d+(?:/overview)?/?\z}) },
|
|
10
|
+
strategy: :stabilize_bitbucket_cloud_thread,
|
|
11
|
+
fallthrough: true,
|
|
12
|
+
notes: "Wait for product-matched Bitbucket Cloud pull-request conversations.",
|
|
13
|
+
tests: "spec/fetch_util/browser_stabilization_spec.rb"
|
|
14
|
+
}.freeze
|
|
15
|
+
|
|
16
|
+
private
|
|
17
|
+
|
|
18
|
+
def stabilize_bitbucket_cloud_thread(page, deadline: stabilization_deadline)
|
|
19
|
+
last_signature = nil
|
|
20
|
+
stable_observations = 0
|
|
21
|
+
incomplete_signature = nil
|
|
22
|
+
incomplete_observations = 0
|
|
23
|
+
terminal_incomplete = false
|
|
24
|
+
observed = retry_until_timeout(
|
|
25
|
+
capped_timeout(6.0, deadline: deadline),
|
|
26
|
+
interval: 0.1,
|
|
27
|
+
deadline: deadline
|
|
28
|
+
) do
|
|
29
|
+
state = safe_evaluate(page, bitbucket_cloud_thread_state_script, default: nil)
|
|
30
|
+
return false if state.is_a?(Hash) && state["product"] == false
|
|
31
|
+
next false unless state.is_a?(Hash) && state["product"]
|
|
32
|
+
|
|
33
|
+
unless state["ready"]
|
|
34
|
+
last_signature = nil
|
|
35
|
+
stable_observations = 0
|
|
36
|
+
if state["loading"]
|
|
37
|
+
incomplete_signature = nil
|
|
38
|
+
incomplete_observations = 0
|
|
39
|
+
elsif state["signature"] == incomplete_signature
|
|
40
|
+
incomplete_observations += 1
|
|
41
|
+
else
|
|
42
|
+
incomplete_signature = state["signature"]
|
|
43
|
+
incomplete_observations = 1
|
|
44
|
+
end
|
|
45
|
+
if incomplete_observations >= 10
|
|
46
|
+
terminal_incomplete = true
|
|
47
|
+
next true
|
|
48
|
+
end
|
|
49
|
+
next false
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
incomplete_signature = nil
|
|
53
|
+
incomplete_observations = 0
|
|
54
|
+
if state["signature"] == last_signature
|
|
55
|
+
stable_observations += 1
|
|
56
|
+
else
|
|
57
|
+
last_signature = state["signature"]
|
|
58
|
+
stable_observations = 1
|
|
59
|
+
end
|
|
60
|
+
stable_observations >= 3
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
return false if terminal_incomplete
|
|
64
|
+
|
|
65
|
+
settle_after_stabilization(0.5, deadline: deadline) if observed
|
|
66
|
+
!!observed
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
@@ -22,17 +22,18 @@ module FetchUtil
|
|
|
22
22
|
|
|
23
23
|
private
|
|
24
24
|
|
|
25
|
-
def stabilize_reddit(page)
|
|
26
|
-
retry_until_timeout(capped_timeout(3.0), interval: 0.1) do
|
|
25
|
+
def stabilize_reddit(page, deadline: stabilization_deadline)
|
|
26
|
+
ready = retry_until_timeout(capped_timeout(3.0, deadline: deadline), interval: 0.1, deadline: deadline) do
|
|
27
27
|
dismiss_reddit_cookie_dialog(page)
|
|
28
28
|
reddit_content_ready?(page)
|
|
29
29
|
end
|
|
30
30
|
|
|
31
|
-
settle_after_stabilization(0.25)
|
|
32
|
-
dismiss_reddit_cookie_dialog(page)
|
|
31
|
+
settle_after_stabilization(0.25, deadline: deadline)
|
|
32
|
+
dismiss_reddit_cookie_dialog(page) if stabilization_time_remaining?(deadline)
|
|
33
|
+
ready
|
|
33
34
|
end
|
|
34
35
|
|
|
35
|
-
def stabilize_ebay_search(page)
|
|
36
|
+
def stabilize_ebay_search(page, deadline: stabilization_deadline)
|
|
36
37
|
accepted_cookies = false
|
|
37
38
|
cookie_config = consent_config(
|
|
38
39
|
accept_labels: [
|
|
@@ -46,7 +47,7 @@ module FetchUtil
|
|
|
46
47
|
]
|
|
47
48
|
)
|
|
48
49
|
|
|
49
|
-
retry_until_timeout(capped_timeout(6.0), interval: 0.15) do
|
|
50
|
+
retry_until_timeout(capped_timeout(6.0, deadline: deadline), interval: 0.15, deadline: deadline) do
|
|
50
51
|
accepted_cookies ||= click_visible_button_by_text(
|
|
51
52
|
page,
|
|
52
53
|
consent_accept_labels(cookie_config),
|
|
@@ -66,15 +67,13 @@ module FetchUtil
|
|
|
66
67
|
state["itemCount"].to_i >= 4 || (state["challengeVisible"] ? 0.35 : false)
|
|
67
68
|
end
|
|
68
69
|
|
|
69
|
-
settle_after_stabilization(0.25) if accepted_cookies
|
|
70
|
+
settle_after_stabilization(0.25, deadline: deadline) if accepted_cookies
|
|
70
71
|
end
|
|
71
72
|
|
|
72
73
|
def reddit_content_ready?(page)
|
|
73
|
-
page
|
|
74
|
+
safe_evaluate(page, <<~JS, default: false)
|
|
74
75
|
!!document.querySelector('shreddit-post, faceplate-screen-reader-content, shreddit-comment, [data-testid="comment"]')
|
|
75
76
|
JS
|
|
76
|
-
rescue Ferrum::JavaScriptError
|
|
77
|
-
false
|
|
78
77
|
end
|
|
79
78
|
|
|
80
79
|
def dismiss_reddit_cookie_dialog(page)
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module FacebookStabilization
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def stabilize_facebook(page, deadline: stabilization_deadline)
|
|
10
|
+
wait_for_idle_or_content(page, deadline: deadline) if @wait_for_idle
|
|
11
|
+
return false unless stabilization_time_remaining?(deadline)
|
|
12
|
+
|
|
13
|
+
social_login_phase_pause(deadline: deadline)
|
|
14
|
+
return false unless stabilization_time_remaining?(deadline)
|
|
15
|
+
|
|
16
|
+
dismiss_facebook_cookie_dialog(page)
|
|
17
|
+
return false unless stabilization_time_remaining?(deadline)
|
|
18
|
+
|
|
19
|
+
social_login_phase_pause(deadline: deadline)
|
|
20
|
+
return false unless stabilization_time_remaining?(deadline)
|
|
21
|
+
|
|
22
|
+
retry_until_timeout(capped_timeout(5.0, deadline: deadline), deadline: deadline) do
|
|
23
|
+
dismiss_facebook_login_dialog(page)
|
|
24
|
+
end
|
|
25
|
+
return false unless stabilization_time_remaining?(deadline)
|
|
26
|
+
|
|
27
|
+
social_login_phase_pause(deadline: deadline)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def dismiss_facebook_cookie_dialog(page)
|
|
31
|
+
config = consent_config(
|
|
32
|
+
accept_labels: ["decline optional cookies", "optionale cookies ablehnen", "refuser les cookies optionnels",
|
|
33
|
+
"rechazar cookies opcionales", "rifiuta i cookie opzionali"],
|
|
34
|
+
fallback_labels: ["allow all cookies", "alle cookies erlauben", "autoriser tous les cookies", "permitir todas las cookies", "consenti tutti i cookie"]
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
click_visible_button_by_text(
|
|
38
|
+
page,
|
|
39
|
+
consent_accept_labels(config),
|
|
40
|
+
consent_fallback_labels(config),
|
|
41
|
+
selectors: consent_button_selectors(config)
|
|
42
|
+
)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def dismiss_facebook_login_dialog(page)
|
|
46
|
+
dismiss_overlay_dialog(
|
|
47
|
+
page,
|
|
48
|
+
close_selectors: ['[aria-label="Close"]', '[aria-label="close"]'],
|
|
49
|
+
dialog_selectors: ['[role="dialog"]', '[aria-modal="true"]'],
|
|
50
|
+
dialog_pattern: "log in|sign up|create (?:new )?account|see more from"
|
|
51
|
+
)
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|