fetch_util 0.5.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +49 -0
- data/README.md +3 -1
- data/Rakefile +0 -2
- data/lib/fetch_util/assets/extract.js +1 -1
- data/lib/fetch_util/browser/interaction_helpers/consent_helpers.rb +12 -17
- data/lib/fetch_util/browser/interaction_helpers/timing_helpers.rb +42 -15
- data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_commit_details_script.rb +48 -0
- data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_product_state_script.rb +80 -0
- data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_request_helpers_script.rb +53 -0
- data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_request_state_script.rb +118 -0
- data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_stabilization.rb +54 -0
- data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_state.rb +27 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_product_state.rb +55 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_request_state.rb +100 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_state.rb +59 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_modules.rb +26 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_state.rb +93 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resources.rb +73 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_thread_state.rb +92 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_threads.rb +71 -0
- data/lib/fetch_util/browser/site_stabilization/community_and_marketplace.rb +9 -10
- data/lib/fetch_util/browser/site_stabilization/facebook_stabilization.rb +56 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_change_product_state_script.rb +97 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_change_request_state_script.rb +75 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_change_stabilization.rb +37 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_change_state.rb +39 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_fetch_state_script.rb +31 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_product_state_script.rb +76 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_request_state_script.rb +111 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_route_state_script.rb +71 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_stabilization.rb +37 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_state.rb +42 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_validation_state_script.rb +49 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_pull_resource_product_state.rb +88 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_pull_resource_state.rb +109 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_pull_resources.rb +30 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_stabilization.rb +60 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_thread_product_state.rb +74 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_thread_timeline_state.rb +69 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_threads.rb +28 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_visibility_script.rb +28 -0
- data/lib/fetch_util/browser/site_stabilization/github_pull_resource_stabilization.rb +31 -0
- data/lib/fetch_util/browser/site_stabilization/github_pull_resources.rb +105 -0
- data/lib/fetch_util/browser/site_stabilization/github_threads.rb +82 -0
- data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state.rb +17 -0
- data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state_script.rb +98 -0
- data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_visibility_script.rb +43 -0
- data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resources.rb +28 -0
- data/lib/fetch_util/browser/site_stabilization/gitlab_repo.rb +3 -11
- data/lib/fetch_util/browser/site_stabilization/gitlab_stabilization.rb +33 -0
- data/lib/fetch_util/browser/site_stabilization/gitlab_threads.rb +92 -0
- data/lib/fetch_util/browser/site_stabilization/social_platforms.rb +19 -38
- data/lib/fetch_util/browser/site_stabilization/travel_and_lodging.rb +3 -3
- data/lib/fetch_util/browser/site_stabilization.rb +89 -0
- data/lib/fetch_util/browser/stabilization/anubis.rb +4 -5
- data/lib/fetch_util/browser/stabilization/page_flow.rb +71 -27
- data/lib/fetch_util/browser/stabilization/spa_hydration.rb +7 -5
- data/lib/fetch_util/browser.rb +88 -17
- data/lib/fetch_util/cli.rb +69 -23
- data/lib/fetch_util/extractor.rb +8 -2
- data/lib/fetch_util/fetcher.rb +144 -43
- data/lib/fetch_util/http_redirect_client.rb +161 -0
- data/lib/fetch_util/parallel_fetcher.rb +125 -28
- data/lib/fetch_util/raw_docs_fallback.rb +158 -21
- data/lib/fetch_util/regulatory/cache_store.rb +47 -14
- data/lib/fetch_util/regulatory/fetch_records.rb +10 -7
- data/lib/fetch_util/regulatory/headers.rb +8 -1
- data/lib/fetch_util/regulatory/http_client.rb +2 -97
- data/lib/fetch_util/regulatory/orchestration.rb +10 -6
- data/lib/fetch_util/regulatory/page.rb +8 -11
- data/lib/fetch_util/regulatory/robots.rb +2 -10
- data/lib/fetch_util/regulatory/source_selection.rb +13 -3
- data/lib/fetch_util/regulatory/tdm_page.rb +9 -4
- data/lib/fetch_util/regulatory/tdm_policy.rb +16 -9
- data/lib/fetch_util/regulatory/tdm_rep.rb +4 -3
- data/lib/fetch_util/regulatory/tdm_support.rb +10 -6
- data/lib/fetch_util/regulatory.rb +5 -1
- data/lib/fetch_util/request_log.rb +24 -3
- data/lib/fetch_util/result.rb +37 -89
- data/lib/fetch_util/search_transport.rb +102 -35
- data/lib/fetch_util/searcher.rb +15 -13
- data/lib/fetch_util/version.rb +1 -1
- data/lib/fetch_util.rb +4 -1
- metadata +66 -2
data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_validation_state_script.rb
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module GerritFileResourceValidationStateScript
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def gerrit_file_resource_validation_state_script
|
|
10
|
+
<<~'JS'
|
|
11
|
+
const objectRecord = (value) => value && typeof value === "object" && !Array.isArray(value);
|
|
12
|
+
const validTextLines = (value) => value == null ||
|
|
13
|
+
(Array.isArray(value) && value.every((line) => typeof line === "string"));
|
|
14
|
+
const validIntralineEdits = (value) => value == null || (Array.isArray(value) && value.every(
|
|
15
|
+
(edit) => Array.isArray(edit) && edit.length === 2 && edit.every(
|
|
16
|
+
(number) => Number.isInteger(number) && number >= 0
|
|
17
|
+
)
|
|
18
|
+
));
|
|
19
|
+
const validDiffBlock = (block) => objectRecord(block) && ["ab", "a", "b"].every(
|
|
20
|
+
(field) => validTextLines(block[field])
|
|
21
|
+
) && ["edit_a", "edit_b"].every(
|
|
22
|
+
(field) => validIntralineEdits(block[field])
|
|
23
|
+
) && (block.skip == null || typeof block.skip === "number" || objectRecord(block.skip));
|
|
24
|
+
const validateDiff = (diff, filePath) => {
|
|
25
|
+
if (diff.content != null &&
|
|
26
|
+
(!Array.isArray(diff.content) || !diff.content.every(validDiffBlock))) {
|
|
27
|
+
throw new Error("invalid Gerrit diff content payload");
|
|
28
|
+
}
|
|
29
|
+
if (diff.diff_header != null && typeof diff.diff_header !== "string" &&
|
|
30
|
+
(!Array.isArray(diff.diff_header) ||
|
|
31
|
+
!diff.diff_header.every((line) => typeof line === "string"))) {
|
|
32
|
+
throw new Error("invalid Gerrit diff header payload");
|
|
33
|
+
}
|
|
34
|
+
if (diff.meta_a != null && !objectRecord(diff.meta_a)) {
|
|
35
|
+
throw new Error("invalid Gerrit base metadata payload");
|
|
36
|
+
}
|
|
37
|
+
if (diff.meta_b != null && !objectRecord(diff.meta_b)) {
|
|
38
|
+
throw new Error("invalid Gerrit target metadata payload");
|
|
39
|
+
}
|
|
40
|
+
if (diff.meta_b && diff.meta_b.name && diff.meta_b.name !== filePath) {
|
|
41
|
+
throw new Error("Gerrit API diff target identity mismatch");
|
|
42
|
+
}
|
|
43
|
+
};
|
|
44
|
+
JS
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module GiteaFamilyPullResourceProductState
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def gitea_family_pull_resource_product_state_script
|
|
10
|
+
gitea_family_visibility_script + <<~'JS'
|
|
11
|
+
const visibleNodes = (node) => [node, ...node.querySelectorAll('*')].filter(visible);
|
|
12
|
+
const ownText = (node) => Array.from(node.childNodes)
|
|
13
|
+
.filter((child) => child.nodeType === Node.TEXT_NODE)
|
|
14
|
+
.map((child) => child.textContent || '').join(' ').trim();
|
|
15
|
+
const visibleText = (node) => visibleNodes(node).map(ownText).filter(Boolean).join(' ');
|
|
16
|
+
const visibleMaterial = (node, excluded) => !!node && visibleNodes(node).some((candidate) => {
|
|
17
|
+
if (excluded && candidate.closest(excluded)) return false;
|
|
18
|
+
return !!ownText(candidate) || candidate.matches('table, pre, code, img, ins, del, .diff-line');
|
|
19
|
+
});
|
|
20
|
+
const runtime = window.config || {};
|
|
21
|
+
const prefix = String(runtime.appSubUrl || '').replace(/^\/+|\/+$/g, '');
|
|
22
|
+
const routePrefix = prefix ? `/${prefix}` : '';
|
|
23
|
+
const escaped = routePrefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
24
|
+
const match = location.pathname.match(new RegExp(
|
|
25
|
+
`^${escaped}/([^/]+)/([^/]+)/pulls/(\\d+)/(commits|files)(?:/([^/]+))?\\/?$`
|
|
26
|
+
));
|
|
27
|
+
if (!match) return null;
|
|
28
|
+
const routeSurface = match[4];
|
|
29
|
+
const detail = match[5] || '';
|
|
30
|
+
const validDetail = routeSurface === 'commits'
|
|
31
|
+
? (!detail || detail === 'list' || /^[a-f0-9]{4,64}$/i.test(detail))
|
|
32
|
+
: (!detail || /^[a-f0-9]{4,64}$/i.test(detail) ||
|
|
33
|
+
/^[a-f0-9]{7,64}\.\.?\.[a-f0-9]{7,64}$/i.test(detail));
|
|
34
|
+
if (!validDetail) return null;
|
|
35
|
+
const surface = routeSurface === 'commits' && detail && detail !== 'list' ? 'files' : routeSurface;
|
|
36
|
+
let runtimeMatch = false;
|
|
37
|
+
try {
|
|
38
|
+
const app = new URL(runtime.appUrl || '', location.href);
|
|
39
|
+
runtimeMatch = !!runtime.appUrl && app.origin === location.origin &&
|
|
40
|
+
app.pathname.replace(/\/$/, '') === routePrefix;
|
|
41
|
+
} catch (_error) { runtimeMatch = false; }
|
|
42
|
+
const brandValues = Array.from(document.querySelectorAll(
|
|
43
|
+
'meta[name="author"], meta[name="generator"], meta[property="og:site_name"]'
|
|
44
|
+
)).map((node) => (node.content || '').trim());
|
|
45
|
+
Array.from(document.querySelectorAll('footer a[href]')).forEach((node) => {
|
|
46
|
+
const text = (node.textContent || '').trim();
|
|
47
|
+
const href = node.getAttribute('href') || '';
|
|
48
|
+
if (/powered by.*\b(?:forgejo|gitea)\b/i.test(text) || /\b(?:forgejo|gitea)\b/i.test(href)) {
|
|
49
|
+
brandValues.push(`${text} ${href}`);
|
|
50
|
+
}
|
|
51
|
+
});
|
|
52
|
+
brandValues.push((runtime.assetVersionEncoded || '').trim());
|
|
53
|
+
const branded = /\b(?:forgejo|gitea)\b/i.test(brandValues.join(' '));
|
|
54
|
+
let runtimeAsset = false;
|
|
55
|
+
try {
|
|
56
|
+
const assetUrl = new URL(runtime.assetUrlPrefix || '', location.href);
|
|
57
|
+
const expectedPath = `${routePrefix}/assets`;
|
|
58
|
+
runtimeAsset = !!runtime.assetUrlPrefix && assetUrl.origin === location.origin &&
|
|
59
|
+
assetUrl.pathname.replace(/\/$/, '') === expectedPath &&
|
|
60
|
+
Array.from(document.querySelectorAll('script[src], link[href]')).some((node) => {
|
|
61
|
+
const value = node.getAttribute('src') || node.getAttribute('href') || '';
|
|
62
|
+
const url = new URL(value, location.href);
|
|
63
|
+
return url.origin === location.origin && url.pathname.startsWith(`${expectedPath}/`);
|
|
64
|
+
});
|
|
65
|
+
} catch (_error) { runtimeAsset = false; }
|
|
66
|
+
const rootSelectors = surface === 'commits'
|
|
67
|
+
? ['.page-content.repository.view.issue.pull.commits', '.repository.view.issue.pull.commits',
|
|
68
|
+
'.page-content.repository.view.issue.pull']
|
|
69
|
+
: ['.page-content.repository.view.issue.pull.files.diff', '.repository.view.issue.pull.files.diff',
|
|
70
|
+
'.page-content.repository.view.issue.pull'];
|
|
71
|
+
let root = null;
|
|
72
|
+
rootSelectors.some((selector) => { root = document.querySelector(selector); return !!root; });
|
|
73
|
+
const empty = root && Array.from(root.querySelectorAll(
|
|
74
|
+
'.ui.message.empty, .empty-state, .nothing-here, .no-results, [data-testid="empty-state"]'
|
|
75
|
+
)).find((node) => visible(node) && (node.textContent || '').trim());
|
|
76
|
+
const structure = surface === 'commits'
|
|
77
|
+
? !!(root && (root.querySelector(
|
|
78
|
+
'#commits-table > tbody.commit-list, .commit-group .commits .commit'
|
|
79
|
+
) || empty))
|
|
80
|
+
: !!(root && (root.querySelector('#diff-container, #diff-file-boxes, #diff-file-tree') || empty));
|
|
81
|
+
const product = runtimeMatch && structure && (branded || runtimeAsset);
|
|
82
|
+
if (!product) return { product: false, ready: false, signature: '' };
|
|
83
|
+
JS
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module GiteaFamilyPullResourceState
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def gitea_family_pull_resource_state_script
|
|
10
|
+
<<~'JS'
|
|
11
|
+
const loading = Array.from(root.querySelectorAll(
|
|
12
|
+
'[aria-busy="true"], .is-loading, .tw-loading, .ui.active.loader'
|
|
13
|
+
)).some(visible);
|
|
14
|
+
const rows = surface === 'commits'
|
|
15
|
+
? Array.from(root.querySelectorAll(
|
|
16
|
+
'#commits-table > tbody.commit-list > tr, .commit-group .commits .commit'
|
|
17
|
+
))
|
|
18
|
+
.filter((row) => visibleMaterial(row))
|
|
19
|
+
: [];
|
|
20
|
+
const rowSignatures = rows.map((row) => [
|
|
21
|
+
row.querySelector('.copy-commit-id[data-clipboard-text]')?.getAttribute('data-clipboard-text') || '',
|
|
22
|
+
row.querySelector('a[href*="/commit/"]')?.getAttribute('href') || '',
|
|
23
|
+
visibleText(row)
|
|
24
|
+
].join('|'));
|
|
25
|
+
const pageData = runtime.pageData || {};
|
|
26
|
+
const treeData = pageData.DiffFileTree || pageData.diffFileTree || [];
|
|
27
|
+
const treeRoots = treeData.TreeRoot || treeData.treeRoot || treeData;
|
|
28
|
+
const treeIds = [];
|
|
29
|
+
const visit = (nodes) => (Array.isArray(nodes) ? nodes : [nodes]).filter(Boolean).forEach((node) => {
|
|
30
|
+
const children = node.Children || node.children || [];
|
|
31
|
+
if (children.length) return visit(children);
|
|
32
|
+
const identity = String(node.NameHash || node.nameHash || '');
|
|
33
|
+
if (identity) treeIds.push(identity.startsWith('diff-') ? identity : `diff-${identity}`);
|
|
34
|
+
});
|
|
35
|
+
const forgejoFiles = (pageData.diffFileInfo || {}).files || [];
|
|
36
|
+
if (forgejoFiles.length) {
|
|
37
|
+
forgejoFiles.forEach((file) => {
|
|
38
|
+
const identity = String(file.NameHash || file.nameHash || '');
|
|
39
|
+
if (identity) treeIds.push(identity.startsWith('diff-') ? identity : `diff-${identity}`);
|
|
40
|
+
});
|
|
41
|
+
} else {
|
|
42
|
+
visit(treeRoots);
|
|
43
|
+
}
|
|
44
|
+
const boxes = surface === 'files'
|
|
45
|
+
? Array.from(root.querySelectorAll(
|
|
46
|
+
'#diff-file-boxes > .diff-file-box, .diff-file-box.file-content[id^="diff-"]'
|
|
47
|
+
)).filter((box, index, all) => box.id !== 'diff-incomplete' && visibleMaterial(box) &&
|
|
48
|
+
!all.some((other, otherIndex) => otherIndex < index && other.contains(box)))
|
|
49
|
+
: [];
|
|
50
|
+
const boxSignatures = boxes.map((box) => [
|
|
51
|
+
box.id || '',
|
|
52
|
+
box.getAttribute('data-new-filename') || '',
|
|
53
|
+
box.getAttribute('data-old-filename') || '',
|
|
54
|
+
visibleText(box)
|
|
55
|
+
].join('|'));
|
|
56
|
+
const deferred = surface === 'files'
|
|
57
|
+
? Array.from(root.querySelectorAll(
|
|
58
|
+
'#diff-show-more-files[data-href], .diff-load-button[data-href], ' +
|
|
59
|
+
'[data-global-click="diffLoadFileBody"][data-href]'
|
|
60
|
+
)).filter(visible).map((node) => node.getAttribute('data-href') || '')
|
|
61
|
+
: [];
|
|
62
|
+
let selectedId = '';
|
|
63
|
+
if (surface === 'files') {
|
|
64
|
+
try { selectedId = decodeURIComponent((location.hash || '').replace(/^#/, '')); }
|
|
65
|
+
catch (_error) { selectedId = ''; }
|
|
66
|
+
}
|
|
67
|
+
const selected = selectedId && boxes.find((box) => box.id === selectedId);
|
|
68
|
+
const selectedBody = selected && selected.querySelector(
|
|
69
|
+
'.diff-file-body, .file-body, .diff-rendered, .image-diff, .binary'
|
|
70
|
+
);
|
|
71
|
+
const excluded = 'button, [role="button"], .diff-load-button, [data-global-click="diffLoadFileBody"]';
|
|
72
|
+
const selectedDeferred = !!(selected && Array.from(selected.querySelectorAll(
|
|
73
|
+
'.diff-load-button[data-href], [data-global-click="diffLoadFileBody"][data-href]'
|
|
74
|
+
)).find(visible));
|
|
75
|
+
const selectedStructured = !!selectedBody && visibleNodes(selectedBody).some((candidate) => {
|
|
76
|
+
if (candidate.closest(excluded)) return false;
|
|
77
|
+
return candidate.matches('table, pre, code, img, ins, del, .diff-line');
|
|
78
|
+
});
|
|
79
|
+
const selectedLoaded = !!selectedBody && visibleMaterial(selectedBody, excluded) &&
|
|
80
|
+
(!selectedDeferred || selectedStructured);
|
|
81
|
+
const selectedIndexed = !!selectedId && treeIds.includes(selectedId);
|
|
82
|
+
const selectedTerminal = !selectedId || selectedLoaded || selectedDeferred || (!selected && selectedIndexed);
|
|
83
|
+
const material = surface === 'commits'
|
|
84
|
+
? rows.length > 0 || !!empty
|
|
85
|
+
: treeIds.length > 0 || boxes.length > 0 || !!empty;
|
|
86
|
+
return {
|
|
87
|
+
product: true,
|
|
88
|
+
ready: !loading && material && selectedTerminal,
|
|
89
|
+
loading,
|
|
90
|
+
signature: [
|
|
91
|
+
surface,
|
|
92
|
+
rowSignatures.join('\u001e'),
|
|
93
|
+
treeIds.join('\u001e'),
|
|
94
|
+
boxSignatures.join('\u001e'),
|
|
95
|
+
deferred.join('\u001e'),
|
|
96
|
+
selectedId,
|
|
97
|
+
selectedLoaded,
|
|
98
|
+
selectedDeferred,
|
|
99
|
+
!!empty,
|
|
100
|
+
loading
|
|
101
|
+
].join(':')
|
|
102
|
+
};
|
|
103
|
+
})()
|
|
104
|
+
JS
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module GiteaFamilyPullResources
|
|
7
|
+
GITEA_FAMILY_PULL_RESOURCE_STABILIZATION_PROFILE = {
|
|
8
|
+
host: true,
|
|
9
|
+
path_query: lambda { |uri|
|
|
10
|
+
uri.path.match?(%r{/(?:[^/]+/){2}pulls/\d+/(?:commits(?:/(?:list|[a-f\d]{4,64}))?|files(?:/(?:[a-f\d]{4,64}|[a-f\d]{7,64}\.\.?\.[a-f\d]{7,64}))?)/?\z}i)
|
|
11
|
+
},
|
|
12
|
+
strategy: :stabilize_gitea_family_pull_resource,
|
|
13
|
+
fallthrough: true,
|
|
14
|
+
notes: "Wait for host-agnostic Gitea and Forgejo pull-request resources.",
|
|
15
|
+
tests: "spec/fetch_util/browser_stabilization_spec.rb"
|
|
16
|
+
}.freeze
|
|
17
|
+
|
|
18
|
+
private
|
|
19
|
+
|
|
20
|
+
def stabilize_gitea_family_pull_resource(page, deadline: stabilization_deadline)
|
|
21
|
+
stabilize_gitea_family_state(
|
|
22
|
+
page,
|
|
23
|
+
gitea_family_pull_resource_product_state_script + gitea_family_pull_resource_state_script,
|
|
24
|
+
deadline: deadline
|
|
25
|
+
)
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module GiteaFamilyStabilization
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def stabilize_gitea_family_state(page, state_script, deadline: stabilization_deadline)
|
|
10
|
+
last_signature = nil
|
|
11
|
+
stable_observations = 0
|
|
12
|
+
incomplete_signature = nil
|
|
13
|
+
incomplete_observations = 0
|
|
14
|
+
terminal_incomplete = false
|
|
15
|
+
observed = retry_until_timeout(
|
|
16
|
+
capped_timeout(6.0, deadline: deadline),
|
|
17
|
+
interval: 0.1,
|
|
18
|
+
deadline: deadline
|
|
19
|
+
) do
|
|
20
|
+
state = safe_evaluate(page, state_script, default: nil)
|
|
21
|
+
return false if state.is_a?(Hash) && state["product"] == false
|
|
22
|
+
next false unless state.is_a?(Hash) && state["product"]
|
|
23
|
+
|
|
24
|
+
unless state["ready"]
|
|
25
|
+
if state["loading"]
|
|
26
|
+
incomplete_signature = nil
|
|
27
|
+
incomplete_observations = 0
|
|
28
|
+
elsif state["signature"] == incomplete_signature
|
|
29
|
+
incomplete_observations += 1
|
|
30
|
+
else
|
|
31
|
+
incomplete_signature = state["signature"]
|
|
32
|
+
incomplete_observations = 1
|
|
33
|
+
end
|
|
34
|
+
if incomplete_observations >= 10
|
|
35
|
+
terminal_incomplete = true
|
|
36
|
+
next true
|
|
37
|
+
end
|
|
38
|
+
next false
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
incomplete_signature = nil
|
|
42
|
+
incomplete_observations = 0
|
|
43
|
+
if state["signature"] == last_signature
|
|
44
|
+
stable_observations += 1
|
|
45
|
+
else
|
|
46
|
+
last_signature = state["signature"]
|
|
47
|
+
stable_observations = 1
|
|
48
|
+
end
|
|
49
|
+
stable_observations >= 3
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
return false if terminal_incomplete
|
|
53
|
+
|
|
54
|
+
settle_after_stabilization(0.5, deadline: deadline) if observed
|
|
55
|
+
!!observed
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module GiteaFamilyThreadProductState
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def gitea_family_thread_product_state_script
|
|
10
|
+
gitea_family_visibility_script + <<~'JS'
|
|
11
|
+
const runtime = window.config || {};
|
|
12
|
+
const prefix = String(runtime.appSubUrl || '').replace(/^\/+|\/+$/g, '');
|
|
13
|
+
const routePrefix = prefix ? `/${prefix}` : '';
|
|
14
|
+
const escaped = routePrefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
15
|
+
const match = location.pathname.match(new RegExp(
|
|
16
|
+
`^${escaped}/([^/]+)/([^/]+)/(issues|pulls)/(\\d+)\\/?$`
|
|
17
|
+
));
|
|
18
|
+
if (!match) return null;
|
|
19
|
+
let runtimeMatch = false;
|
|
20
|
+
try {
|
|
21
|
+
const app = new URL(runtime.appUrl || '', location.href);
|
|
22
|
+
runtimeMatch = !!runtime.appUrl && app.origin === location.origin &&
|
|
23
|
+
app.pathname.replace(/\/$/, '') === routePrefix;
|
|
24
|
+
} catch (_error) { runtimeMatch = false; }
|
|
25
|
+
const brandValues = Array.from(document.querySelectorAll(
|
|
26
|
+
'meta[name="author"], meta[name="generator"], meta[property="og:site_name"]'
|
|
27
|
+
)).map((node) => (node.content || '').trim());
|
|
28
|
+
Array.from(document.querySelectorAll('footer a[href]')).forEach((node) => {
|
|
29
|
+
const value = `${node.textContent || ''} ${node.getAttribute('href') || ''}`;
|
|
30
|
+
if (/\b(?:forgejo|gitea)\b/i.test(value)) brandValues.push(value);
|
|
31
|
+
});
|
|
32
|
+
brandValues.push((runtime.assetVersionEncoded || '').trim());
|
|
33
|
+
const branded = /\b(?:forgejo|gitea)\b/i.test(brandValues.join(' '));
|
|
34
|
+
let runtimeAsset = false;
|
|
35
|
+
try {
|
|
36
|
+
const assetUrl = new URL(runtime.assetUrlPrefix || '', location.href);
|
|
37
|
+
const expectedPath = `${routePrefix}/assets`;
|
|
38
|
+
runtimeAsset = !!runtime.assetUrlPrefix && assetUrl.origin === location.origin &&
|
|
39
|
+
assetUrl.pathname.replace(/\/$/, '') === expectedPath &&
|
|
40
|
+
Array.from(document.querySelectorAll('script[src], link[href]')).some((node) => {
|
|
41
|
+
const value = node.getAttribute('src') || node.getAttribute('href') || '';
|
|
42
|
+
const url = new URL(value, location.href);
|
|
43
|
+
return url.origin === location.origin && url.pathname.startsWith(`${expectedPath}/`);
|
|
44
|
+
});
|
|
45
|
+
} catch (_error) { runtimeAsset = false; }
|
|
46
|
+
const root = document.querySelector(
|
|
47
|
+
'.page-content.repository.view.issue .issue-content, .repository.view.issue .issue-content, .page-content .issue-content'
|
|
48
|
+
);
|
|
49
|
+
const opening = root && root.querySelector(
|
|
50
|
+
'.issue-content-left .timeline-item.comment.issue-content-comment[id^="issue-"], ' +
|
|
51
|
+
'.issue-content-left .timeline-item.comment.issue-content-comment, ' +
|
|
52
|
+
'.issue-content-left .timeline-item.comment.first[id^="issue-"], ' +
|
|
53
|
+
'.issue-content-left .timeline-item.comment.first, ' +
|
|
54
|
+
'.comment-list .timeline-item.comment.issue-content-comment, ' +
|
|
55
|
+
'.comment-list .timeline-item.comment.first'
|
|
56
|
+
);
|
|
57
|
+
const openingBody = opening && opening.querySelector('.comment-body');
|
|
58
|
+
const rendered = openingBody && openingBody.querySelector('.render-content.markup');
|
|
59
|
+
const empty = openingBody && openingBody.querySelector('.no-content');
|
|
60
|
+
const attachment = openingBody && Array.from(openingBody.querySelectorAll(
|
|
61
|
+
'.dropzone-attachments a[href], .attachments a[href]'
|
|
62
|
+
)).some(visible);
|
|
63
|
+
const structure = !!(root && opening && openingBody &&
|
|
64
|
+
root.querySelector('.issue-content-left .comment-list, .issue-content-left.comment-list') &&
|
|
65
|
+
opening.querySelector('.content.comment-container, .comment-body') &&
|
|
66
|
+
((visible(rendered) && (rendered.innerText || '').trim()) || visible(empty) || attachment));
|
|
67
|
+
const product = runtimeMatch && structure && (branded || runtimeAsset);
|
|
68
|
+
if (!product) return { product: false, ready: false, signature: '' };
|
|
69
|
+
JS
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module GiteaFamilyThreadTimelineState
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def gitea_family_thread_timeline_state_script
|
|
10
|
+
<<~'JS'
|
|
11
|
+
const candidates = Array.from(root.querySelectorAll([
|
|
12
|
+
'.issue-content-left .timeline-item-group',
|
|
13
|
+
'.issue-content-left .timeline-item',
|
|
14
|
+
'.comment-list .timeline-item-group',
|
|
15
|
+
'.comment-list .timeline-item'
|
|
16
|
+
].join(', ')));
|
|
17
|
+
const rows = [];
|
|
18
|
+
candidates.forEach((node) => {
|
|
19
|
+
const group = node.closest('.timeline-item-group');
|
|
20
|
+
const row = group && root.contains(group) ? group : node;
|
|
21
|
+
if (subtreeHidden(row) || row.matches(
|
|
22
|
+
'.pull-merge-box, .timeline-item.comment.merge.box, .form, #timeline-comments-end'
|
|
23
|
+
)) return;
|
|
24
|
+
if (rows.some((existing) => existing === row || existing.contains(row))) return;
|
|
25
|
+
rows.push(row);
|
|
26
|
+
});
|
|
27
|
+
const author = opening.querySelector(
|
|
28
|
+
'.comment-header .author, .timeline-avatar + * .author, ' +
|
|
29
|
+
'.comment-header-left a.tw-font-semibold, .comment-header-left span.tw-font-semibold, ' +
|
|
30
|
+
'.comment-header a.tw-font-semibold, a.tw-font-semibold, .author'
|
|
31
|
+
);
|
|
32
|
+
const comments = rows.filter((row) => !!row.querySelector('.comment-body'));
|
|
33
|
+
const events = rows.length - comments.length;
|
|
34
|
+
const loading = Array.from(root.querySelectorAll(
|
|
35
|
+
'[aria-busy="true"], .is-loading, .tw-loading, .ui.active.loader'
|
|
36
|
+
)).some(visible);
|
|
37
|
+
const openingReady = visible(openingBody) && !!(
|
|
38
|
+
(rendered && (rendered.innerText || '').trim()) ||
|
|
39
|
+
(openingBody && openingBody.querySelector('.no-content')) ||
|
|
40
|
+
Array.from(openingBody.querySelectorAll('.dropzone-attachments, .attachments')).some(visible)
|
|
41
|
+
);
|
|
42
|
+
const rowSignatures = rows.map((row) => [
|
|
43
|
+
row.id || '',
|
|
44
|
+
row.className || '',
|
|
45
|
+
Array.from(row.querySelectorAll('relative-time[datetime], time[datetime]'))
|
|
46
|
+
.map((time) => time.getAttribute('datetime') || '').join(','),
|
|
47
|
+
(row.innerText || '').trim()
|
|
48
|
+
].join('|')).join('\u001e');
|
|
49
|
+
return {
|
|
50
|
+
product: true,
|
|
51
|
+
ready: !loading && openingReady,
|
|
52
|
+
loading,
|
|
53
|
+
signature: [
|
|
54
|
+
rows.length,
|
|
55
|
+
comments.length,
|
|
56
|
+
events,
|
|
57
|
+
visible(author) ? (author.textContent || '').trim() : '',
|
|
58
|
+
openingReady,
|
|
59
|
+
loading,
|
|
60
|
+
rowSignatures
|
|
61
|
+
].join(':')
|
|
62
|
+
};
|
|
63
|
+
})()
|
|
64
|
+
JS
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module GiteaFamilyThreads
|
|
7
|
+
GITEA_FAMILY_THREAD_STABILIZATION_PROFILE = {
|
|
8
|
+
host: true,
|
|
9
|
+
path_query: ->(uri) { uri.path.match?(%r{/(?:[^/]+/){2}(?:issues|pulls)/\d+/?\z}) },
|
|
10
|
+
strategy: :stabilize_gitea_family_thread,
|
|
11
|
+
fallthrough: true,
|
|
12
|
+
notes: "Wait for host-agnostic Gitea and Forgejo issue and pull-request conversations.",
|
|
13
|
+
tests: "spec/fetch_util/browser_stabilization_spec.rb"
|
|
14
|
+
}.freeze
|
|
15
|
+
|
|
16
|
+
private
|
|
17
|
+
|
|
18
|
+
def stabilize_gitea_family_thread(page, deadline: stabilization_deadline)
|
|
19
|
+
stabilize_gitea_family_state(
|
|
20
|
+
page,
|
|
21
|
+
gitea_family_thread_product_state_script + gitea_family_thread_timeline_state_script,
|
|
22
|
+
deadline: deadline
|
|
23
|
+
)
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module GiteaFamilyVisibilityScript
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def gitea_family_visibility_script
|
|
10
|
+
<<~'JS'
|
|
11
|
+
(() => {
|
|
12
|
+
const subtreeHidden = (node) => {
|
|
13
|
+
let current = node;
|
|
14
|
+
while (current && current !== document) {
|
|
15
|
+
const style = getComputedStyle(current);
|
|
16
|
+
if (current.hidden || style.display === 'none') return true;
|
|
17
|
+
current = current.parentElement;
|
|
18
|
+
}
|
|
19
|
+
return false;
|
|
20
|
+
};
|
|
21
|
+
const visible = (node) => !!node && !subtreeHidden(node) &&
|
|
22
|
+
!['hidden', 'collapse'].includes(getComputedStyle(node).visibility);
|
|
23
|
+
JS
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module GithubPullResourceStabilization
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def stabilize_github_pull_resource(page, deadline: stabilization_deadline)
|
|
10
|
+
last_signature = nil
|
|
11
|
+
stable_observations = 0
|
|
12
|
+
ready = retry_until_timeout(capped_timeout(8.0, deadline: deadline), interval: 0.1, deadline: deadline) do
|
|
13
|
+
state = safe_evaluate(page, github_pull_resource_state_script, default: nil)
|
|
14
|
+
next false unless state.is_a?(Hash) && state["ready"]
|
|
15
|
+
|
|
16
|
+
if state["signature"] == last_signature
|
|
17
|
+
stable_observations += 1
|
|
18
|
+
stable_observations >= 3
|
|
19
|
+
else
|
|
20
|
+
last_signature = state["signature"]
|
|
21
|
+
stable_observations = 1
|
|
22
|
+
false
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
settle_after_stabilization(0.5, deadline: deadline) if ready
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|