fetch_util 0.5.2 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +49 -0
  3. data/README.md +3 -1
  4. data/Rakefile +0 -2
  5. data/lib/fetch_util/assets/extract.js +1 -1
  6. data/lib/fetch_util/browser/interaction_helpers/consent_helpers.rb +12 -17
  7. data/lib/fetch_util/browser/interaction_helpers/timing_helpers.rb +42 -15
  8. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_commit_details_script.rb +48 -0
  9. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_product_state_script.rb +80 -0
  10. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_request_helpers_script.rb +53 -0
  11. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_request_state_script.rb +118 -0
  12. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_stabilization.rb +54 -0
  13. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_state.rb +27 -0
  14. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_product_state.rb +55 -0
  15. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_request_state.rb +100 -0
  16. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_state.rb +59 -0
  17. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_modules.rb +26 -0
  18. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_state.rb +93 -0
  19. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resources.rb +73 -0
  20. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_thread_state.rb +92 -0
  21. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_threads.rb +71 -0
  22. data/lib/fetch_util/browser/site_stabilization/community_and_marketplace.rb +9 -10
  23. data/lib/fetch_util/browser/site_stabilization/facebook_stabilization.rb +56 -0
  24. data/lib/fetch_util/browser/site_stabilization/gerrit_change_product_state_script.rb +97 -0
  25. data/lib/fetch_util/browser/site_stabilization/gerrit_change_request_state_script.rb +75 -0
  26. data/lib/fetch_util/browser/site_stabilization/gerrit_change_stabilization.rb +37 -0
  27. data/lib/fetch_util/browser/site_stabilization/gerrit_change_state.rb +39 -0
  28. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_fetch_state_script.rb +31 -0
  29. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_product_state_script.rb +76 -0
  30. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_request_state_script.rb +111 -0
  31. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_route_state_script.rb +71 -0
  32. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_stabilization.rb +37 -0
  33. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_state.rb +42 -0
  34. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_validation_state_script.rb +49 -0
  35. data/lib/fetch_util/browser/site_stabilization/gitea_family_pull_resource_product_state.rb +88 -0
  36. data/lib/fetch_util/browser/site_stabilization/gitea_family_pull_resource_state.rb +109 -0
  37. data/lib/fetch_util/browser/site_stabilization/gitea_family_pull_resources.rb +30 -0
  38. data/lib/fetch_util/browser/site_stabilization/gitea_family_stabilization.rb +60 -0
  39. data/lib/fetch_util/browser/site_stabilization/gitea_family_thread_product_state.rb +74 -0
  40. data/lib/fetch_util/browser/site_stabilization/gitea_family_thread_timeline_state.rb +69 -0
  41. data/lib/fetch_util/browser/site_stabilization/gitea_family_threads.rb +28 -0
  42. data/lib/fetch_util/browser/site_stabilization/gitea_family_visibility_script.rb +28 -0
  43. data/lib/fetch_util/browser/site_stabilization/github_pull_resource_stabilization.rb +31 -0
  44. data/lib/fetch_util/browser/site_stabilization/github_pull_resources.rb +105 -0
  45. data/lib/fetch_util/browser/site_stabilization/github_threads.rb +82 -0
  46. data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state.rb +17 -0
  47. data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state_script.rb +98 -0
  48. data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_visibility_script.rb +43 -0
  49. data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resources.rb +28 -0
  50. data/lib/fetch_util/browser/site_stabilization/gitlab_repo.rb +3 -11
  51. data/lib/fetch_util/browser/site_stabilization/gitlab_stabilization.rb +33 -0
  52. data/lib/fetch_util/browser/site_stabilization/gitlab_threads.rb +92 -0
  53. data/lib/fetch_util/browser/site_stabilization/social_platforms.rb +19 -38
  54. data/lib/fetch_util/browser/site_stabilization/travel_and_lodging.rb +3 -3
  55. data/lib/fetch_util/browser/site_stabilization.rb +89 -0
  56. data/lib/fetch_util/browser/stabilization/anubis.rb +4 -5
  57. data/lib/fetch_util/browser/stabilization/page_flow.rb +71 -27
  58. data/lib/fetch_util/browser/stabilization/spa_hydration.rb +7 -5
  59. data/lib/fetch_util/browser.rb +88 -17
  60. data/lib/fetch_util/cli.rb +69 -23
  61. data/lib/fetch_util/extractor.rb +8 -2
  62. data/lib/fetch_util/fetcher.rb +144 -43
  63. data/lib/fetch_util/http_redirect_client.rb +161 -0
  64. data/lib/fetch_util/parallel_fetcher.rb +125 -28
  65. data/lib/fetch_util/raw_docs_fallback.rb +158 -21
  66. data/lib/fetch_util/regulatory/cache_store.rb +47 -14
  67. data/lib/fetch_util/regulatory/fetch_records.rb +10 -7
  68. data/lib/fetch_util/regulatory/headers.rb +8 -1
  69. data/lib/fetch_util/regulatory/http_client.rb +2 -97
  70. data/lib/fetch_util/regulatory/orchestration.rb +10 -6
  71. data/lib/fetch_util/regulatory/page.rb +8 -11
  72. data/lib/fetch_util/regulatory/robots.rb +2 -10
  73. data/lib/fetch_util/regulatory/source_selection.rb +13 -3
  74. data/lib/fetch_util/regulatory/tdm_page.rb +9 -4
  75. data/lib/fetch_util/regulatory/tdm_policy.rb +16 -9
  76. data/lib/fetch_util/regulatory/tdm_rep.rb +4 -3
  77. data/lib/fetch_util/regulatory/tdm_support.rb +10 -6
  78. data/lib/fetch_util/regulatory.rb +5 -1
  79. data/lib/fetch_util/request_log.rb +24 -3
  80. data/lib/fetch_util/result.rb +37 -89
  81. data/lib/fetch_util/search_transport.rb +102 -35
  82. data/lib/fetch_util/searcher.rb +15 -13
  83. data/lib/fetch_util/version.rb +1 -1
  84. data/lib/fetch_util.rb +4 -1
  85. metadata +66 -2
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module GerritFileResourceValidationStateScript
7
+ private
8
+
9
+ def gerrit_file_resource_validation_state_script
10
+ <<~'JS'
11
+ const objectRecord = (value) => value && typeof value === "object" && !Array.isArray(value);
12
+ const validTextLines = (value) => value == null ||
13
+ (Array.isArray(value) && value.every((line) => typeof line === "string"));
14
+ const validIntralineEdits = (value) => value == null || (Array.isArray(value) && value.every(
15
+ (edit) => Array.isArray(edit) && edit.length === 2 && edit.every(
16
+ (number) => Number.isInteger(number) && number >= 0
17
+ )
18
+ ));
19
+ const validDiffBlock = (block) => objectRecord(block) && ["ab", "a", "b"].every(
20
+ (field) => validTextLines(block[field])
21
+ ) && ["edit_a", "edit_b"].every(
22
+ (field) => validIntralineEdits(block[field])
23
+ ) && (block.skip == null || typeof block.skip === "number" || objectRecord(block.skip));
24
+ const validateDiff = (diff, filePath) => {
25
+ if (diff.content != null &&
26
+ (!Array.isArray(diff.content) || !diff.content.every(validDiffBlock))) {
27
+ throw new Error("invalid Gerrit diff content payload");
28
+ }
29
+ if (diff.diff_header != null && typeof diff.diff_header !== "string" &&
30
+ (!Array.isArray(diff.diff_header) ||
31
+ !diff.diff_header.every((line) => typeof line === "string"))) {
32
+ throw new Error("invalid Gerrit diff header payload");
33
+ }
34
+ if (diff.meta_a != null && !objectRecord(diff.meta_a)) {
35
+ throw new Error("invalid Gerrit base metadata payload");
36
+ }
37
+ if (diff.meta_b != null && !objectRecord(diff.meta_b)) {
38
+ throw new Error("invalid Gerrit target metadata payload");
39
+ }
40
+ if (diff.meta_b && diff.meta_b.name && diff.meta_b.name !== filePath) {
41
+ throw new Error("Gerrit API diff target identity mismatch");
42
+ }
43
+ };
44
+ JS
45
+ end
46
+ end
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,88 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module GiteaFamilyPullResourceProductState
7
+ private
8
+
9
+ def gitea_family_pull_resource_product_state_script
10
+ gitea_family_visibility_script + <<~'JS'
11
+ const visibleNodes = (node) => [node, ...node.querySelectorAll('*')].filter(visible);
12
+ const ownText = (node) => Array.from(node.childNodes)
13
+ .filter((child) => child.nodeType === Node.TEXT_NODE)
14
+ .map((child) => child.textContent || '').join(' ').trim();
15
+ const visibleText = (node) => visibleNodes(node).map(ownText).filter(Boolean).join(' ');
16
+ const visibleMaterial = (node, excluded) => !!node && visibleNodes(node).some((candidate) => {
17
+ if (excluded && candidate.closest(excluded)) return false;
18
+ return !!ownText(candidate) || candidate.matches('table, pre, code, img, ins, del, .diff-line');
19
+ });
20
+ const runtime = window.config || {};
21
+ const prefix = String(runtime.appSubUrl || '').replace(/^\/+|\/+$/g, '');
22
+ const routePrefix = prefix ? `/${prefix}` : '';
23
+ const escaped = routePrefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
24
+ const match = location.pathname.match(new RegExp(
25
+ `^${escaped}/([^/]+)/([^/]+)/pulls/(\\d+)/(commits|files)(?:/([^/]+))?\\/?$`
26
+ ));
27
+ if (!match) return null;
28
+ const routeSurface = match[4];
29
+ const detail = match[5] || '';
30
+ const validDetail = routeSurface === 'commits'
31
+ ? (!detail || detail === 'list' || /^[a-f0-9]{4,64}$/i.test(detail))
32
+ : (!detail || /^[a-f0-9]{4,64}$/i.test(detail) ||
33
+ /^[a-f0-9]{7,64}\.\.?\.[a-f0-9]{7,64}$/i.test(detail));
34
+ if (!validDetail) return null;
35
+ const surface = routeSurface === 'commits' && detail && detail !== 'list' ? 'files' : routeSurface;
36
+ let runtimeMatch = false;
37
+ try {
38
+ const app = new URL(runtime.appUrl || '', location.href);
39
+ runtimeMatch = !!runtime.appUrl && app.origin === location.origin &&
40
+ app.pathname.replace(/\/$/, '') === routePrefix;
41
+ } catch (_error) { runtimeMatch = false; }
42
+ const brandValues = Array.from(document.querySelectorAll(
43
+ 'meta[name="author"], meta[name="generator"], meta[property="og:site_name"]'
44
+ )).map((node) => (node.content || '').trim());
45
+ Array.from(document.querySelectorAll('footer a[href]')).forEach((node) => {
46
+ const text = (node.textContent || '').trim();
47
+ const href = node.getAttribute('href') || '';
48
+ if (/powered by.*\b(?:forgejo|gitea)\b/i.test(text) || /\b(?:forgejo|gitea)\b/i.test(href)) {
49
+ brandValues.push(`${text} ${href}`);
50
+ }
51
+ });
52
+ brandValues.push((runtime.assetVersionEncoded || '').trim());
53
+ const branded = /\b(?:forgejo|gitea)\b/i.test(brandValues.join(' '));
54
+ let runtimeAsset = false;
55
+ try {
56
+ const assetUrl = new URL(runtime.assetUrlPrefix || '', location.href);
57
+ const expectedPath = `${routePrefix}/assets`;
58
+ runtimeAsset = !!runtime.assetUrlPrefix && assetUrl.origin === location.origin &&
59
+ assetUrl.pathname.replace(/\/$/, '') === expectedPath &&
60
+ Array.from(document.querySelectorAll('script[src], link[href]')).some((node) => {
61
+ const value = node.getAttribute('src') || node.getAttribute('href') || '';
62
+ const url = new URL(value, location.href);
63
+ return url.origin === location.origin && url.pathname.startsWith(`${expectedPath}/`);
64
+ });
65
+ } catch (_error) { runtimeAsset = false; }
66
+ const rootSelectors = surface === 'commits'
67
+ ? ['.page-content.repository.view.issue.pull.commits', '.repository.view.issue.pull.commits',
68
+ '.page-content.repository.view.issue.pull']
69
+ : ['.page-content.repository.view.issue.pull.files.diff', '.repository.view.issue.pull.files.diff',
70
+ '.page-content.repository.view.issue.pull'];
71
+ let root = null;
72
+ rootSelectors.some((selector) => { root = document.querySelector(selector); return !!root; });
73
+ const empty = root && Array.from(root.querySelectorAll(
74
+ '.ui.message.empty, .empty-state, .nothing-here, .no-results, [data-testid="empty-state"]'
75
+ )).find((node) => visible(node) && (node.textContent || '').trim());
76
+ const structure = surface === 'commits'
77
+ ? !!(root && (root.querySelector(
78
+ '#commits-table > tbody.commit-list, .commit-group .commits .commit'
79
+ ) || empty))
80
+ : !!(root && (root.querySelector('#diff-container, #diff-file-boxes, #diff-file-tree') || empty));
81
+ const product = runtimeMatch && structure && (branded || runtimeAsset);
82
+ if (!product) return { product: false, ready: false, signature: '' };
83
+ JS
84
+ end
85
+ end
86
+ end
87
+ end
88
+ end
@@ -0,0 +1,109 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module GiteaFamilyPullResourceState
7
+ private
8
+
9
+ def gitea_family_pull_resource_state_script
10
+ <<~'JS'
11
+ const loading = Array.from(root.querySelectorAll(
12
+ '[aria-busy="true"], .is-loading, .tw-loading, .ui.active.loader'
13
+ )).some(visible);
14
+ const rows = surface === 'commits'
15
+ ? Array.from(root.querySelectorAll(
16
+ '#commits-table > tbody.commit-list > tr, .commit-group .commits .commit'
17
+ ))
18
+ .filter((row) => visibleMaterial(row))
19
+ : [];
20
+ const rowSignatures = rows.map((row) => [
21
+ row.querySelector('.copy-commit-id[data-clipboard-text]')?.getAttribute('data-clipboard-text') || '',
22
+ row.querySelector('a[href*="/commit/"]')?.getAttribute('href') || '',
23
+ visibleText(row)
24
+ ].join('|'));
25
+ const pageData = runtime.pageData || {};
26
+ const treeData = pageData.DiffFileTree || pageData.diffFileTree || [];
27
+ const treeRoots = treeData.TreeRoot || treeData.treeRoot || treeData;
28
+ const treeIds = [];
29
+ const visit = (nodes) => (Array.isArray(nodes) ? nodes : [nodes]).filter(Boolean).forEach((node) => {
30
+ const children = node.Children || node.children || [];
31
+ if (children.length) return visit(children);
32
+ const identity = String(node.NameHash || node.nameHash || '');
33
+ if (identity) treeIds.push(identity.startsWith('diff-') ? identity : `diff-${identity}`);
34
+ });
35
+ const forgejoFiles = (pageData.diffFileInfo || {}).files || [];
36
+ if (forgejoFiles.length) {
37
+ forgejoFiles.forEach((file) => {
38
+ const identity = String(file.NameHash || file.nameHash || '');
39
+ if (identity) treeIds.push(identity.startsWith('diff-') ? identity : `diff-${identity}`);
40
+ });
41
+ } else {
42
+ visit(treeRoots);
43
+ }
44
+ const boxes = surface === 'files'
45
+ ? Array.from(root.querySelectorAll(
46
+ '#diff-file-boxes > .diff-file-box, .diff-file-box.file-content[id^="diff-"]'
47
+ )).filter((box, index, all) => box.id !== 'diff-incomplete' && visibleMaterial(box) &&
48
+ !all.some((other, otherIndex) => otherIndex < index && other.contains(box)))
49
+ : [];
50
+ const boxSignatures = boxes.map((box) => [
51
+ box.id || '',
52
+ box.getAttribute('data-new-filename') || '',
53
+ box.getAttribute('data-old-filename') || '',
54
+ visibleText(box)
55
+ ].join('|'));
56
+ const deferred = surface === 'files'
57
+ ? Array.from(root.querySelectorAll(
58
+ '#diff-show-more-files[data-href], .diff-load-button[data-href], ' +
59
+ '[data-global-click="diffLoadFileBody"][data-href]'
60
+ )).filter(visible).map((node) => node.getAttribute('data-href') || '')
61
+ : [];
62
+ let selectedId = '';
63
+ if (surface === 'files') {
64
+ try { selectedId = decodeURIComponent((location.hash || '').replace(/^#/, '')); }
65
+ catch (_error) { selectedId = ''; }
66
+ }
67
+ const selected = selectedId && boxes.find((box) => box.id === selectedId);
68
+ const selectedBody = selected && selected.querySelector(
69
+ '.diff-file-body, .file-body, .diff-rendered, .image-diff, .binary'
70
+ );
71
+ const excluded = 'button, [role="button"], .diff-load-button, [data-global-click="diffLoadFileBody"]';
72
+ const selectedDeferred = !!(selected && Array.from(selected.querySelectorAll(
73
+ '.diff-load-button[data-href], [data-global-click="diffLoadFileBody"][data-href]'
74
+ )).find(visible));
75
+ const selectedStructured = !!selectedBody && visibleNodes(selectedBody).some((candidate) => {
76
+ if (candidate.closest(excluded)) return false;
77
+ return candidate.matches('table, pre, code, img, ins, del, .diff-line');
78
+ });
79
+ const selectedLoaded = !!selectedBody && visibleMaterial(selectedBody, excluded) &&
80
+ (!selectedDeferred || selectedStructured);
81
+ const selectedIndexed = !!selectedId && treeIds.includes(selectedId);
82
+ const selectedTerminal = !selectedId || selectedLoaded || selectedDeferred || (!selected && selectedIndexed);
83
+ const material = surface === 'commits'
84
+ ? rows.length > 0 || !!empty
85
+ : treeIds.length > 0 || boxes.length > 0 || !!empty;
86
+ return {
87
+ product: true,
88
+ ready: !loading && material && selectedTerminal,
89
+ loading,
90
+ signature: [
91
+ surface,
92
+ rowSignatures.join('\u001e'),
93
+ treeIds.join('\u001e'),
94
+ boxSignatures.join('\u001e'),
95
+ deferred.join('\u001e'),
96
+ selectedId,
97
+ selectedLoaded,
98
+ selectedDeferred,
99
+ !!empty,
100
+ loading
101
+ ].join(':')
102
+ };
103
+ })()
104
+ JS
105
+ end
106
+ end
107
+ end
108
+ end
109
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module GiteaFamilyPullResources
7
+ GITEA_FAMILY_PULL_RESOURCE_STABILIZATION_PROFILE = {
8
+ host: true,
9
+ path_query: lambda { |uri|
10
+ uri.path.match?(%r{/(?:[^/]+/){2}pulls/\d+/(?:commits(?:/(?:list|[a-f\d]{4,64}))?|files(?:/(?:[a-f\d]{4,64}|[a-f\d]{7,64}\.\.?\.[a-f\d]{7,64}))?)/?\z}i)
11
+ },
12
+ strategy: :stabilize_gitea_family_pull_resource,
13
+ fallthrough: true,
14
+ notes: "Wait for host-agnostic Gitea and Forgejo pull-request resources.",
15
+ tests: "spec/fetch_util/browser_stabilization_spec.rb"
16
+ }.freeze
17
+
18
+ private
19
+
20
+ def stabilize_gitea_family_pull_resource(page, deadline: stabilization_deadline)
21
+ stabilize_gitea_family_state(
22
+ page,
23
+ gitea_family_pull_resource_product_state_script + gitea_family_pull_resource_state_script,
24
+ deadline: deadline
25
+ )
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,60 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module GiteaFamilyStabilization
7
+ private
8
+
9
+ def stabilize_gitea_family_state(page, state_script, deadline: stabilization_deadline)
10
+ last_signature = nil
11
+ stable_observations = 0
12
+ incomplete_signature = nil
13
+ incomplete_observations = 0
14
+ terminal_incomplete = false
15
+ observed = retry_until_timeout(
16
+ capped_timeout(6.0, deadline: deadline),
17
+ interval: 0.1,
18
+ deadline: deadline
19
+ ) do
20
+ state = safe_evaluate(page, state_script, default: nil)
21
+ return false if state.is_a?(Hash) && state["product"] == false
22
+ next false unless state.is_a?(Hash) && state["product"]
23
+
24
+ unless state["ready"]
25
+ if state["loading"]
26
+ incomplete_signature = nil
27
+ incomplete_observations = 0
28
+ elsif state["signature"] == incomplete_signature
29
+ incomplete_observations += 1
30
+ else
31
+ incomplete_signature = state["signature"]
32
+ incomplete_observations = 1
33
+ end
34
+ if incomplete_observations >= 10
35
+ terminal_incomplete = true
36
+ next true
37
+ end
38
+ next false
39
+ end
40
+
41
+ incomplete_signature = nil
42
+ incomplete_observations = 0
43
+ if state["signature"] == last_signature
44
+ stable_observations += 1
45
+ else
46
+ last_signature = state["signature"]
47
+ stable_observations = 1
48
+ end
49
+ stable_observations >= 3
50
+ end
51
+
52
+ return false if terminal_incomplete
53
+
54
+ settle_after_stabilization(0.5, deadline: deadline) if observed
55
+ !!observed
56
+ end
57
+ end
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,74 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module GiteaFamilyThreadProductState
7
+ private
8
+
9
+ def gitea_family_thread_product_state_script
10
+ gitea_family_visibility_script + <<~'JS'
11
+ const runtime = window.config || {};
12
+ const prefix = String(runtime.appSubUrl || '').replace(/^\/+|\/+$/g, '');
13
+ const routePrefix = prefix ? `/${prefix}` : '';
14
+ const escaped = routePrefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
15
+ const match = location.pathname.match(new RegExp(
16
+ `^${escaped}/([^/]+)/([^/]+)/(issues|pulls)/(\\d+)\\/?$`
17
+ ));
18
+ if (!match) return null;
19
+ let runtimeMatch = false;
20
+ try {
21
+ const app = new URL(runtime.appUrl || '', location.href);
22
+ runtimeMatch = !!runtime.appUrl && app.origin === location.origin &&
23
+ app.pathname.replace(/\/$/, '') === routePrefix;
24
+ } catch (_error) { runtimeMatch = false; }
25
+ const brandValues = Array.from(document.querySelectorAll(
26
+ 'meta[name="author"], meta[name="generator"], meta[property="og:site_name"]'
27
+ )).map((node) => (node.content || '').trim());
28
+ Array.from(document.querySelectorAll('footer a[href]')).forEach((node) => {
29
+ const value = `${node.textContent || ''} ${node.getAttribute('href') || ''}`;
30
+ if (/\b(?:forgejo|gitea)\b/i.test(value)) brandValues.push(value);
31
+ });
32
+ brandValues.push((runtime.assetVersionEncoded || '').trim());
33
+ const branded = /\b(?:forgejo|gitea)\b/i.test(brandValues.join(' '));
34
+ let runtimeAsset = false;
35
+ try {
36
+ const assetUrl = new URL(runtime.assetUrlPrefix || '', location.href);
37
+ const expectedPath = `${routePrefix}/assets`;
38
+ runtimeAsset = !!runtime.assetUrlPrefix && assetUrl.origin === location.origin &&
39
+ assetUrl.pathname.replace(/\/$/, '') === expectedPath &&
40
+ Array.from(document.querySelectorAll('script[src], link[href]')).some((node) => {
41
+ const value = node.getAttribute('src') || node.getAttribute('href') || '';
42
+ const url = new URL(value, location.href);
43
+ return url.origin === location.origin && url.pathname.startsWith(`${expectedPath}/`);
44
+ });
45
+ } catch (_error) { runtimeAsset = false; }
46
+ const root = document.querySelector(
47
+ '.page-content.repository.view.issue .issue-content, .repository.view.issue .issue-content, .page-content .issue-content'
48
+ );
49
+ const opening = root && root.querySelector(
50
+ '.issue-content-left .timeline-item.comment.issue-content-comment[id^="issue-"], ' +
51
+ '.issue-content-left .timeline-item.comment.issue-content-comment, ' +
52
+ '.issue-content-left .timeline-item.comment.first[id^="issue-"], ' +
53
+ '.issue-content-left .timeline-item.comment.first, ' +
54
+ '.comment-list .timeline-item.comment.issue-content-comment, ' +
55
+ '.comment-list .timeline-item.comment.first'
56
+ );
57
+ const openingBody = opening && opening.querySelector('.comment-body');
58
+ const rendered = openingBody && openingBody.querySelector('.render-content.markup');
59
+ const empty = openingBody && openingBody.querySelector('.no-content');
60
+ const attachment = openingBody && Array.from(openingBody.querySelectorAll(
61
+ '.dropzone-attachments a[href], .attachments a[href]'
62
+ )).some(visible);
63
+ const structure = !!(root && opening && openingBody &&
64
+ root.querySelector('.issue-content-left .comment-list, .issue-content-left.comment-list') &&
65
+ opening.querySelector('.content.comment-container, .comment-body') &&
66
+ ((visible(rendered) && (rendered.innerText || '').trim()) || visible(empty) || attachment));
67
+ const product = runtimeMatch && structure && (branded || runtimeAsset);
68
+ if (!product) return { product: false, ready: false, signature: '' };
69
+ JS
70
+ end
71
+ end
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,69 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module GiteaFamilyThreadTimelineState
7
+ private
8
+
9
+ def gitea_family_thread_timeline_state_script
10
+ <<~'JS'
11
+ const candidates = Array.from(root.querySelectorAll([
12
+ '.issue-content-left .timeline-item-group',
13
+ '.issue-content-left .timeline-item',
14
+ '.comment-list .timeline-item-group',
15
+ '.comment-list .timeline-item'
16
+ ].join(', ')));
17
+ const rows = [];
18
+ candidates.forEach((node) => {
19
+ const group = node.closest('.timeline-item-group');
20
+ const row = group && root.contains(group) ? group : node;
21
+ if (subtreeHidden(row) || row.matches(
22
+ '.pull-merge-box, .timeline-item.comment.merge.box, .form, #timeline-comments-end'
23
+ )) return;
24
+ if (rows.some((existing) => existing === row || existing.contains(row))) return;
25
+ rows.push(row);
26
+ });
27
+ const author = opening.querySelector(
28
+ '.comment-header .author, .timeline-avatar + * .author, ' +
29
+ '.comment-header-left a.tw-font-semibold, .comment-header-left span.tw-font-semibold, ' +
30
+ '.comment-header a.tw-font-semibold, a.tw-font-semibold, .author'
31
+ );
32
+ const comments = rows.filter((row) => !!row.querySelector('.comment-body'));
33
+ const events = rows.length - comments.length;
34
+ const loading = Array.from(root.querySelectorAll(
35
+ '[aria-busy="true"], .is-loading, .tw-loading, .ui.active.loader'
36
+ )).some(visible);
37
+ const openingReady = visible(openingBody) && !!(
38
+ (rendered && (rendered.innerText || '').trim()) ||
39
+ (openingBody && openingBody.querySelector('.no-content')) ||
40
+ Array.from(openingBody.querySelectorAll('.dropzone-attachments, .attachments')).some(visible)
41
+ );
42
+ const rowSignatures = rows.map((row) => [
43
+ row.id || '',
44
+ row.className || '',
45
+ Array.from(row.querySelectorAll('relative-time[datetime], time[datetime]'))
46
+ .map((time) => time.getAttribute('datetime') || '').join(','),
47
+ (row.innerText || '').trim()
48
+ ].join('|')).join('\u001e');
49
+ return {
50
+ product: true,
51
+ ready: !loading && openingReady,
52
+ loading,
53
+ signature: [
54
+ rows.length,
55
+ comments.length,
56
+ events,
57
+ visible(author) ? (author.textContent || '').trim() : '',
58
+ openingReady,
59
+ loading,
60
+ rowSignatures
61
+ ].join(':')
62
+ };
63
+ })()
64
+ JS
65
+ end
66
+ end
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module GiteaFamilyThreads
7
+ GITEA_FAMILY_THREAD_STABILIZATION_PROFILE = {
8
+ host: true,
9
+ path_query: ->(uri) { uri.path.match?(%r{/(?:[^/]+/){2}(?:issues|pulls)/\d+/?\z}) },
10
+ strategy: :stabilize_gitea_family_thread,
11
+ fallthrough: true,
12
+ notes: "Wait for host-agnostic Gitea and Forgejo issue and pull-request conversations.",
13
+ tests: "spec/fetch_util/browser_stabilization_spec.rb"
14
+ }.freeze
15
+
16
+ private
17
+
18
+ def stabilize_gitea_family_thread(page, deadline: stabilization_deadline)
19
+ stabilize_gitea_family_state(
20
+ page,
21
+ gitea_family_thread_product_state_script + gitea_family_thread_timeline_state_script,
22
+ deadline: deadline
23
+ )
24
+ end
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module GiteaFamilyVisibilityScript
7
+ private
8
+
9
+ def gitea_family_visibility_script
10
+ <<~'JS'
11
+ (() => {
12
+ const subtreeHidden = (node) => {
13
+ let current = node;
14
+ while (current && current !== document) {
15
+ const style = getComputedStyle(current);
16
+ if (current.hidden || style.display === 'none') return true;
17
+ current = current.parentElement;
18
+ }
19
+ return false;
20
+ };
21
+ const visible = (node) => !!node && !subtreeHidden(node) &&
22
+ !['hidden', 'collapse'].includes(getComputedStyle(node).visibility);
23
+ JS
24
+ end
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module GithubPullResourceStabilization
7
+ private
8
+
9
+ def stabilize_github_pull_resource(page, deadline: stabilization_deadline)
10
+ last_signature = nil
11
+ stable_observations = 0
12
+ ready = retry_until_timeout(capped_timeout(8.0, deadline: deadline), interval: 0.1, deadline: deadline) do
13
+ state = safe_evaluate(page, github_pull_resource_state_script, default: nil)
14
+ next false unless state.is_a?(Hash) && state["ready"]
15
+
16
+ if state["signature"] == last_signature
17
+ stable_observations += 1
18
+ stable_observations >= 3
19
+ else
20
+ last_signature = state["signature"]
21
+ stable_observations = 1
22
+ false
23
+ end
24
+ end
25
+
26
+ settle_after_stabilization(0.5, deadline: deadline) if ready
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end