fetch_util 0.5.2 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +49 -0
  3. data/README.md +3 -1
  4. data/Rakefile +0 -2
  5. data/lib/fetch_util/assets/extract.js +1 -1
  6. data/lib/fetch_util/browser/interaction_helpers/consent_helpers.rb +12 -17
  7. data/lib/fetch_util/browser/interaction_helpers/timing_helpers.rb +42 -15
  8. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_commit_details_script.rb +48 -0
  9. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_product_state_script.rb +80 -0
  10. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_request_helpers_script.rb +53 -0
  11. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_request_state_script.rb +118 -0
  12. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_stabilization.rb +54 -0
  13. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_state.rb +27 -0
  14. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_product_state.rb +55 -0
  15. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_request_state.rb +100 -0
  16. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_state.rb +59 -0
  17. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_modules.rb +26 -0
  18. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_state.rb +93 -0
  19. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resources.rb +73 -0
  20. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_thread_state.rb +92 -0
  21. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_threads.rb +71 -0
  22. data/lib/fetch_util/browser/site_stabilization/community_and_marketplace.rb +9 -10
  23. data/lib/fetch_util/browser/site_stabilization/facebook_stabilization.rb +56 -0
  24. data/lib/fetch_util/browser/site_stabilization/gerrit_change_product_state_script.rb +97 -0
  25. data/lib/fetch_util/browser/site_stabilization/gerrit_change_request_state_script.rb +75 -0
  26. data/lib/fetch_util/browser/site_stabilization/gerrit_change_stabilization.rb +37 -0
  27. data/lib/fetch_util/browser/site_stabilization/gerrit_change_state.rb +39 -0
  28. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_fetch_state_script.rb +31 -0
  29. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_product_state_script.rb +76 -0
  30. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_request_state_script.rb +111 -0
  31. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_route_state_script.rb +71 -0
  32. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_stabilization.rb +37 -0
  33. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_state.rb +42 -0
  34. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_validation_state_script.rb +49 -0
  35. data/lib/fetch_util/browser/site_stabilization/gitea_family_pull_resource_product_state.rb +88 -0
  36. data/lib/fetch_util/browser/site_stabilization/gitea_family_pull_resource_state.rb +109 -0
  37. data/lib/fetch_util/browser/site_stabilization/gitea_family_pull_resources.rb +30 -0
  38. data/lib/fetch_util/browser/site_stabilization/gitea_family_stabilization.rb +60 -0
  39. data/lib/fetch_util/browser/site_stabilization/gitea_family_thread_product_state.rb +74 -0
  40. data/lib/fetch_util/browser/site_stabilization/gitea_family_thread_timeline_state.rb +69 -0
  41. data/lib/fetch_util/browser/site_stabilization/gitea_family_threads.rb +28 -0
  42. data/lib/fetch_util/browser/site_stabilization/gitea_family_visibility_script.rb +28 -0
  43. data/lib/fetch_util/browser/site_stabilization/github_pull_resource_stabilization.rb +31 -0
  44. data/lib/fetch_util/browser/site_stabilization/github_pull_resources.rb +105 -0
  45. data/lib/fetch_util/browser/site_stabilization/github_threads.rb +82 -0
  46. data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state.rb +17 -0
  47. data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state_script.rb +98 -0
  48. data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_visibility_script.rb +43 -0
  49. data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resources.rb +28 -0
  50. data/lib/fetch_util/browser/site_stabilization/gitlab_repo.rb +3 -11
  51. data/lib/fetch_util/browser/site_stabilization/gitlab_stabilization.rb +33 -0
  52. data/lib/fetch_util/browser/site_stabilization/gitlab_threads.rb +92 -0
  53. data/lib/fetch_util/browser/site_stabilization/social_platforms.rb +19 -38
  54. data/lib/fetch_util/browser/site_stabilization/travel_and_lodging.rb +3 -3
  55. data/lib/fetch_util/browser/site_stabilization.rb +89 -0
  56. data/lib/fetch_util/browser/stabilization/anubis.rb +4 -5
  57. data/lib/fetch_util/browser/stabilization/page_flow.rb +71 -27
  58. data/lib/fetch_util/browser/stabilization/spa_hydration.rb +7 -5
  59. data/lib/fetch_util/browser.rb +88 -17
  60. data/lib/fetch_util/cli.rb +69 -23
  61. data/lib/fetch_util/extractor.rb +8 -2
  62. data/lib/fetch_util/fetcher.rb +144 -43
  63. data/lib/fetch_util/http_redirect_client.rb +161 -0
  64. data/lib/fetch_util/parallel_fetcher.rb +125 -28
  65. data/lib/fetch_util/raw_docs_fallback.rb +158 -21
  66. data/lib/fetch_util/regulatory/cache_store.rb +47 -14
  67. data/lib/fetch_util/regulatory/fetch_records.rb +10 -7
  68. data/lib/fetch_util/regulatory/headers.rb +8 -1
  69. data/lib/fetch_util/regulatory/http_client.rb +2 -97
  70. data/lib/fetch_util/regulatory/orchestration.rb +10 -6
  71. data/lib/fetch_util/regulatory/page.rb +8 -11
  72. data/lib/fetch_util/regulatory/robots.rb +2 -10
  73. data/lib/fetch_util/regulatory/source_selection.rb +13 -3
  74. data/lib/fetch_util/regulatory/tdm_page.rb +9 -4
  75. data/lib/fetch_util/regulatory/tdm_policy.rb +16 -9
  76. data/lib/fetch_util/regulatory/tdm_rep.rb +4 -3
  77. data/lib/fetch_util/regulatory/tdm_support.rb +10 -6
  78. data/lib/fetch_util/regulatory.rb +5 -1
  79. data/lib/fetch_util/request_log.rb +24 -3
  80. data/lib/fetch_util/result.rb +37 -89
  81. data/lib/fetch_util/search_transport.rb +102 -35
  82. data/lib/fetch_util/searcher.rb +15 -13
  83. data/lib/fetch_util/version.rb +1 -1
  84. data/lib/fetch_util.rb +4 -1
  85. metadata +66 -2
@@ -0,0 +1,97 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module GerritChangeProductStateScript
7
+ private
8
+
9
+ def gerrit_change_product_state_script
10
+ <<~'JS'
11
+ const stateKey = "__fetchUtilGerritChange";
12
+ const pathname = (location.pathname || "").replace(/\/+$/, "");
13
+ const marker = pathname.lastIndexOf('/+/');
14
+ if (marker < 1) return { matched: false };
15
+
16
+ const tail = pathname.slice(marker + 3).split("/").filter(Boolean);
17
+ if (tail.length < 1 || tail.length > 2 || !/^\d+$/.test(tail[0]) ||
18
+ (tail[1] && !/^\d+$/.test(tail[1]))) return { matched: false };
19
+
20
+ const beforeMarker = pathname.slice(0, marker);
21
+ const baseNode = document.querySelector("base[href]");
22
+ let prefix = "";
23
+ if (baseNode) {
24
+ try {
25
+ const baseUrl = new URL(baseNode.getAttribute("href"), location.href);
26
+ if (baseUrl.origin === location.origin) prefix = baseUrl.pathname.replace(/\/+$/, "");
27
+ } catch (_error) { prefix = ""; }
28
+ }
29
+
30
+ let projectPath = beforeMarker;
31
+ if (projectPath.indexOf("/c/") === 0) {
32
+ projectPath = projectPath.slice(3);
33
+ } else {
34
+ const cMarker = projectPath.indexOf("/c/");
35
+ if (cMarker >= 0) {
36
+ prefix = projectPath.slice(0, cMarker);
37
+ projectPath = projectPath.slice(cMarker + 3);
38
+ } else {
39
+ if (prefix && projectPath.indexOf(prefix + "/") === 0) projectPath = projectPath.slice(prefix.length);
40
+ projectPath = projectPath.replace(/^\/+/, "");
41
+ }
42
+ }
43
+
44
+ try { projectPath = decodeURIComponent(projectPath); }
45
+ catch (_error) { return { matched: false }; }
46
+ if (!projectPath || projectPath.split("/").some((part) => !part || part === "." || part === "..")) {
47
+ return { matched: false };
48
+ }
49
+
50
+ const description = document.querySelector('meta[name="description"]');
51
+ const descriptionMatch = /^Gerrit Code Review$/i.test((description && description.content || "").trim());
52
+ const appMatch = !!document.querySelector("gr-app");
53
+ const footerMatch = Array.from(document.querySelectorAll("footer, [role='contentinfo']")).some(
54
+ (node) => /Powered by Gerrit Code Review/i.test((node.textContent || "").trim())
55
+ );
56
+ const assetMatch = Array.from(document.querySelectorAll("script[src], link[href]")).some((node) => {
57
+ const value = node.getAttribute("src") || node.getAttribute("href") || "";
58
+ if (!/\/polygerrit_ui\/.*\/gr-app\.js(?:$|[?#])/i.test(value)) return false;
59
+ try { return new URL(value, location.href).origin === location.origin; } catch (_error) { return false; }
60
+ });
61
+ const preloadMatch = Array.from(document.querySelectorAll("link[href]")).some((node) => {
62
+ const value = node.getAttribute("href") || "";
63
+ if (!/\/changes\/[^/?]+\/(?:detail|comments)(?:$|[?])/i.test(value)) return false;
64
+ try { return new URL(value, location.href).origin === location.origin; } catch (_error) { return false; }
65
+ });
66
+ const product = descriptionMatch && appMatch && (footerMatch || assetMatch || preloadMatch);
67
+ if (!product) return { matched: true, product: false };
68
+
69
+ const number = tail[0];
70
+ const patchset = tail[1] || null;
71
+ const changePath = beforeMarker + "/+/" + number;
72
+ const apiPath = (prefix || "") + "/changes/" + encodeURIComponent(projectPath + "~" + number);
73
+ const routeKey = [location.origin, changePath, patchset || ""].join(":");
74
+ const existing = window[stateKey];
75
+ if (existing && existing.routeKey === routeKey) {
76
+ return {
77
+ matched: true,
78
+ product: true,
79
+ status: existing.status || "",
80
+ signature: [existing.status, existing.messageCount || 0, existing.commentCount || 0,
81
+ existing.fileCount || 0].join(":")
82
+ };
83
+ }
84
+
85
+ const route = {
86
+ project: projectPath,
87
+ number: number,
88
+ patchset: patchset,
89
+ changePath: changePath,
90
+ apiPath: apiPath
91
+ };
92
+ JS
93
+ end
94
+ end
95
+ end
96
+ end
97
+ end
@@ -0,0 +1,75 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module GerritChangeRequestStateScript
7
+ private
8
+
9
+ def gerrit_change_request_state_script
10
+ <<~'JS'
11
+ const prepared = { status: "loading", product: true, routeKey: routeKey, route: route };
12
+ window[stateKey] = prepared;
13
+
14
+ const requestJson = async (suffix) => {
15
+ const url = new URL(apiPath + suffix, location.origin);
16
+ if (url.origin !== location.origin) throw new Error("cross-origin Gerrit API route");
17
+ const response = await fetch(url.href, {
18
+ credentials: "same-origin",
19
+ headers: { Accept: "application/json" }
20
+ });
21
+ if (!response.ok) throw new Error("Gerrit API returned " + response.status);
22
+ const text = await response.text();
23
+ const value = JSON.parse(text.replace(/^\)\]\}'(?:\r?\n)?/, ""));
24
+ if (!value || typeof value !== "object" || Array.isArray(value)) {
25
+ throw new Error("invalid Gerrit API payload");
26
+ }
27
+ return value;
28
+ };
29
+ const selectedRevision = route.patchset || "current";
30
+ Promise.all([
31
+ requestJson("/detail?o=ALL_REVISIONS&o=ALL_COMMITS&o=DETAILED_LABELS&o=DETAILED_ACCOUNTS&o=MESSAGES"),
32
+ requestJson("/comments?enable-context=true&context-padding=3"),
33
+ requestJson("/revisions/" + encodeURIComponent(selectedRevision) + "/files/")
34
+ ]).then(([detail, comments, files]) => {
35
+ if (window[stateKey] !== prepared || prepared.status !== "loading") return;
36
+ if (String(detail._number || "") !== number || !detail.project || detail.project !== projectPath) {
37
+ throw new Error("Gerrit API change identity mismatch");
38
+ }
39
+ const selectedExists = !route.patchset || Object.values(detail.revisions || {}).some(
40
+ (revision) => String(revision && revision._number || "") === route.patchset
41
+ );
42
+ if (!selectedExists) throw new Error("Gerrit API patch set identity mismatch");
43
+ const countComments = (collection) => Object.keys(collection || {}).reduce(
44
+ (total, path) => total + (Array.isArray(collection[path]) ? collection[path].length : 0), 0
45
+ );
46
+ window[stateKey] = {
47
+ status: "ready",
48
+ product: true,
49
+ routeKey: routeKey,
50
+ route: route,
51
+ detail: detail,
52
+ comments: comments,
53
+ files: files,
54
+ messageCount: Array.isArray(detail.messages) ? detail.messages.length : 0,
55
+ commentCount: countComments(comments),
56
+ fileCount: Object.keys(files).length
57
+ };
58
+ }).catch((error) => {
59
+ if (window[stateKey] !== prepared || prepared.status !== "loading") return;
60
+ window[stateKey] = {
61
+ status: "failed",
62
+ product: true,
63
+ routeKey: routeKey,
64
+ route: route,
65
+ reason: String(error && error.message || error || "Gerrit preparation failed")
66
+ };
67
+ });
68
+
69
+ return { matched: true, product: true, status: "loading", signature: "loading" };
70
+ JS
71
+ end
72
+ end
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module GerritChangeStabilization
7
+ GERRIT_CHANGE_STABILIZATION_PROFILE = {
8
+ host: true,
9
+ path_query: ->(uri) { uri.path.match?(%r{/\+/\d+(?:/\d+)?/?\z}) },
10
+ strategy: :stabilize_gerrit_change,
11
+ fallthrough: true,
12
+ notes: "Prepare product-matched Gerrit change REST resources.",
13
+ tests: "spec/fetch_util/browser_stabilization_spec.rb"
14
+ }.freeze
15
+
16
+ private
17
+
18
+ def stabilize_gerrit_change(page, deadline: stabilization_deadline)
19
+ state = nil
20
+ ready = retry_until_timeout(
21
+ capped_timeout(10.0, deadline: deadline),
22
+ interval: 0.1,
23
+ deadline: deadline
24
+ ) do
25
+ state = gerrit_change_state(page)
26
+ return false if state.is_a?(Hash) && (state["product"] == false || state["status"] == "failed")
27
+
28
+ state.is_a?(Hash) && state["status"] == "ready"
29
+ end
30
+ fail_gerrit_change_preparation(page) if !ready && state.is_a?(Hash) && state["status"] == "loading"
31
+ settle_after_stabilization(0.25, deadline: deadline) if ready
32
+ ready
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module GerritChangeState
7
+ include GerritChangeProductStateScript
8
+ include GerritChangeRequestStateScript
9
+
10
+ private
11
+
12
+ def gerrit_change_state(page)
13
+ safe_evaluate(page, gerrit_change_state_script, default: nil)
14
+ end
15
+
16
+ def fail_gerrit_change_preparation(page)
17
+ safe_evaluate(page, <<~'JS', default: false)
18
+ (() => {
19
+ const prepared = window.__fetchUtilGerritChange;
20
+ if (!prepared || prepared.status !== "loading") return false;
21
+ prepared.status = "failed";
22
+ prepared.reason = "Gerrit preparation timed out";
23
+ return true;
24
+ })()
25
+ JS
26
+ end
27
+
28
+ def gerrit_change_state_script
29
+ <<~JS
30
+ (() => {
31
+ #{gerrit_change_product_state_script}
32
+ #{gerrit_change_request_state_script}
33
+ })()
34
+ JS
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module GerritFileResourceFetchStateScript
7
+ private
8
+
9
+ def gerrit_file_resource_fetch_state_script
10
+ <<~'JS'
11
+ const requestJson = async (suffix) => {
12
+ const url = new URL(apiPath + suffix, location.origin);
13
+ if (url.origin !== location.origin) throw new Error("cross-origin Gerrit API route");
14
+ const response = await fetch(url.href, {
15
+ credentials: "same-origin",
16
+ headers: { Accept: "application/json" }
17
+ });
18
+ if (!response.ok) throw new Error("Gerrit API returned " + response.status);
19
+ const text = await response.text();
20
+ const value = JSON.parse(text.replace(/^\)\]\}'(?:\r?\n)?/, ""));
21
+ if (!value || typeof value !== "object" || Array.isArray(value)) {
22
+ throw new Error("invalid Gerrit API payload");
23
+ }
24
+ return value;
25
+ };
26
+ JS
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,76 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module GerritFileResourceProductStateScript
7
+ private
8
+
9
+ def gerrit_file_resource_product_state_script
10
+ gerrit_file_resource_route_state_script + <<~'JS'
11
+ const description = document.querySelector('meta[name="description"]');
12
+ const descriptionMatch = /^Gerrit Code Review$/i.test((description && description.content || "").trim());
13
+ const appMatch = !!document.querySelector("gr-app");
14
+ const footerMatch = Array.from(document.querySelectorAll("footer, [role='contentinfo']")).some(
15
+ (node) => /Powered by Gerrit Code Review/i.test((node.textContent || "").trim())
16
+ );
17
+ const assetMatch = Array.from(document.querySelectorAll("script[src], link[href]")).some((node) => {
18
+ const value = node.getAttribute("src") || node.getAttribute("href") || "";
19
+ if (!/\/polygerrit_ui\/.*\/gr-app\.js(?:$|[?#])/i.test(value)) return false;
20
+ try { return new URL(value, location.href).origin === location.origin; } catch (_error) { return false; }
21
+ });
22
+ const preloadMatch = Array.from(document.querySelectorAll("link[href]")).some((node) => {
23
+ const value = node.getAttribute("href") || "";
24
+ if (!/\/changes\/[^/?]+\/(?:detail|comments)(?:$|[?])/i.test(value)) return false;
25
+ try { return new URL(value, location.href).origin === location.origin; } catch (_error) { return false; }
26
+ });
27
+ const product = descriptionMatch && appMatch && (footerMatch || assetMatch || preloadMatch);
28
+ if (!product) return { matched: true, product: false };
29
+
30
+ let comparisonKind = "implicit";
31
+ let comparisonValue = null;
32
+ if (base != null) {
33
+ if (Number(base) > 0) {
34
+ comparisonKind = "patchset";
35
+ comparisonValue = base;
36
+ } else if (Number(base) < 0) {
37
+ comparisonKind = "parent";
38
+ comparisonValue = String(Math.abs(Number(base)));
39
+ } else {
40
+ comparisonKind = "auto_merge";
41
+ }
42
+ }
43
+
44
+ const changePath = beforeMarker + "/+/" + number;
45
+ const apiPath = (prefix || "") + "/changes/" + encodeURIComponent(projectPath + "~" + number);
46
+ const routeKey = [location.origin, changePath, selector, filePath].join(":");
47
+ const existing = window[stateKey];
48
+ if (existing && existing.routeKey === routeKey) {
49
+ return {
50
+ matched: true,
51
+ product: true,
52
+ status: existing.status || "",
53
+ signature: [existing.status, existing.fileCount || 0, existing.commentCount || 0,
54
+ existing.diffBlockCount || 0, existing.firstParentDiffBlockCount || 0].join(":")
55
+ };
56
+ }
57
+
58
+ const route = {
59
+ project: projectPath,
60
+ number: number,
61
+ patchset: target,
62
+ selector: selector,
63
+ target: target,
64
+ base: base,
65
+ comparison: comparisonKind,
66
+ comparisonValue: comparisonValue,
67
+ filePath: filePath,
68
+ changePath: changePath,
69
+ apiPath: apiPath
70
+ };
71
+ JS
72
+ end
73
+ end
74
+ end
75
+ end
76
+ end
@@ -0,0 +1,111 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module GerritFileResourceRequestStateScript
7
+ private
8
+
9
+ def gerrit_file_resource_request_state_script
10
+ gerrit_file_resource_fetch_state_script + gerrit_file_resource_validation_state_script + <<~'JS'
11
+ const prepared = { status: "loading", product: true, routeKey: routeKey, route: route };
12
+ window[stateKey] = prepared;
13
+
14
+ const comparisonQuery = new URLSearchParams();
15
+ if (route.comparison === "patchset") comparisonQuery.set("base", route.comparisonValue);
16
+ if (route.comparison === "parent") comparisonQuery.set("parent", route.comparisonValue);
17
+ const query = comparisonQuery.toString();
18
+ const filesSuffix = "/revisions/" + encodeURIComponent(route.target) + "/files/" +
19
+ (query ? "?" + query : "");
20
+
21
+ Promise.all([
22
+ requestJson("/detail?o=ALL_REVISIONS&o=ALL_COMMITS&o=DETAILED_LABELS&o=DETAILED_ACCOUNTS&o=MESSAGES"),
23
+ requestJson("/comments?enable-context=true&context-padding=3"),
24
+ requestJson(filesSuffix)
25
+ ]).then(async ([detail, comments, files]) => {
26
+ if (window[stateKey] !== prepared || prepared.status !== "loading") return;
27
+ if (String(detail._number || "") !== route.number || detail.project !== route.project) {
28
+ throw new Error("Gerrit API change identity mismatch");
29
+ }
30
+ if (!objectRecord(detail.revisions)) throw new Error("invalid Gerrit revisions payload");
31
+ if (!Object.values(comments).every(
32
+ (records) => Array.isArray(records) && records.every(objectRecord)
33
+ )) throw new Error("invalid Gerrit comments payload");
34
+ if (!Object.values(files).every(objectRecord)) throw new Error("invalid Gerrit files payload");
35
+ const revisions = Object.values(detail.revisions || {});
36
+ const targetRevision = revisions.find(
37
+ (revision) => String(revision && revision._number || "") === route.target
38
+ );
39
+ if (!targetRevision) throw new Error("Gerrit API target patch set identity mismatch");
40
+ if (route.comparison === "patchset" && !revisions.some(
41
+ (revision) => String(revision && revision._number || "") === route.comparisonValue
42
+ )) throw new Error("Gerrit API base patch set identity mismatch");
43
+ if (route.comparison === "parent" &&
44
+ (!targetRevision.commit || !Array.isArray(targetRevision.commit.parents) ||
45
+ targetRevision.commit.parents.length < Number(route.comparisonValue))) {
46
+ throw new Error("Gerrit API parent identity mismatch");
47
+ }
48
+
49
+ let filePath = route.filePath;
50
+ if (!Object.prototype.hasOwnProperty.call(files, filePath)) {
51
+ filePath = Object.keys(files).find((path) => files[path] && files[path].old_path === route.filePath);
52
+ }
53
+ if (!filePath || !files[filePath]) throw new Error("Gerrit API file identity mismatch");
54
+
55
+ const diffQuery = new URLSearchParams(comparisonQuery);
56
+ diffQuery.set("context", "ALL");
57
+ const encodedFile = encodeURIComponent(filePath);
58
+ const diff = await requestJson("/revisions/" + encodeURIComponent(route.target) +
59
+ "/files/" + encodedFile + "/diff?" + diffQuery.toString());
60
+ validateDiff(diff, filePath);
61
+ let firstParentDiff = null;
62
+ if (route.comparison === "implicit" && targetRevision.commit &&
63
+ Array.isArray(targetRevision.commit.parents) && targetRevision.commit.parents.length > 1) {
64
+ const firstParentQuery = new URLSearchParams();
65
+ firstParentQuery.set("parent", "1");
66
+ firstParentQuery.set("context", "ALL");
67
+ firstParentDiff = await requestJson("/revisions/" + encodeURIComponent(route.target) +
68
+ "/files/" + encodedFile + "/diff?" + firstParentQuery.toString());
69
+ validateDiff(firstParentDiff, filePath);
70
+ }
71
+ if (window[stateKey] !== prepared || prepared.status !== "loading") return;
72
+ const countComments = Object.keys(comments || {}).reduce(
73
+ (total, path) => total + (Array.isArray(comments[path]) ? comments[path].length : 0), 0
74
+ );
75
+ window[stateKey] = {
76
+ status: "ready",
77
+ product: true,
78
+ routeKey: routeKey,
79
+ route: route,
80
+ detail: detail,
81
+ comments: comments,
82
+ files: files,
83
+ filePath: filePath,
84
+ requestedFilePath: route.filePath,
85
+ file: files[filePath],
86
+ diff: diff,
87
+ firstParentDiff: firstParentDiff,
88
+ fileCount: Object.keys(files).length,
89
+ commentCount: countComments,
90
+ diffBlockCount: Array.isArray(diff.content) ? diff.content.length : 0,
91
+ firstParentDiffBlockCount: firstParentDiff && Array.isArray(firstParentDiff.content)
92
+ ? firstParentDiff.content.length : 0
93
+ };
94
+ }).catch((error) => {
95
+ if (window[stateKey] !== prepared || prepared.status !== "loading") return;
96
+ window[stateKey] = {
97
+ status: "failed",
98
+ product: true,
99
+ routeKey: routeKey,
100
+ route: route,
101
+ reason: String(error && error.message || error || "Gerrit file preparation failed")
102
+ };
103
+ });
104
+
105
+ return { matched: true, product: true, status: "loading", signature: "loading" };
106
+ JS
107
+ end
108
+ end
109
+ end
110
+ end
111
+ end
@@ -0,0 +1,71 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module GerritFileResourceRouteStateScript
7
+ private
8
+
9
+ def gerrit_file_resource_route_state_script
10
+ <<~'JS'
11
+ const stateKey = "__fetchUtilGerritFileResource";
12
+ const pathname = (location.pathname || "").replace(/\/+$/, "");
13
+ const marker = pathname.lastIndexOf('/+/');
14
+ if (marker < 1) return { matched: false };
15
+
16
+ const tail = pathname.slice(marker + 3).split("/");
17
+ if (tail.length < 3 || !/^\d+$/.test(tail[0])) return { matched: false };
18
+ const selector = tail[1];
19
+ const comparison = selector.match(/^(-?[1-9]\d*|0)\.\.([1-9]\d*)$/);
20
+ const targetOnly = selector.match(/^([1-9]\d*)$/);
21
+ if (!comparison && !targetOnly) return { matched: false };
22
+
23
+ const number = tail[0];
24
+ const target = comparison ? comparison[2] : targetOnly[1];
25
+ const base = comparison ? comparison[1] : null;
26
+ const rawFilePath = tail.slice(2).join("/");
27
+ let filePath;
28
+ try { filePath = decodeURIComponent(rawFilePath); }
29
+ catch (_error) { return { matched: false }; }
30
+ const fileParts = filePath.split("/");
31
+ const magicPath = fileParts[0] === "" && fileParts.length === 2 && fileParts[1];
32
+ if (!filePath || fileParts.some((part, index) => !part && !(index === 0 && magicPath)) ||
33
+ fileParts.some((part) => part === "." || part === "..")) return { matched: false };
34
+
35
+ const beforeMarker = pathname.slice(0, marker);
36
+ const baseNode = document.querySelector("base[href]");
37
+ let prefix = "";
38
+ if (baseNode) {
39
+ try {
40
+ const baseUrl = new URL(baseNode.getAttribute("href"), location.href);
41
+ if (baseUrl.origin === location.origin) prefix = baseUrl.pathname.replace(/\/+$/, "");
42
+ } catch (_error) { prefix = ""; }
43
+ }
44
+
45
+ let projectPath = beforeMarker;
46
+ if (prefix && projectPath.indexOf(prefix + "/") === 0) {
47
+ projectPath = projectPath.slice(prefix.length);
48
+ }
49
+ if (projectPath.indexOf("/c/") === 0) {
50
+ projectPath = projectPath.slice(3);
51
+ } else {
52
+ const cMarker = projectPath.indexOf("/c/");
53
+ if (cMarker >= 0) {
54
+ prefix = projectPath.slice(0, cMarker);
55
+ projectPath = projectPath.slice(cMarker + 3);
56
+ } else {
57
+ projectPath = projectPath.replace(/^\/+/, "");
58
+ }
59
+ }
60
+
61
+ try { projectPath = decodeURIComponent(projectPath); }
62
+ catch (_error) { return { matched: false }; }
63
+ if (!projectPath || projectPath.split("/").some((part) => !part || part === "." || part === "..")) {
64
+ return { matched: false };
65
+ }
66
+ JS
67
+ end
68
+ end
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module GerritFileResourceStabilization
7
+ GERRIT_FILE_RESOURCE_STABILIZATION_PROFILE = {
8
+ host: true,
9
+ path_query: ->(uri) { uri.path.match?(%r{/\+/\d+/(?:-?\d+\.\.)?\d+/.+}) },
10
+ strategy: :stabilize_gerrit_file_resource,
11
+ fallthrough: true,
12
+ notes: "Prepare product-matched Gerrit file diff REST resources.",
13
+ tests: "spec/fetch_util/browser_stabilization_spec.rb"
14
+ }.freeze
15
+
16
+ private
17
+
18
+ def stabilize_gerrit_file_resource(page, deadline: stabilization_deadline)
19
+ state = nil
20
+ ready = retry_until_timeout(
21
+ capped_timeout(10.0, deadline: deadline),
22
+ interval: 0.1,
23
+ deadline: deadline
24
+ ) do
25
+ state = gerrit_file_resource_state(page)
26
+ return false if state.is_a?(Hash) && (state["product"] == false || state["status"] == "failed")
27
+
28
+ state.is_a?(Hash) && state["status"] == "ready"
29
+ end
30
+ fail_gerrit_file_resource_preparation(page) if !ready && state.is_a?(Hash) && state["status"] == "loading"
31
+ settle_after_stabilization(0.25, deadline: deadline) if ready
32
+ ready
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchUtil
4
+ class Browser
5
+ module SiteStabilization
6
+ module GerritFileResourceState
7
+ include GerritFileResourceRouteStateScript
8
+ include GerritFileResourceProductStateScript
9
+ include GerritFileResourceFetchStateScript
10
+ include GerritFileResourceValidationStateScript
11
+ include GerritFileResourceRequestStateScript
12
+
13
+ private
14
+
15
+ def gerrit_file_resource_state(page)
16
+ safe_evaluate(page, gerrit_file_resource_state_script, default: nil)
17
+ end
18
+
19
+ def fail_gerrit_file_resource_preparation(page)
20
+ safe_evaluate(page, <<~'JS', default: false)
21
+ (() => {
22
+ const prepared = window.__fetchUtilGerritFileResource;
23
+ if (!prepared || prepared.status !== "loading") return false;
24
+ prepared.status = "failed";
25
+ prepared.reason = "Gerrit file preparation timed out";
26
+ return true;
27
+ })()
28
+ JS
29
+ end
30
+
31
+ def gerrit_file_resource_state_script
32
+ <<~JS
33
+ (() => {
34
+ #{gerrit_file_resource_product_state_script}
35
+ #{gerrit_file_resource_request_state_script}
36
+ })()
37
+ JS
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end