fetch_util 0.5.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +49 -0
- data/README.md +3 -1
- data/Rakefile +0 -2
- data/lib/fetch_util/assets/extract.js +1 -1
- data/lib/fetch_util/browser/interaction_helpers/consent_helpers.rb +12 -17
- data/lib/fetch_util/browser/interaction_helpers/timing_helpers.rb +42 -15
- data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_commit_details_script.rb +48 -0
- data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_product_state_script.rb +80 -0
- data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_request_helpers_script.rb +53 -0
- data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_request_state_script.rb +118 -0
- data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_stabilization.rb +54 -0
- data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_state.rb +27 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_product_state.rb +55 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_request_state.rb +100 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_state.rb +59 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_modules.rb +26 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_state.rb +93 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resources.rb +73 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_thread_state.rb +92 -0
- data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_threads.rb +71 -0
- data/lib/fetch_util/browser/site_stabilization/community_and_marketplace.rb +9 -10
- data/lib/fetch_util/browser/site_stabilization/facebook_stabilization.rb +56 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_change_product_state_script.rb +97 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_change_request_state_script.rb +75 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_change_stabilization.rb +37 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_change_state.rb +39 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_fetch_state_script.rb +31 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_product_state_script.rb +76 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_request_state_script.rb +111 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_route_state_script.rb +71 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_stabilization.rb +37 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_state.rb +42 -0
- data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_validation_state_script.rb +49 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_pull_resource_product_state.rb +88 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_pull_resource_state.rb +109 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_pull_resources.rb +30 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_stabilization.rb +60 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_thread_product_state.rb +74 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_thread_timeline_state.rb +69 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_threads.rb +28 -0
- data/lib/fetch_util/browser/site_stabilization/gitea_family_visibility_script.rb +28 -0
- data/lib/fetch_util/browser/site_stabilization/github_pull_resource_stabilization.rb +31 -0
- data/lib/fetch_util/browser/site_stabilization/github_pull_resources.rb +105 -0
- data/lib/fetch_util/browser/site_stabilization/github_threads.rb +82 -0
- data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state.rb +17 -0
- data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state_script.rb +98 -0
- data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_visibility_script.rb +43 -0
- data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resources.rb +28 -0
- data/lib/fetch_util/browser/site_stabilization/gitlab_repo.rb +3 -11
- data/lib/fetch_util/browser/site_stabilization/gitlab_stabilization.rb +33 -0
- data/lib/fetch_util/browser/site_stabilization/gitlab_threads.rb +92 -0
- data/lib/fetch_util/browser/site_stabilization/social_platforms.rb +19 -38
- data/lib/fetch_util/browser/site_stabilization/travel_and_lodging.rb +3 -3
- data/lib/fetch_util/browser/site_stabilization.rb +89 -0
- data/lib/fetch_util/browser/stabilization/anubis.rb +4 -5
- data/lib/fetch_util/browser/stabilization/page_flow.rb +71 -27
- data/lib/fetch_util/browser/stabilization/spa_hydration.rb +7 -5
- data/lib/fetch_util/browser.rb +88 -17
- data/lib/fetch_util/cli.rb +69 -23
- data/lib/fetch_util/extractor.rb +8 -2
- data/lib/fetch_util/fetcher.rb +144 -43
- data/lib/fetch_util/http_redirect_client.rb +161 -0
- data/lib/fetch_util/parallel_fetcher.rb +125 -28
- data/lib/fetch_util/raw_docs_fallback.rb +158 -21
- data/lib/fetch_util/regulatory/cache_store.rb +47 -14
- data/lib/fetch_util/regulatory/fetch_records.rb +10 -7
- data/lib/fetch_util/regulatory/headers.rb +8 -1
- data/lib/fetch_util/regulatory/http_client.rb +2 -97
- data/lib/fetch_util/regulatory/orchestration.rb +10 -6
- data/lib/fetch_util/regulatory/page.rb +8 -11
- data/lib/fetch_util/regulatory/robots.rb +2 -10
- data/lib/fetch_util/regulatory/source_selection.rb +13 -3
- data/lib/fetch_util/regulatory/tdm_page.rb +9 -4
- data/lib/fetch_util/regulatory/tdm_policy.rb +16 -9
- data/lib/fetch_util/regulatory/tdm_rep.rb +4 -3
- data/lib/fetch_util/regulatory/tdm_support.rb +10 -6
- data/lib/fetch_util/regulatory.rb +5 -1
- data/lib/fetch_util/request_log.rb +24 -3
- data/lib/fetch_util/result.rb +37 -89
- data/lib/fetch_util/search_transport.rb +102 -35
- data/lib/fetch_util/searcher.rb +15 -13
- data/lib/fetch_util/version.rb +1 -1
- data/lib/fetch_util.rb +4 -1
- metadata +66 -2
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module GerritChangeProductStateScript
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def gerrit_change_product_state_script
|
|
10
|
+
<<~'JS'
|
|
11
|
+
const stateKey = "__fetchUtilGerritChange";
|
|
12
|
+
const pathname = (location.pathname || "").replace(/\/+$/, "");
|
|
13
|
+
const marker = pathname.lastIndexOf('/+/');
|
|
14
|
+
if (marker < 1) return { matched: false };
|
|
15
|
+
|
|
16
|
+
const tail = pathname.slice(marker + 3).split("/").filter(Boolean);
|
|
17
|
+
if (tail.length < 1 || tail.length > 2 || !/^\d+$/.test(tail[0]) ||
|
|
18
|
+
(tail[1] && !/^\d+$/.test(tail[1]))) return { matched: false };
|
|
19
|
+
|
|
20
|
+
const beforeMarker = pathname.slice(0, marker);
|
|
21
|
+
const baseNode = document.querySelector("base[href]");
|
|
22
|
+
let prefix = "";
|
|
23
|
+
if (baseNode) {
|
|
24
|
+
try {
|
|
25
|
+
const baseUrl = new URL(baseNode.getAttribute("href"), location.href);
|
|
26
|
+
if (baseUrl.origin === location.origin) prefix = baseUrl.pathname.replace(/\/+$/, "");
|
|
27
|
+
} catch (_error) { prefix = ""; }
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
let projectPath = beforeMarker;
|
|
31
|
+
if (projectPath.indexOf("/c/") === 0) {
|
|
32
|
+
projectPath = projectPath.slice(3);
|
|
33
|
+
} else {
|
|
34
|
+
const cMarker = projectPath.indexOf("/c/");
|
|
35
|
+
if (cMarker >= 0) {
|
|
36
|
+
prefix = projectPath.slice(0, cMarker);
|
|
37
|
+
projectPath = projectPath.slice(cMarker + 3);
|
|
38
|
+
} else {
|
|
39
|
+
if (prefix && projectPath.indexOf(prefix + "/") === 0) projectPath = projectPath.slice(prefix.length);
|
|
40
|
+
projectPath = projectPath.replace(/^\/+/, "");
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
try { projectPath = decodeURIComponent(projectPath); }
|
|
45
|
+
catch (_error) { return { matched: false }; }
|
|
46
|
+
if (!projectPath || projectPath.split("/").some((part) => !part || part === "." || part === "..")) {
|
|
47
|
+
return { matched: false };
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const description = document.querySelector('meta[name="description"]');
|
|
51
|
+
const descriptionMatch = /^Gerrit Code Review$/i.test((description && description.content || "").trim());
|
|
52
|
+
const appMatch = !!document.querySelector("gr-app");
|
|
53
|
+
const footerMatch = Array.from(document.querySelectorAll("footer, [role='contentinfo']")).some(
|
|
54
|
+
(node) => /Powered by Gerrit Code Review/i.test((node.textContent || "").trim())
|
|
55
|
+
);
|
|
56
|
+
const assetMatch = Array.from(document.querySelectorAll("script[src], link[href]")).some((node) => {
|
|
57
|
+
const value = node.getAttribute("src") || node.getAttribute("href") || "";
|
|
58
|
+
if (!/\/polygerrit_ui\/.*\/gr-app\.js(?:$|[?#])/i.test(value)) return false;
|
|
59
|
+
try { return new URL(value, location.href).origin === location.origin; } catch (_error) { return false; }
|
|
60
|
+
});
|
|
61
|
+
const preloadMatch = Array.from(document.querySelectorAll("link[href]")).some((node) => {
|
|
62
|
+
const value = node.getAttribute("href") || "";
|
|
63
|
+
if (!/\/changes\/[^/?]+\/(?:detail|comments)(?:$|[?])/i.test(value)) return false;
|
|
64
|
+
try { return new URL(value, location.href).origin === location.origin; } catch (_error) { return false; }
|
|
65
|
+
});
|
|
66
|
+
const product = descriptionMatch && appMatch && (footerMatch || assetMatch || preloadMatch);
|
|
67
|
+
if (!product) return { matched: true, product: false };
|
|
68
|
+
|
|
69
|
+
const number = tail[0];
|
|
70
|
+
const patchset = tail[1] || null;
|
|
71
|
+
const changePath = beforeMarker + "/+/" + number;
|
|
72
|
+
const apiPath = (prefix || "") + "/changes/" + encodeURIComponent(projectPath + "~" + number);
|
|
73
|
+
const routeKey = [location.origin, changePath, patchset || ""].join(":");
|
|
74
|
+
const existing = window[stateKey];
|
|
75
|
+
if (existing && existing.routeKey === routeKey) {
|
|
76
|
+
return {
|
|
77
|
+
matched: true,
|
|
78
|
+
product: true,
|
|
79
|
+
status: existing.status || "",
|
|
80
|
+
signature: [existing.status, existing.messageCount || 0, existing.commentCount || 0,
|
|
81
|
+
existing.fileCount || 0].join(":")
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const route = {
|
|
86
|
+
project: projectPath,
|
|
87
|
+
number: number,
|
|
88
|
+
patchset: patchset,
|
|
89
|
+
changePath: changePath,
|
|
90
|
+
apiPath: apiPath
|
|
91
|
+
};
|
|
92
|
+
JS
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module GerritChangeRequestStateScript
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def gerrit_change_request_state_script
|
|
10
|
+
<<~'JS'
|
|
11
|
+
const prepared = { status: "loading", product: true, routeKey: routeKey, route: route };
|
|
12
|
+
window[stateKey] = prepared;
|
|
13
|
+
|
|
14
|
+
const requestJson = async (suffix) => {
|
|
15
|
+
const url = new URL(apiPath + suffix, location.origin);
|
|
16
|
+
if (url.origin !== location.origin) throw new Error("cross-origin Gerrit API route");
|
|
17
|
+
const response = await fetch(url.href, {
|
|
18
|
+
credentials: "same-origin",
|
|
19
|
+
headers: { Accept: "application/json" }
|
|
20
|
+
});
|
|
21
|
+
if (!response.ok) throw new Error("Gerrit API returned " + response.status);
|
|
22
|
+
const text = await response.text();
|
|
23
|
+
const value = JSON.parse(text.replace(/^\)\]\}'(?:\r?\n)?/, ""));
|
|
24
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
25
|
+
throw new Error("invalid Gerrit API payload");
|
|
26
|
+
}
|
|
27
|
+
return value;
|
|
28
|
+
};
|
|
29
|
+
const selectedRevision = route.patchset || "current";
|
|
30
|
+
Promise.all([
|
|
31
|
+
requestJson("/detail?o=ALL_REVISIONS&o=ALL_COMMITS&o=DETAILED_LABELS&o=DETAILED_ACCOUNTS&o=MESSAGES"),
|
|
32
|
+
requestJson("/comments?enable-context=true&context-padding=3"),
|
|
33
|
+
requestJson("/revisions/" + encodeURIComponent(selectedRevision) + "/files/")
|
|
34
|
+
]).then(([detail, comments, files]) => {
|
|
35
|
+
if (window[stateKey] !== prepared || prepared.status !== "loading") return;
|
|
36
|
+
if (String(detail._number || "") !== number || !detail.project || detail.project !== projectPath) {
|
|
37
|
+
throw new Error("Gerrit API change identity mismatch");
|
|
38
|
+
}
|
|
39
|
+
const selectedExists = !route.patchset || Object.values(detail.revisions || {}).some(
|
|
40
|
+
(revision) => String(revision && revision._number || "") === route.patchset
|
|
41
|
+
);
|
|
42
|
+
if (!selectedExists) throw new Error("Gerrit API patch set identity mismatch");
|
|
43
|
+
const countComments = (collection) => Object.keys(collection || {}).reduce(
|
|
44
|
+
(total, path) => total + (Array.isArray(collection[path]) ? collection[path].length : 0), 0
|
|
45
|
+
);
|
|
46
|
+
window[stateKey] = {
|
|
47
|
+
status: "ready",
|
|
48
|
+
product: true,
|
|
49
|
+
routeKey: routeKey,
|
|
50
|
+
route: route,
|
|
51
|
+
detail: detail,
|
|
52
|
+
comments: comments,
|
|
53
|
+
files: files,
|
|
54
|
+
messageCount: Array.isArray(detail.messages) ? detail.messages.length : 0,
|
|
55
|
+
commentCount: countComments(comments),
|
|
56
|
+
fileCount: Object.keys(files).length
|
|
57
|
+
};
|
|
58
|
+
}).catch((error) => {
|
|
59
|
+
if (window[stateKey] !== prepared || prepared.status !== "loading") return;
|
|
60
|
+
window[stateKey] = {
|
|
61
|
+
status: "failed",
|
|
62
|
+
product: true,
|
|
63
|
+
routeKey: routeKey,
|
|
64
|
+
route: route,
|
|
65
|
+
reason: String(error && error.message || error || "Gerrit preparation failed")
|
|
66
|
+
};
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
return { matched: true, product: true, status: "loading", signature: "loading" };
|
|
70
|
+
JS
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module GerritChangeStabilization
|
|
7
|
+
GERRIT_CHANGE_STABILIZATION_PROFILE = {
|
|
8
|
+
host: true,
|
|
9
|
+
path_query: ->(uri) { uri.path.match?(%r{/\+/\d+(?:/\d+)?/?\z}) },
|
|
10
|
+
strategy: :stabilize_gerrit_change,
|
|
11
|
+
fallthrough: true,
|
|
12
|
+
notes: "Prepare product-matched Gerrit change REST resources.",
|
|
13
|
+
tests: "spec/fetch_util/browser_stabilization_spec.rb"
|
|
14
|
+
}.freeze
|
|
15
|
+
|
|
16
|
+
private
|
|
17
|
+
|
|
18
|
+
def stabilize_gerrit_change(page, deadline: stabilization_deadline)
|
|
19
|
+
state = nil
|
|
20
|
+
ready = retry_until_timeout(
|
|
21
|
+
capped_timeout(10.0, deadline: deadline),
|
|
22
|
+
interval: 0.1,
|
|
23
|
+
deadline: deadline
|
|
24
|
+
) do
|
|
25
|
+
state = gerrit_change_state(page)
|
|
26
|
+
return false if state.is_a?(Hash) && (state["product"] == false || state["status"] == "failed")
|
|
27
|
+
|
|
28
|
+
state.is_a?(Hash) && state["status"] == "ready"
|
|
29
|
+
end
|
|
30
|
+
fail_gerrit_change_preparation(page) if !ready && state.is_a?(Hash) && state["status"] == "loading"
|
|
31
|
+
settle_after_stabilization(0.25, deadline: deadline) if ready
|
|
32
|
+
ready
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module GerritChangeState
|
|
7
|
+
include GerritChangeProductStateScript
|
|
8
|
+
include GerritChangeRequestStateScript
|
|
9
|
+
|
|
10
|
+
private
|
|
11
|
+
|
|
12
|
+
def gerrit_change_state(page)
|
|
13
|
+
safe_evaluate(page, gerrit_change_state_script, default: nil)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def fail_gerrit_change_preparation(page)
|
|
17
|
+
safe_evaluate(page, <<~'JS', default: false)
|
|
18
|
+
(() => {
|
|
19
|
+
const prepared = window.__fetchUtilGerritChange;
|
|
20
|
+
if (!prepared || prepared.status !== "loading") return false;
|
|
21
|
+
prepared.status = "failed";
|
|
22
|
+
prepared.reason = "Gerrit preparation timed out";
|
|
23
|
+
return true;
|
|
24
|
+
})()
|
|
25
|
+
JS
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def gerrit_change_state_script
|
|
29
|
+
<<~JS
|
|
30
|
+
(() => {
|
|
31
|
+
#{gerrit_change_product_state_script}
|
|
32
|
+
#{gerrit_change_request_state_script}
|
|
33
|
+
})()
|
|
34
|
+
JS
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module GerritFileResourceFetchStateScript
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def gerrit_file_resource_fetch_state_script
|
|
10
|
+
<<~'JS'
|
|
11
|
+
const requestJson = async (suffix) => {
|
|
12
|
+
const url = new URL(apiPath + suffix, location.origin);
|
|
13
|
+
if (url.origin !== location.origin) throw new Error("cross-origin Gerrit API route");
|
|
14
|
+
const response = await fetch(url.href, {
|
|
15
|
+
credentials: "same-origin",
|
|
16
|
+
headers: { Accept: "application/json" }
|
|
17
|
+
});
|
|
18
|
+
if (!response.ok) throw new Error("Gerrit API returned " + response.status);
|
|
19
|
+
const text = await response.text();
|
|
20
|
+
const value = JSON.parse(text.replace(/^\)\]\}'(?:\r?\n)?/, ""));
|
|
21
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
22
|
+
throw new Error("invalid Gerrit API payload");
|
|
23
|
+
}
|
|
24
|
+
return value;
|
|
25
|
+
};
|
|
26
|
+
JS
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module GerritFileResourceProductStateScript
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def gerrit_file_resource_product_state_script
|
|
10
|
+
gerrit_file_resource_route_state_script + <<~'JS'
|
|
11
|
+
const description = document.querySelector('meta[name="description"]');
|
|
12
|
+
const descriptionMatch = /^Gerrit Code Review$/i.test((description && description.content || "").trim());
|
|
13
|
+
const appMatch = !!document.querySelector("gr-app");
|
|
14
|
+
const footerMatch = Array.from(document.querySelectorAll("footer, [role='contentinfo']")).some(
|
|
15
|
+
(node) => /Powered by Gerrit Code Review/i.test((node.textContent || "").trim())
|
|
16
|
+
);
|
|
17
|
+
const assetMatch = Array.from(document.querySelectorAll("script[src], link[href]")).some((node) => {
|
|
18
|
+
const value = node.getAttribute("src") || node.getAttribute("href") || "";
|
|
19
|
+
if (!/\/polygerrit_ui\/.*\/gr-app\.js(?:$|[?#])/i.test(value)) return false;
|
|
20
|
+
try { return new URL(value, location.href).origin === location.origin; } catch (_error) { return false; }
|
|
21
|
+
});
|
|
22
|
+
const preloadMatch = Array.from(document.querySelectorAll("link[href]")).some((node) => {
|
|
23
|
+
const value = node.getAttribute("href") || "";
|
|
24
|
+
if (!/\/changes\/[^/?]+\/(?:detail|comments)(?:$|[?])/i.test(value)) return false;
|
|
25
|
+
try { return new URL(value, location.href).origin === location.origin; } catch (_error) { return false; }
|
|
26
|
+
});
|
|
27
|
+
const product = descriptionMatch && appMatch && (footerMatch || assetMatch || preloadMatch);
|
|
28
|
+
if (!product) return { matched: true, product: false };
|
|
29
|
+
|
|
30
|
+
let comparisonKind = "implicit";
|
|
31
|
+
let comparisonValue = null;
|
|
32
|
+
if (base != null) {
|
|
33
|
+
if (Number(base) > 0) {
|
|
34
|
+
comparisonKind = "patchset";
|
|
35
|
+
comparisonValue = base;
|
|
36
|
+
} else if (Number(base) < 0) {
|
|
37
|
+
comparisonKind = "parent";
|
|
38
|
+
comparisonValue = String(Math.abs(Number(base)));
|
|
39
|
+
} else {
|
|
40
|
+
comparisonKind = "auto_merge";
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const changePath = beforeMarker + "/+/" + number;
|
|
45
|
+
const apiPath = (prefix || "") + "/changes/" + encodeURIComponent(projectPath + "~" + number);
|
|
46
|
+
const routeKey = [location.origin, changePath, selector, filePath].join(":");
|
|
47
|
+
const existing = window[stateKey];
|
|
48
|
+
if (existing && existing.routeKey === routeKey) {
|
|
49
|
+
return {
|
|
50
|
+
matched: true,
|
|
51
|
+
product: true,
|
|
52
|
+
status: existing.status || "",
|
|
53
|
+
signature: [existing.status, existing.fileCount || 0, existing.commentCount || 0,
|
|
54
|
+
existing.diffBlockCount || 0, existing.firstParentDiffBlockCount || 0].join(":")
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const route = {
|
|
59
|
+
project: projectPath,
|
|
60
|
+
number: number,
|
|
61
|
+
patchset: target,
|
|
62
|
+
selector: selector,
|
|
63
|
+
target: target,
|
|
64
|
+
base: base,
|
|
65
|
+
comparison: comparisonKind,
|
|
66
|
+
comparisonValue: comparisonValue,
|
|
67
|
+
filePath: filePath,
|
|
68
|
+
changePath: changePath,
|
|
69
|
+
apiPath: apiPath
|
|
70
|
+
};
|
|
71
|
+
JS
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module GerritFileResourceRequestStateScript
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def gerrit_file_resource_request_state_script
|
|
10
|
+
gerrit_file_resource_fetch_state_script + gerrit_file_resource_validation_state_script + <<~'JS'
|
|
11
|
+
const prepared = { status: "loading", product: true, routeKey: routeKey, route: route };
|
|
12
|
+
window[stateKey] = prepared;
|
|
13
|
+
|
|
14
|
+
const comparisonQuery = new URLSearchParams();
|
|
15
|
+
if (route.comparison === "patchset") comparisonQuery.set("base", route.comparisonValue);
|
|
16
|
+
if (route.comparison === "parent") comparisonQuery.set("parent", route.comparisonValue);
|
|
17
|
+
const query = comparisonQuery.toString();
|
|
18
|
+
const filesSuffix = "/revisions/" + encodeURIComponent(route.target) + "/files/" +
|
|
19
|
+
(query ? "?" + query : "");
|
|
20
|
+
|
|
21
|
+
Promise.all([
|
|
22
|
+
requestJson("/detail?o=ALL_REVISIONS&o=ALL_COMMITS&o=DETAILED_LABELS&o=DETAILED_ACCOUNTS&o=MESSAGES"),
|
|
23
|
+
requestJson("/comments?enable-context=true&context-padding=3"),
|
|
24
|
+
requestJson(filesSuffix)
|
|
25
|
+
]).then(async ([detail, comments, files]) => {
|
|
26
|
+
if (window[stateKey] !== prepared || prepared.status !== "loading") return;
|
|
27
|
+
if (String(detail._number || "") !== route.number || detail.project !== route.project) {
|
|
28
|
+
throw new Error("Gerrit API change identity mismatch");
|
|
29
|
+
}
|
|
30
|
+
if (!objectRecord(detail.revisions)) throw new Error("invalid Gerrit revisions payload");
|
|
31
|
+
if (!Object.values(comments).every(
|
|
32
|
+
(records) => Array.isArray(records) && records.every(objectRecord)
|
|
33
|
+
)) throw new Error("invalid Gerrit comments payload");
|
|
34
|
+
if (!Object.values(files).every(objectRecord)) throw new Error("invalid Gerrit files payload");
|
|
35
|
+
const revisions = Object.values(detail.revisions || {});
|
|
36
|
+
const targetRevision = revisions.find(
|
|
37
|
+
(revision) => String(revision && revision._number || "") === route.target
|
|
38
|
+
);
|
|
39
|
+
if (!targetRevision) throw new Error("Gerrit API target patch set identity mismatch");
|
|
40
|
+
if (route.comparison === "patchset" && !revisions.some(
|
|
41
|
+
(revision) => String(revision && revision._number || "") === route.comparisonValue
|
|
42
|
+
)) throw new Error("Gerrit API base patch set identity mismatch");
|
|
43
|
+
if (route.comparison === "parent" &&
|
|
44
|
+
(!targetRevision.commit || !Array.isArray(targetRevision.commit.parents) ||
|
|
45
|
+
targetRevision.commit.parents.length < Number(route.comparisonValue))) {
|
|
46
|
+
throw new Error("Gerrit API parent identity mismatch");
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
let filePath = route.filePath;
|
|
50
|
+
if (!Object.prototype.hasOwnProperty.call(files, filePath)) {
|
|
51
|
+
filePath = Object.keys(files).find((path) => files[path] && files[path].old_path === route.filePath);
|
|
52
|
+
}
|
|
53
|
+
if (!filePath || !files[filePath]) throw new Error("Gerrit API file identity mismatch");
|
|
54
|
+
|
|
55
|
+
const diffQuery = new URLSearchParams(comparisonQuery);
|
|
56
|
+
diffQuery.set("context", "ALL");
|
|
57
|
+
const encodedFile = encodeURIComponent(filePath);
|
|
58
|
+
const diff = await requestJson("/revisions/" + encodeURIComponent(route.target) +
|
|
59
|
+
"/files/" + encodedFile + "/diff?" + diffQuery.toString());
|
|
60
|
+
validateDiff(diff, filePath);
|
|
61
|
+
let firstParentDiff = null;
|
|
62
|
+
if (route.comparison === "implicit" && targetRevision.commit &&
|
|
63
|
+
Array.isArray(targetRevision.commit.parents) && targetRevision.commit.parents.length > 1) {
|
|
64
|
+
const firstParentQuery = new URLSearchParams();
|
|
65
|
+
firstParentQuery.set("parent", "1");
|
|
66
|
+
firstParentQuery.set("context", "ALL");
|
|
67
|
+
firstParentDiff = await requestJson("/revisions/" + encodeURIComponent(route.target) +
|
|
68
|
+
"/files/" + encodedFile + "/diff?" + firstParentQuery.toString());
|
|
69
|
+
validateDiff(firstParentDiff, filePath);
|
|
70
|
+
}
|
|
71
|
+
if (window[stateKey] !== prepared || prepared.status !== "loading") return;
|
|
72
|
+
const countComments = Object.keys(comments || {}).reduce(
|
|
73
|
+
(total, path) => total + (Array.isArray(comments[path]) ? comments[path].length : 0), 0
|
|
74
|
+
);
|
|
75
|
+
window[stateKey] = {
|
|
76
|
+
status: "ready",
|
|
77
|
+
product: true,
|
|
78
|
+
routeKey: routeKey,
|
|
79
|
+
route: route,
|
|
80
|
+
detail: detail,
|
|
81
|
+
comments: comments,
|
|
82
|
+
files: files,
|
|
83
|
+
filePath: filePath,
|
|
84
|
+
requestedFilePath: route.filePath,
|
|
85
|
+
file: files[filePath],
|
|
86
|
+
diff: diff,
|
|
87
|
+
firstParentDiff: firstParentDiff,
|
|
88
|
+
fileCount: Object.keys(files).length,
|
|
89
|
+
commentCount: countComments,
|
|
90
|
+
diffBlockCount: Array.isArray(diff.content) ? diff.content.length : 0,
|
|
91
|
+
firstParentDiffBlockCount: firstParentDiff && Array.isArray(firstParentDiff.content)
|
|
92
|
+
? firstParentDiff.content.length : 0
|
|
93
|
+
};
|
|
94
|
+
}).catch((error) => {
|
|
95
|
+
if (window[stateKey] !== prepared || prepared.status !== "loading") return;
|
|
96
|
+
window[stateKey] = {
|
|
97
|
+
status: "failed",
|
|
98
|
+
product: true,
|
|
99
|
+
routeKey: routeKey,
|
|
100
|
+
route: route,
|
|
101
|
+
reason: String(error && error.message || error || "Gerrit file preparation failed")
|
|
102
|
+
};
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
return { matched: true, product: true, status: "loading", signature: "loading" };
|
|
106
|
+
JS
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module GerritFileResourceRouteStateScript
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def gerrit_file_resource_route_state_script
|
|
10
|
+
<<~'JS'
|
|
11
|
+
const stateKey = "__fetchUtilGerritFileResource";
|
|
12
|
+
const pathname = (location.pathname || "").replace(/\/+$/, "");
|
|
13
|
+
const marker = pathname.lastIndexOf('/+/');
|
|
14
|
+
if (marker < 1) return { matched: false };
|
|
15
|
+
|
|
16
|
+
const tail = pathname.slice(marker + 3).split("/");
|
|
17
|
+
if (tail.length < 3 || !/^\d+$/.test(tail[0])) return { matched: false };
|
|
18
|
+
const selector = tail[1];
|
|
19
|
+
const comparison = selector.match(/^(-?[1-9]\d*|0)\.\.([1-9]\d*)$/);
|
|
20
|
+
const targetOnly = selector.match(/^([1-9]\d*)$/);
|
|
21
|
+
if (!comparison && !targetOnly) return { matched: false };
|
|
22
|
+
|
|
23
|
+
const number = tail[0];
|
|
24
|
+
const target = comparison ? comparison[2] : targetOnly[1];
|
|
25
|
+
const base = comparison ? comparison[1] : null;
|
|
26
|
+
const rawFilePath = tail.slice(2).join("/");
|
|
27
|
+
let filePath;
|
|
28
|
+
try { filePath = decodeURIComponent(rawFilePath); }
|
|
29
|
+
catch (_error) { return { matched: false }; }
|
|
30
|
+
const fileParts = filePath.split("/");
|
|
31
|
+
const magicPath = fileParts[0] === "" && fileParts.length === 2 && fileParts[1];
|
|
32
|
+
if (!filePath || fileParts.some((part, index) => !part && !(index === 0 && magicPath)) ||
|
|
33
|
+
fileParts.some((part) => part === "." || part === "..")) return { matched: false };
|
|
34
|
+
|
|
35
|
+
const beforeMarker = pathname.slice(0, marker);
|
|
36
|
+
const baseNode = document.querySelector("base[href]");
|
|
37
|
+
let prefix = "";
|
|
38
|
+
if (baseNode) {
|
|
39
|
+
try {
|
|
40
|
+
const baseUrl = new URL(baseNode.getAttribute("href"), location.href);
|
|
41
|
+
if (baseUrl.origin === location.origin) prefix = baseUrl.pathname.replace(/\/+$/, "");
|
|
42
|
+
} catch (_error) { prefix = ""; }
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
let projectPath = beforeMarker;
|
|
46
|
+
if (prefix && projectPath.indexOf(prefix + "/") === 0) {
|
|
47
|
+
projectPath = projectPath.slice(prefix.length);
|
|
48
|
+
}
|
|
49
|
+
if (projectPath.indexOf("/c/") === 0) {
|
|
50
|
+
projectPath = projectPath.slice(3);
|
|
51
|
+
} else {
|
|
52
|
+
const cMarker = projectPath.indexOf("/c/");
|
|
53
|
+
if (cMarker >= 0) {
|
|
54
|
+
prefix = projectPath.slice(0, cMarker);
|
|
55
|
+
projectPath = projectPath.slice(cMarker + 3);
|
|
56
|
+
} else {
|
|
57
|
+
projectPath = projectPath.replace(/^\/+/, "");
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
try { projectPath = decodeURIComponent(projectPath); }
|
|
62
|
+
catch (_error) { return { matched: false }; }
|
|
63
|
+
if (!projectPath || projectPath.split("/").some((part) => !part || part === "." || part === "..")) {
|
|
64
|
+
return { matched: false };
|
|
65
|
+
}
|
|
66
|
+
JS
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module GerritFileResourceStabilization
|
|
7
|
+
GERRIT_FILE_RESOURCE_STABILIZATION_PROFILE = {
|
|
8
|
+
host: true,
|
|
9
|
+
path_query: ->(uri) { uri.path.match?(%r{/\+/\d+/(?:-?\d+\.\.)?\d+/.+}) },
|
|
10
|
+
strategy: :stabilize_gerrit_file_resource,
|
|
11
|
+
fallthrough: true,
|
|
12
|
+
notes: "Prepare product-matched Gerrit file diff REST resources.",
|
|
13
|
+
tests: "spec/fetch_util/browser_stabilization_spec.rb"
|
|
14
|
+
}.freeze
|
|
15
|
+
|
|
16
|
+
private
|
|
17
|
+
|
|
18
|
+
def stabilize_gerrit_file_resource(page, deadline: stabilization_deadline)
|
|
19
|
+
state = nil
|
|
20
|
+
ready = retry_until_timeout(
|
|
21
|
+
capped_timeout(10.0, deadline: deadline),
|
|
22
|
+
interval: 0.1,
|
|
23
|
+
deadline: deadline
|
|
24
|
+
) do
|
|
25
|
+
state = gerrit_file_resource_state(page)
|
|
26
|
+
return false if state.is_a?(Hash) && (state["product"] == false || state["status"] == "failed")
|
|
27
|
+
|
|
28
|
+
state.is_a?(Hash) && state["status"] == "ready"
|
|
29
|
+
end
|
|
30
|
+
fail_gerrit_file_resource_preparation(page) if !ready && state.is_a?(Hash) && state["status"] == "loading"
|
|
31
|
+
settle_after_stabilization(0.25, deadline: deadline) if ready
|
|
32
|
+
ready
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FetchUtil
|
|
4
|
+
class Browser
|
|
5
|
+
module SiteStabilization
|
|
6
|
+
module GerritFileResourceState
|
|
7
|
+
include GerritFileResourceRouteStateScript
|
|
8
|
+
include GerritFileResourceProductStateScript
|
|
9
|
+
include GerritFileResourceFetchStateScript
|
|
10
|
+
include GerritFileResourceValidationStateScript
|
|
11
|
+
include GerritFileResourceRequestStateScript
|
|
12
|
+
|
|
13
|
+
private
|
|
14
|
+
|
|
15
|
+
def gerrit_file_resource_state(page)
|
|
16
|
+
safe_evaluate(page, gerrit_file_resource_state_script, default: nil)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def fail_gerrit_file_resource_preparation(page)
|
|
20
|
+
safe_evaluate(page, <<~'JS', default: false)
|
|
21
|
+
(() => {
|
|
22
|
+
const prepared = window.__fetchUtilGerritFileResource;
|
|
23
|
+
if (!prepared || prepared.status !== "loading") return false;
|
|
24
|
+
prepared.status = "failed";
|
|
25
|
+
prepared.reason = "Gerrit file preparation timed out";
|
|
26
|
+
return true;
|
|
27
|
+
})()
|
|
28
|
+
JS
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def gerrit_file_resource_state_script
|
|
32
|
+
<<~JS
|
|
33
|
+
(() => {
|
|
34
|
+
#{gerrit_file_resource_product_state_script}
|
|
35
|
+
#{gerrit_file_resource_request_state_script}
|
|
36
|
+
})()
|
|
37
|
+
JS
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|