katello 4.13.0 → 4.13.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of katello might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/app/controllers/katello/api/rhsm/candlepin_proxies_controller.rb +8 -0
- data/app/lib/actions/katello/repository/discover.rb +11 -4
- data/app/lib/actions/pulp3/repository/create_publication.rb +4 -0
- data/app/lib/katello/repo_discovery.rb +4 -190
- data/app/lib/katello/resources/discovery/container.rb +127 -0
- data/app/lib/katello/resources/discovery/yum.rb +95 -0
- data/app/lib/katello/util/http_helper.rb +15 -0
- data/app/models/732bd3db9f64c621c64d2be4f2a838727aac0845.patch +61 -0
- data/app/models/katello/repository.rb.bak +978 -0
- data/app/models/katello/root_repository.rb +10 -0
- data/app/services/katello/pulp3/content_view_version/import_validator.rb.bak +166 -0
- data/app/services/katello/pulp3/content_view_version/importable_repositories.rb.bak +164 -0
- data/app/services/katello/repository_type.rb +1 -1
- data/config/initializers/monkeys.rb +0 -1
- data/engines/bastion_katello/app/assets/javascripts/bastion_katello/products/details/repositories/details/views/repository-info.html +3 -0
- data/lib/katello/repository_types/docker.rb +1 -0
- data/lib/katello/repository_types/yum.rb +1 -0
- data/lib/katello/tasks/update_repository_expiry.rake +114 -0
- data/lib/katello/version.rb +1 -1
- data/lib/katello.rb +0 -2
- data/locale/bn/katello.po.time_stamp +0 -0
- data/locale/bn_IN/katello.po.time_stamp +0 -0
- data/locale/ca/katello.po.time_stamp +0 -0
- data/locale/cs/katello.po.time_stamp +0 -0
- data/locale/cs_CZ/katello.po.time_stamp +0 -0
- data/locale/de/katello.po.time_stamp +0 -0
- data/locale/de_AT/katello.po.time_stamp +0 -0
- data/locale/de_DE/katello.po.time_stamp +0 -0
- data/locale/el/katello.po.time_stamp +0 -0
- data/locale/en/katello.po.time_stamp +0 -0
- data/locale/en_GB/katello.po.time_stamp +0 -0
- data/locale/en_US/katello.po.time_stamp +0 -0
- data/locale/es/katello.po.time_stamp +0 -0
- data/locale/et_EE/katello.po.time_stamp +0 -0
- data/locale/fr/katello.po.time_stamp +0 -0
- data/locale/gl/katello.po.time_stamp +0 -0
- data/locale/gu/katello.po.time_stamp +0 -0
- data/locale/he_IL/katello.po.time_stamp +0 -0
- data/locale/hi/katello.po.time_stamp +0 -0
- data/locale/id/katello.po.time_stamp +0 -0
- data/locale/it/katello.po.time_stamp +0 -0
- data/locale/ja/katello.po.time_stamp +0 -0
- data/locale/ka/katello.po.time_stamp +0 -0
- data/locale/kn/katello.po.time_stamp +0 -0
- data/locale/ko/katello.po.time_stamp +0 -0
- data/locale/ml_IN/katello.po.time_stamp +0 -0
- data/locale/mr/katello.po.time_stamp +0 -0
- data/locale/nl_NL/katello.po.time_stamp +0 -0
- data/locale/or/katello.po.time_stamp +0 -0
- data/locale/pa/katello.po.time_stamp +0 -0
- data/locale/pl/katello.po.time_stamp +0 -0
- data/locale/pl_PL/katello.po.time_stamp +0 -0
- data/locale/pt/katello.po.time_stamp +0 -0
- data/locale/pt_BR/katello.po.time_stamp +0 -0
- data/locale/ro/katello.po.time_stamp +0 -0
- data/locale/ro_RO/katello.po.time_stamp +0 -0
- data/locale/ru/katello.po.time_stamp +0 -0
- data/locale/sl/katello.po.time_stamp +0 -0
- data/locale/sv_SE/katello.po.time_stamp +0 -0
- data/locale/ta/katello.po.time_stamp +0 -0
- data/locale/ta_IN/katello.po.time_stamp +0 -0
- data/locale/te/katello.po.time_stamp +0 -0
- data/locale/tr/katello.po.time_stamp +0 -0
- data/locale/vi/katello.po.time_stamp +0 -0
- data/locale/vi_VN/katello.po.time_stamp +0 -0
- data/locale/zh/katello.po.time_stamp +0 -0
- data/locale/zh_CN/katello.po.time_stamp +0 -0
- data/locale/zh_TW/katello.po.time_stamp +0 -0
- data/webpack/global_test_setup.js.bak +59 -0
- metadata +83 -24
- data/db/migrate/20240531193030_remove_sha1_repository_checksum_type.rb +0 -10
- data/lib/monkeys/anemone.rb +0 -33
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7b483b80f6a5f7194a08c115f000a799cf5fbfadb5ff15a3c23425474858083e
|
4
|
+
data.tar.gz: 40c8c3a668d0cf608ad9d4c051a5eebd6fa8c1bca6f6b5ee9821ebb53aabaff1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4752d03a68ac8daa4499926108b811b644d29fea06dde68e8a7ebe0f3571e23d6329c23d8acc6f66eb49b675118d3a7bb7e9e00d08f499b77597f841be20c63c
|
7
|
+
data.tar.gz: ed21c09ff884e6a3b8a7e23dd2e824810507b0d84b058368c731e5479fcd9a61d778d0d7e4110db5b2ca8d6c14e9e64521b9e80660bfc06a05579b9226e366e6
|
@@ -259,6 +259,14 @@ module Katello
|
|
259
259
|
def facts
|
260
260
|
User.current = User.anonymous_admin
|
261
261
|
@host.update_candlepin_associations(rhsm_params)
|
262
|
+
if params[:environments]
|
263
|
+
new_envs = params[:environments].map do |env|
|
264
|
+
get_content_view_environment("cp_id", env['id'])
|
265
|
+
end
|
266
|
+
new_envs.compact!
|
267
|
+
Rails.logger.debug "Setting new content view environments for host #{@host.to_label}: #{new_envs.map(&:label)}"
|
268
|
+
@host.content_facet.content_view_environments = new_envs
|
269
|
+
end
|
262
270
|
update_host_registered_through(@host, request.headers)
|
263
271
|
@host.refresh_statuses([::Katello::RhelLifecycleStatus])
|
264
272
|
render :json => {:content => _("Facts successfully updated.")}, :status => :ok
|
@@ -31,10 +31,17 @@ module Actions
|
|
31
31
|
(on nil do
|
32
32
|
unless output[:to_follow].empty?
|
33
33
|
password = decrypt_field(input[:upstream_password])
|
34
|
-
repo_discovery = ::Katello::RepoDiscovery.
|
35
|
-
|
36
|
-
|
37
|
-
|
34
|
+
repo_discovery = ::Katello::RepoDiscovery.class_for(input[:content_type]).new(
|
35
|
+
input[:url],
|
36
|
+
output[:crawled],
|
37
|
+
output[:repo_urls],
|
38
|
+
output[:to_follow],
|
39
|
+
{
|
40
|
+
upstream_username: input[:upstream_username],
|
41
|
+
upstream_password: password,
|
42
|
+
search: input[:search]
|
43
|
+
}
|
44
|
+
)
|
38
45
|
|
39
46
|
repo_discovery.run(output[:to_follow].shift)
|
40
47
|
suspend { |suspended_action| world.clock.ping suspended_action, 0.001 }
|
@@ -14,6 +14,10 @@ module Actions
|
|
14
14
|
def invoke_external_task
|
15
15
|
unless input[:skip_publication_creation]
|
16
16
|
repository = ::Katello::Repository.find(input[:repository_id])
|
17
|
+
if repository.root.sha1_checksum?
|
18
|
+
repository.root.remove_sha1_checksum_type
|
19
|
+
repository.root.save!
|
20
|
+
end
|
17
21
|
output[:response] = repository.backend_service(smart_proxy).with_mirror_adapter.create_publication
|
18
22
|
end
|
19
23
|
end
|
@@ -4,22 +4,11 @@ module Katello
|
|
4
4
|
class RepoDiscovery
|
5
5
|
include Katello::Util::HttpProxy
|
6
6
|
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
upstream_password = nil, search = '*', crawled = [],
|
12
|
-
found = [], to_follow = [])
|
13
|
-
@uri = uri(url)
|
14
|
-
@content_type = content_type
|
15
|
-
@upstream_username = upstream_username.empty? ? nil : upstream_username
|
16
|
-
@upstream_password = upstream_password.empty? ? nil : upstream_password
|
17
|
-
@search = search
|
18
|
-
@found = found
|
19
|
-
@crawled = crawled
|
20
|
-
@to_follow = to_follow
|
7
|
+
def self.class_for(content_type)
|
8
|
+
repo_discovery_class = RepositoryTypeManager.find_repository_type(content_type)&.repo_discovery_class
|
9
|
+
fail _("Content type does not support repo discovery") unless repo_discovery_class
|
10
|
+
repo_discovery_class
|
21
11
|
end
|
22
|
-
# rubocop:enable Metrics/ParameterLists
|
23
12
|
|
24
13
|
def uri(url)
|
25
14
|
#add a / on the end, as directories require it or else
|
@@ -27,180 +16,5 @@ module Katello
|
|
27
16
|
url += '/' unless url.ends_with?('/')
|
28
17
|
URI(url)
|
29
18
|
end
|
30
|
-
|
31
|
-
def run(resume_point)
|
32
|
-
if @content_type == 'docker'
|
33
|
-
docker_search
|
34
|
-
else
|
35
|
-
if @uri.scheme == 'file'
|
36
|
-
file_crawl(uri(resume_point))
|
37
|
-
elsif %w(http https).include?(@uri.scheme)
|
38
|
-
http_crawl(uri(resume_point))
|
39
|
-
else
|
40
|
-
fail _("Unsupported URL protocol %s.") % @uri.scheme
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
private
|
46
|
-
|
47
|
-
def parse_parameter(field_value)
|
48
|
-
field_value.lstrip!
|
49
|
-
i = field_value.index(/[ \t=;,]/) || field_value.length
|
50
|
-
name = field_value.slice!(0, i).downcase(:ascii)
|
51
|
-
field_value.lstrip!
|
52
|
-
if field_value.delete_prefix!('=')
|
53
|
-
field_value.lstrip!
|
54
|
-
if field_value.delete_prefix!('"')
|
55
|
-
value = ''
|
56
|
-
until field_value.empty?
|
57
|
-
break if field_value.delete_prefix!('"')
|
58
|
-
field_value.delete_prefix!("\\")
|
59
|
-
value += field_value.slice!(0, 1) || ''
|
60
|
-
end
|
61
|
-
else
|
62
|
-
i = field_value.index(/[;,]/) || field_value.length
|
63
|
-
value = field_value.slice!(0, i)
|
64
|
-
end
|
65
|
-
end
|
66
|
-
{name: name, value: value || ''}
|
67
|
-
end
|
68
|
-
|
69
|
-
def parse_parameters(field_value)
|
70
|
-
seen_rel = false
|
71
|
-
has_next_rel = false
|
72
|
-
has_anchor = false
|
73
|
-
until field_value.empty?
|
74
|
-
field_value.lstrip!
|
75
|
-
break if field_value.delete_prefix!(';').nil?
|
76
|
-
param = parse_parameter(field_value)
|
77
|
-
case
|
78
|
-
when param[:name] == 'rel' && !seen_rel
|
79
|
-
seen_rel = true
|
80
|
-
has_next_rel = param[:value].downcase(:ascii).split(/[ \t]/).include?('next')
|
81
|
-
when param[:name] == 'anchor'
|
82
|
-
has_anchor = true
|
83
|
-
end
|
84
|
-
end
|
85
|
-
{has_next_rel: has_next_rel, has_anchor: has_anchor}
|
86
|
-
end
|
87
|
-
|
88
|
-
def get_next_link(link_header)
|
89
|
-
# This code mostly follows Appendix B "Algorithms of Parsing Link Header Fields" of RFC 8288
|
90
|
-
# "Web Linking", <https://www.rfc-editor.org/rfc/rfc8288#appendix-B> (that RFC appears to be
|
91
|
-
# silent about multiple "next" links, so just use the first one and ignore any additional
|
92
|
-
# ones, in the general spirit of being lenient):
|
93
|
-
return nil if link_header.nil?
|
94
|
-
field_value = link_header.clone
|
95
|
-
until field_value.empty?
|
96
|
-
# The following ignores any junk preceding the next <...> link URL:
|
97
|
-
m = field_value.match(/<(.*)>/)
|
98
|
-
break unless m
|
99
|
-
target_string = m[1]
|
100
|
-
field_value = m.post_match
|
101
|
-
params = parse_parameters(field_value)
|
102
|
-
if params[:has_next_rel]
|
103
|
-
# To keep it simple, ignore a link with an (unlikely) anchor parameter; but the RFC
|
104
|
-
# mandates that we "MUST NOT process the link without applying the anchor", so just raise
|
105
|
-
# an exception in that (unlikely) case:
|
106
|
-
fail "anchor not supported" if params[:has_anchor]
|
107
|
-
return target_string
|
108
|
-
end
|
109
|
-
end
|
110
|
-
nil
|
111
|
-
end
|
112
|
-
|
113
|
-
def docker_search
|
114
|
-
request_params = {
|
115
|
-
method: :get,
|
116
|
-
headers: { accept: :json },
|
117
|
-
url: "#{@uri}v1/search?q=#{@search}"
|
118
|
-
}
|
119
|
-
|
120
|
-
request_params[:user] = @upstream_username unless @upstream_username.empty?
|
121
|
-
request_params[:password] = @upstream_password unless @upstream_password.empty?
|
122
|
-
request_params[:proxy] = proxy_uri if proxy
|
123
|
-
|
124
|
-
begin
|
125
|
-
results = RestClient::Request.execute(request_params)
|
126
|
-
JSON.parse(results)['results'].each do |result|
|
127
|
-
@found << result['name']
|
128
|
-
end
|
129
|
-
rescue
|
130
|
-
# Note: v2 endpoint does not support search
|
131
|
-
request_params[:url] = "#{@uri}v2/_catalog"
|
132
|
-
loop do
|
133
|
-
results = RestClient::Request.execute(request_params)
|
134
|
-
JSON.parse(results)['repositories'].each do |result|
|
135
|
-
@found << result
|
136
|
-
end
|
137
|
-
next_uri = get_next_link(results.headers[:link])
|
138
|
-
break if next_uri.nil?
|
139
|
-
request_params[:url] = URI(request_params[:url]).merge(next_uri).to_s
|
140
|
-
end
|
141
|
-
end
|
142
|
-
@found.sort!
|
143
|
-
end
|
144
|
-
|
145
|
-
def anemone_proxy_details
|
146
|
-
details = {}
|
147
|
-
|
148
|
-
if proxy
|
149
|
-
details[:proxy_host] = proxy_host
|
150
|
-
details[:proxy_port] = proxy_port
|
151
|
-
details[:proxy_user] = proxy.username
|
152
|
-
details[:proxy_password] = proxy.password
|
153
|
-
end
|
154
|
-
|
155
|
-
details
|
156
|
-
end
|
157
|
-
|
158
|
-
def http_crawl(resume_point)
|
159
|
-
resume_point_uri = URI(resume_point)
|
160
|
-
resume_point_uri.user = @upstream_username if @upstream_username
|
161
|
-
resume_point_uri.password = @upstream_password if @upstream_password
|
162
|
-
|
163
|
-
Anemone.crawl(resume_point_uri, anemone_proxy_details) do |anemone|
|
164
|
-
anemone.focus_crawl do |page|
|
165
|
-
@crawled << page.url.path
|
166
|
-
|
167
|
-
page.links.each do |link|
|
168
|
-
if link.path.ends_with?('/repodata/')
|
169
|
-
page_url = page.url.clone
|
170
|
-
page_url.user = nil
|
171
|
-
page_url.password = nil
|
172
|
-
@found << page_url.to_s
|
173
|
-
else
|
174
|
-
@to_follow << link.to_s if should_follow?(link.path)
|
175
|
-
end
|
176
|
-
end
|
177
|
-
page.discard_doc! #saves memory, doc not needed
|
178
|
-
[]
|
179
|
-
end
|
180
|
-
end
|
181
|
-
end
|
182
|
-
|
183
|
-
def file_crawl(resume_point)
|
184
|
-
if resume_point.path.ends_with?('/repodata/')
|
185
|
-
found_path = Pathname(resume_point.path).parent.to_s
|
186
|
-
@found << "file://#{found_path}"
|
187
|
-
end
|
188
|
-
if resume_point.path == @uri.path
|
189
|
-
Dir.glob("#{@uri.path}**/").each { |path| @to_follow << path }
|
190
|
-
@to_follow.shift
|
191
|
-
end
|
192
|
-
@crawled << resume_point.path
|
193
|
-
end
|
194
|
-
|
195
|
-
def should_follow?(path)
|
196
|
-
#Verify:
|
197
|
-
# * link's path starts with the base url
|
198
|
-
# * link hasn't already been crawled
|
199
|
-
# * link ends with '/' so it should be a directory
|
200
|
-
# * link doesn't end with '/Packages/', as this increases
|
201
|
-
# processing time and memory usage considerably
|
202
|
-
return path.starts_with?(@uri.path) && !@crawled.include?(path) &&
|
203
|
-
path.ends_with?('/') && !path.ends_with?('/Packages/')
|
204
|
-
end
|
205
19
|
end
|
206
20
|
end
|
@@ -0,0 +1,127 @@
|
|
1
|
+
module Katello
|
2
|
+
module Resources
|
3
|
+
module Discovery
|
4
|
+
class Container < RepoDiscovery
|
5
|
+
attr_reader :found, :crawled, :to_follow
|
6
|
+
def initialize(url, crawled = [], found = [], to_follow = [],
|
7
|
+
upstream_credentials_and_search = {
|
8
|
+
upstream_username: nil,
|
9
|
+
upstream_password: nil,
|
10
|
+
search: '*'
|
11
|
+
})
|
12
|
+
@uri = uri(url)
|
13
|
+
@upstream_username = upstream_credentials_and_search[:upstream_username].presence
|
14
|
+
@upstream_password = upstream_credentials_and_search[:upstream_password].presence
|
15
|
+
@search = upstream_credentials_and_search.fetch(:search, '*')
|
16
|
+
@found = found
|
17
|
+
@crawled = crawled
|
18
|
+
@to_follow = to_follow
|
19
|
+
end
|
20
|
+
|
21
|
+
def run(_resume_point)
|
22
|
+
docker_search
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
def parse_parameter(field_value)
|
28
|
+
field_value.lstrip!
|
29
|
+
i = field_value.index(/[ \t=;,]/) || field_value.length
|
30
|
+
name = field_value.slice!(0, i).downcase(:ascii)
|
31
|
+
field_value.lstrip!
|
32
|
+
if field_value.delete_prefix!('=')
|
33
|
+
field_value.lstrip!
|
34
|
+
if field_value.delete_prefix!('"')
|
35
|
+
value = ''
|
36
|
+
until field_value.empty?
|
37
|
+
break if field_value.delete_prefix!('"')
|
38
|
+
field_value.delete_prefix!("\\")
|
39
|
+
value += field_value.slice!(0, 1) || ''
|
40
|
+
end
|
41
|
+
else
|
42
|
+
i = field_value.index(/[;,]/) || field_value.length
|
43
|
+
value = field_value.slice!(0, i)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
{name: name, value: value || ''}
|
47
|
+
end
|
48
|
+
|
49
|
+
def parse_parameters(field_value)
|
50
|
+
seen_rel = false
|
51
|
+
has_next_rel = false
|
52
|
+
has_anchor = false
|
53
|
+
until field_value.empty?
|
54
|
+
field_value.lstrip!
|
55
|
+
break if field_value.delete_prefix!(';').nil?
|
56
|
+
param = parse_parameter(field_value)
|
57
|
+
case
|
58
|
+
when param[:name] == 'rel' && !seen_rel
|
59
|
+
seen_rel = true
|
60
|
+
has_next_rel = param[:value].downcase(:ascii).split(/[ \t]/).include?('next')
|
61
|
+
when param[:name] == 'anchor'
|
62
|
+
has_anchor = true
|
63
|
+
end
|
64
|
+
end
|
65
|
+
{has_next_rel: has_next_rel, has_anchor: has_anchor}
|
66
|
+
end
|
67
|
+
|
68
|
+
def get_next_link(link_header)
|
69
|
+
# This code mostly follows Appendix B "Algorithms of Parsing Link Header Fields" of RFC 8288
|
70
|
+
# "Web Linking", <https://www.rfc-editor.org/rfc/rfc8288#appendix-B> (that RFC appears to be
|
71
|
+
# silent about multiple "next" links, so just use the first one and ignore any additional
|
72
|
+
# ones, in the general spirit of being lenient):
|
73
|
+
return nil if link_header.nil?
|
74
|
+
field_value = link_header.clone
|
75
|
+
until field_value.empty?
|
76
|
+
# The following ignores any junk preceding the next <...> link URL:
|
77
|
+
m = field_value.match(/<(.*)>/)
|
78
|
+
break unless m
|
79
|
+
target_string = m[1]
|
80
|
+
field_value = m.post_match
|
81
|
+
params = parse_parameters(field_value)
|
82
|
+
if params[:has_next_rel]
|
83
|
+
# To keep it simple, ignore a link with an (unlikely) anchor parameter; but the RFC
|
84
|
+
# mandates that we "MUST NOT process the link without applying the anchor", so just raise
|
85
|
+
# an exception in that (unlikely) case:
|
86
|
+
fail "anchor not supported" if params[:has_anchor]
|
87
|
+
return target_string
|
88
|
+
end
|
89
|
+
end
|
90
|
+
nil
|
91
|
+
end
|
92
|
+
|
93
|
+
def docker_search
|
94
|
+
request_params = {
|
95
|
+
method: :get,
|
96
|
+
headers: { accept: :json },
|
97
|
+
url: "#{@uri}v1/search?q=#{@search}"
|
98
|
+
}
|
99
|
+
|
100
|
+
request_params[:user] = @upstream_username if @upstream_username
|
101
|
+
request_params[:password] = @upstream_password if @upstream_password
|
102
|
+
request_params[:proxy] = proxy_uri if proxy
|
103
|
+
|
104
|
+
begin
|
105
|
+
results = RestClient::Request.execute(request_params)
|
106
|
+
JSON.parse(results)['results'].each do |result|
|
107
|
+
@found << result['name']
|
108
|
+
end
|
109
|
+
rescue
|
110
|
+
# Note: v2 endpoint does not support search
|
111
|
+
request_params[:url] = "#{@uri}v2/_catalog"
|
112
|
+
loop do
|
113
|
+
results = RestClient::Request.execute(request_params)
|
114
|
+
JSON.parse(results)['repositories'].each do |result|
|
115
|
+
@found << result
|
116
|
+
end
|
117
|
+
next_uri = get_next_link(results.headers[:link])
|
118
|
+
break if next_uri.nil?
|
119
|
+
request_params[:url] = URI(request_params[:url]).merge(next_uri).to_s
|
120
|
+
end
|
121
|
+
end
|
122
|
+
@found.sort!
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
require 'spidr'
|
2
|
+
|
3
|
+
module Katello
|
4
|
+
module Resources
|
5
|
+
module Discovery
|
6
|
+
class Yum < RepoDiscovery
|
7
|
+
attr_reader :found, :crawled, :to_follow
|
8
|
+
def initialize(url, crawled = [], found = [], to_follow = [],
|
9
|
+
upstream_credentials_and_search = {
|
10
|
+
upstream_username: nil,
|
11
|
+
upstream_password: nil
|
12
|
+
})
|
13
|
+
@uri = uri(url)
|
14
|
+
@upstream_username = upstream_credentials_and_search[:upstream_username].presence
|
15
|
+
@upstream_password = upstream_credentials_and_search[:upstream_password].presence
|
16
|
+
@found = found
|
17
|
+
@crawled = crawled
|
18
|
+
@to_follow = to_follow
|
19
|
+
end
|
20
|
+
|
21
|
+
def run(resume_point)
|
22
|
+
if @uri.scheme == 'file'
|
23
|
+
crawl_file_path(uri(resume_point))
|
24
|
+
elsif %w(http https).include?(@uri.scheme)
|
25
|
+
spidr_crawl_pages(resume_point)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def crawl_file_path(url)
|
32
|
+
if url.path.ends_with?('/repodata/')
|
33
|
+
found_path = Pathname(url.path).parent.to_s
|
34
|
+
@found << "file://#{found_path}"
|
35
|
+
end
|
36
|
+
if url.path == @uri.path
|
37
|
+
Dir.glob("#{@uri.path}**/").each { |path| @to_follow << path }
|
38
|
+
@to_follow.shift
|
39
|
+
end
|
40
|
+
@crawled << url.path
|
41
|
+
end
|
42
|
+
|
43
|
+
def spidr_proxy_details
|
44
|
+
details = {}
|
45
|
+
|
46
|
+
if proxy
|
47
|
+
details[:host] = proxy_host
|
48
|
+
details[:port] = proxy_port
|
49
|
+
details[:user] = proxy.username
|
50
|
+
details[:password] = proxy.password
|
51
|
+
end
|
52
|
+
|
53
|
+
details
|
54
|
+
end
|
55
|
+
|
56
|
+
def spidr_crawl_pages(url)
|
57
|
+
user, password = @upstream_username, @upstream_password
|
58
|
+
Spidr.site(url, proxy: spidr_proxy_details) do |spider|
|
59
|
+
spider.authorized.add(url, user, password) if user && password
|
60
|
+
spider.every_page do |page|
|
61
|
+
page.url.query = nil
|
62
|
+
@crawled << page.url.to_s
|
63
|
+
process_page_urls(page.urls)
|
64
|
+
spider.skip_page!
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def process_page_urls(urls)
|
70
|
+
urls.each do |url|
|
71
|
+
# Remove query parameters to avoid duplicate processing of URLs with sorting parameters etc
|
72
|
+
url.query = nil
|
73
|
+
if url.path.ends_with? 'repodata/'
|
74
|
+
@found << url.to_s.split('repodata/').first
|
75
|
+
else
|
76
|
+
@to_follow << url.to_s if should_follow?(url)
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def should_follow?(url)
|
82
|
+
#Verify:
|
83
|
+
# * link's path includes the base url
|
84
|
+
# * link hasn't already been crawled
|
85
|
+
# * link ends with '/' so it should be a directory
|
86
|
+
# * link doesn't end with '/Packages/', as this increases
|
87
|
+
# processing time and memory usage considerably
|
88
|
+
|
89
|
+
return url.hostname == @uri.hostname && !@crawled.include?(url.to_s) &&
|
90
|
+
url.path.ends_with?('/') && !url.path.ends_with?('/Packages/')
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
#
|
2
|
+
# Commons used helper methods pertaining to HTTP/Rest client etc
|
3
|
+
#
|
4
|
+
module Katello
|
5
|
+
module Util
|
6
|
+
module HttpHelper
|
7
|
+
def ignore_404_exception(*)
|
8
|
+
yield
|
9
|
+
rescue api.api_exception_class => e
|
10
|
+
raise e unless e.code == 404
|
11
|
+
nil
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
From 732bd3db9f64c621c64d2be4f2a838727aac0845 Mon Sep 17 00:00:00 2001
|
2
|
+
From: Quinn James <qjames@redhat.com>
|
3
|
+
Date: Fri, 23 Jun 2023 19:06:34 +0000
|
4
|
+
Subject: [PATCH] Fixes #36530 - Fixed up
|
5
|
+
/katello/api/sync_plans/:sync_plan_id/products and
|
6
|
+
/katello/api/repositories/:repository_id/sync API endpoints.
|
7
|
+
|
8
|
+
---
|
9
|
+
app/controllers/katello/api/v2/sync_controller.rb | 1 -
|
10
|
+
app/models/katello/repository.rb | 4 ++++
|
11
|
+
app/views/katello/api/v2/sync/index.json.rabl | 1 -
|
12
|
+
config/routes/api/v2.rb | 1 +
|
13
|
+
4 files changed, 5 insertions(+), 2 deletions(-)
|
14
|
+
delete mode 100644 app/views/katello/api/v2/sync/index.json.rabl
|
15
|
+
|
16
|
+
diff --git a/app/controllers/katello/api/v2/sync_controller.rb b/app/controllers/katello/api/v2/sync_controller.rb
|
17
|
+
index b43d502264f..cf324a004f6 100644
|
18
|
+
--- a/app/controllers/katello/api/v2/sync_controller.rb
|
19
|
+
+++ b/app/controllers/katello/api/v2/sync_controller.rb
|
20
|
+
@@ -4,7 +4,6 @@ class Api::V2::SyncController < Api::V2::ApiController
|
21
|
+
before_action :find_object, :only => [:index]
|
22
|
+
before_action :ensure_library, :only => [:index]
|
23
|
+
|
24
|
+
- api :GET, "/organizations/:organization_id/products/:product_id/sync", N_("Get status of repo synchronisation for given product")
|
25
|
+
api :GET, "/repositories/:repository_id/sync", N_("Get status of synchronisation for given repository")
|
26
|
+
def index
|
27
|
+
respond_for_async(:resource => @obj.sync_status)
|
28
|
+
diff --git a/app/models/katello/repository.rb b/app/models/katello/repository.rb
|
29
|
+
index 1fbf499ec44..ccd76734ba6 100644
|
30
|
+
--- a/app/models/katello/repository.rb
|
31
|
+
+++ b/app/models/katello/repository.rb
|
32
|
+
@@ -947,6 +947,10 @@ def in_content_view?(content_view)
|
33
|
+
content_view.repositories.include? self
|
34
|
+
end
|
35
|
+
|
36
|
+
+ def sync_status
|
37
|
+
+ return latest_dynflow_sync
|
38
|
+
+ end
|
39
|
+
+
|
40
|
+
protected
|
41
|
+
|
42
|
+
def unit_type_for_removal(type_class = nil)
|
43
|
+
diff --git a/app/views/katello/api/v2/sync/index.json.rabl b/app/views/katello/api/v2/sync/index.json.rabl
|
44
|
+
deleted file mode 100644
|
45
|
+
index 4296aca962b..00000000000
|
46
|
+
--- a/app/views/katello/api/v2/sync/index.json.rabl
|
47
|
+
+++ /dev/null
|
48
|
+
@@ -1 +0,0 @@
|
49
|
+
-extends 'katello/api/v2/common/async'
|
50
|
+
diff --git a/config/routes/api/v2.rb b/config/routes/api/v2.rb
|
51
|
+
index 67ad91dbc2c..9ec5312b4e0 100644
|
52
|
+
--- a/config/routes/api/v2.rb
|
53
|
+
+++ b/config/routes/api/v2.rb
|
54
|
+
@@ -484,6 +484,7 @@ class ActionDispatch::Routing::Mapper
|
55
|
+
end
|
56
|
+
|
57
|
+
api_resources :sync_plans, :only => [:index, :show, :update, :destroy] do
|
58
|
+
+ api_resources :products, :only => [:index]
|
59
|
+
get :auto_complete_search, :on => :collection
|
60
|
+
put :sync
|
61
|
+
end
|