dependabot-python 0.398.0 → 0.400.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/helpers/requirements.txt +2 -2
- data/lib/dependabot/python/dependency_grapher/requirements_layers.rb +3 -1
- data/lib/dependabot/python/file_parser/pipfile_document.rb +105 -0
- data/lib/dependabot/python/file_parser/pipfile_files_parser.rb +27 -87
- data/lib/dependabot/python/file_parser/pipfile_lock_document.rb +67 -0
- data/lib/dependabot/python/file_parser/pyproject_document.rb +28 -0
- data/lib/dependabot/python/file_parser/setup_file_parser.rb +71 -72
- data/lib/dependabot/python/file_parser.rb +2 -0
- data/lib/dependabot/python/file_updater/pip_compile_file_updater.rb +10 -0
- data/lib/dependabot/python/package/distribution.rb +180 -0
- data/lib/dependabot/python/package/package_details_fetcher.rb +48 -102
- data/lib/dependabot/python/package/pypi_json_parser.rb +48 -0
- data/lib/dependabot/python/package/simple_api_parser.rb +25 -34
- data/lib/dependabot/python/update_checker/pip_version_resolver.rb +16 -42
- metadata +8 -4
|
@@ -363,9 +363,19 @@ module Dependabot
|
|
|
363
363
|
content = replace_header_with_original(updated_content, T.must(file.content))
|
|
364
364
|
content = remove_new_warnings(content, T.must(file.content))
|
|
365
365
|
content = update_hashes_if_required(content, T.must(file.content))
|
|
366
|
+
content = scrub_index_url_credentials(content)
|
|
366
367
|
replace_absolute_file_paths(content, T.must(file.content))
|
|
367
368
|
end
|
|
368
369
|
|
|
370
|
+
# Index URLs are passed to pip-compile with credentials embedded so the
|
|
371
|
+
# private index can be fetched, and pip-tools emits those URLs verbatim
|
|
372
|
+
# into the compiled file when index-url emission is active. Strip any
|
|
373
|
+
# userinfo so tokens are never written into the regenerated file.
|
|
374
|
+
sig { params(content: String).returns(String) }
|
|
375
|
+
def scrub_index_url_credentials(content)
|
|
376
|
+
content.gsub(%r{^(--(?:extra-)?index-url[= ]https?://)[^/\s]+@}, "\\1")
|
|
377
|
+
end
|
|
378
|
+
|
|
369
379
|
sig { params(updated_content: String, original_content: String).returns(String) }
|
|
370
380
|
def replace_header_with_original(updated_content, original_content)
|
|
371
381
|
original_header_lines =
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
# typed: strong
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "json"
|
|
5
|
+
require "time"
|
|
6
|
+
require "uri"
|
|
7
|
+
require "cgi/escape"
|
|
8
|
+
require "nokogiri"
|
|
9
|
+
require "sorbet-runtime"
|
|
10
|
+
require "dependabot/errors"
|
|
11
|
+
|
|
12
|
+
module Dependabot
|
|
13
|
+
module Python
|
|
14
|
+
module Package
|
|
15
|
+
class Distribution < T::ImmutableStruct
|
|
16
|
+
extend T::Sig
|
|
17
|
+
|
|
18
|
+
ObjectHash = T.type_alias { T::Hash[String, Object] }
|
|
19
|
+
|
|
20
|
+
const :version_string, String
|
|
21
|
+
const :released_at, T.nilable(Time), default: nil
|
|
22
|
+
const :requires_python, T.nilable(String), default: nil
|
|
23
|
+
const :python_version, T.nilable(String), default: nil
|
|
24
|
+
const :yanked, T::Boolean, default: false
|
|
25
|
+
const :yanked_reason, T.nilable(String), default: nil
|
|
26
|
+
const :downloads, Integer, default: -1
|
|
27
|
+
const :url, T.nilable(String), default: nil
|
|
28
|
+
const :package_type, T.nilable(String), default: nil
|
|
29
|
+
|
|
30
|
+
sig { params(value: Object, version_string: String, context: String).returns(Distribution) }
|
|
31
|
+
def self.from_pypi(value, version_string:, context:)
|
|
32
|
+
fields = object(value, context)
|
|
33
|
+
new(
|
|
34
|
+
version_string: version_string,
|
|
35
|
+
released_at: timestamp(fields["upload_time"], "#{context}.upload_time"),
|
|
36
|
+
requires_python: optional_string(fields["requires_python"], "#{context}.requires_python"),
|
|
37
|
+
python_version: optional_string(fields["python_version"], "#{context}.python_version"),
|
|
38
|
+
yanked: boolean(fields["yanked"], "#{context}.yanked"),
|
|
39
|
+
yanked_reason: optional_string(fields["yanked_reason"], "#{context}.yanked_reason"),
|
|
40
|
+
downloads: downloads(fields["downloads"], "#{context}.downloads"),
|
|
41
|
+
url: optional_string(fields["url"], "#{context}.url"),
|
|
42
|
+
package_type: optional_string(fields["packagetype"], "#{context}.packagetype")
|
|
43
|
+
)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
sig do
|
|
47
|
+
params(value: Object, version_string: String, context: String, project_url: String).returns(Distribution)
|
|
48
|
+
end
|
|
49
|
+
def self.from_simple(value, version_string:, context:, project_url:)
|
|
50
|
+
fields = object(value, context)
|
|
51
|
+
withdrawn = fields["yanked"]
|
|
52
|
+
yanked = withdrawn.is_a?(String) || boolean(withdrawn, "#{context}.yanked")
|
|
53
|
+
new(
|
|
54
|
+
version_string: version_string,
|
|
55
|
+
released_at: timestamp(fields["upload-time"], "#{context}.upload-time"),
|
|
56
|
+
requires_python: optional_string(fields["requires-python"], "#{context}.requires-python"),
|
|
57
|
+
yanked: yanked,
|
|
58
|
+
yanked_reason: withdrawn.is_a?(String) ? withdrawn : nil,
|
|
59
|
+
url: resolve_url(optional_string(fields["url"], "#{context}.url"), project_url, "#{context}.url")
|
|
60
|
+
)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
sig do
|
|
64
|
+
params(
|
|
65
|
+
link: Nokogiri::XML::Node, version_string: String, context: String, project_url: String
|
|
66
|
+
).returns(Distribution)
|
|
67
|
+
end
|
|
68
|
+
def self.from_html(link, version_string:, context:, project_url:)
|
|
69
|
+
requirement = optional_string(T.cast(link["data-requires-python"], Object), "#{context}.data-requires-python")
|
|
70
|
+
reason = optional_string(T.cast(link["data-yanked"], Object), "#{context}.data-yanked")
|
|
71
|
+
withdrawn = !T.cast(link.attribute("data-yanked"), Object).nil?
|
|
72
|
+
new(
|
|
73
|
+
version_string: version_string,
|
|
74
|
+
requires_python: requirement ? CGI.unescapeHTML(requirement) : nil,
|
|
75
|
+
yanked: withdrawn,
|
|
76
|
+
yanked_reason: reason,
|
|
77
|
+
url: resolve_url(
|
|
78
|
+
optional_string(T.cast(link["href"], Object), "#{context}.href"),
|
|
79
|
+
project_url,
|
|
80
|
+
"#{context}.href"
|
|
81
|
+
)
|
|
82
|
+
)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
sig { params(format: String, source_url: String).returns(String) }
|
|
86
|
+
def self.context(format, source_url)
|
|
87
|
+
uri = URI.parse(source_url)
|
|
88
|
+
uri.userinfo = nil
|
|
89
|
+
uri.query = nil
|
|
90
|
+
uri.fragment = nil
|
|
91
|
+
"#{format} registry #{String(uri)}"
|
|
92
|
+
rescue URI::InvalidURIError
|
|
93
|
+
invalid(format, "registry URL is invalid")
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
sig { params(content: String, context: String).returns(Object) }
|
|
97
|
+
def self.parse_json(content, context)
|
|
98
|
+
T.cast(JSON.parse(content), Object)
|
|
99
|
+
rescue JSON::ParserError
|
|
100
|
+
invalid(context, "must contain valid JSON")
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
sig { params(value: Object, context: String).returns(ObjectHash) }
|
|
104
|
+
def self.object(value, context)
|
|
105
|
+
invalid(context, "must be an object") unless value.is_a?(Hash)
|
|
106
|
+
|
|
107
|
+
value.to_h do |raw_key, raw_value|
|
|
108
|
+
key = T.cast(raw_key, Object)
|
|
109
|
+
invalid(context, "keys must be strings") unless key.is_a?(String)
|
|
110
|
+
|
|
111
|
+
[key, T.cast(raw_value, Object)]
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
sig { params(value: Object, context: String).returns(T::Array[Object]) }
|
|
116
|
+
def self.array(value, context)
|
|
117
|
+
invalid(context, "must be an array") unless value.is_a?(Array)
|
|
118
|
+
|
|
119
|
+
value.map { |entry| T.cast(entry, Object) }
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
sig { params(value: Object, context: String).returns(T.nilable(String)) }
|
|
123
|
+
def self.optional_string(value, context)
|
|
124
|
+
return if value.nil?
|
|
125
|
+
return value if value.is_a?(String)
|
|
126
|
+
|
|
127
|
+
invalid(context, "must be a string or nil")
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
sig { params(context: String, message: String).returns(T.noreturn) }
|
|
131
|
+
def self.invalid(context, message)
|
|
132
|
+
raise DependencyFileNotResolvable.new("#{context} #{message}"), cause: nil
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
sig { params(value: Object, context: String).returns(T.nilable(Time)) }
|
|
136
|
+
def self.timestamp(value, context)
|
|
137
|
+
text = optional_string(value, context)
|
|
138
|
+
return if text.nil?
|
|
139
|
+
|
|
140
|
+
Time.parse(text)
|
|
141
|
+
rescue ArgumentError
|
|
142
|
+
invalid(context, "must be a valid timestamp")
|
|
143
|
+
end
|
|
144
|
+
private_class_method :timestamp
|
|
145
|
+
|
|
146
|
+
sig { params(value: Object, context: String).returns(T::Boolean) }
|
|
147
|
+
def self.boolean(value, context)
|
|
148
|
+
case value
|
|
149
|
+
when nil, false then false
|
|
150
|
+
when true then true
|
|
151
|
+
else invalid(context, "must be a boolean or nil")
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
private_class_method :boolean
|
|
155
|
+
|
|
156
|
+
sig { params(value: Object, context: String).returns(Integer) }
|
|
157
|
+
def self.downloads(value, context)
|
|
158
|
+
return -1 if value.nil?
|
|
159
|
+
return value if value.is_a?(Integer)
|
|
160
|
+
|
|
161
|
+
invalid(context, "must be an integer or nil")
|
|
162
|
+
end
|
|
163
|
+
private_class_method :downloads
|
|
164
|
+
|
|
165
|
+
sig { params(url: T.nilable(String), project_url: String, context: String).returns(T.nilable(String)) }
|
|
166
|
+
def self.resolve_url(url, project_url, context)
|
|
167
|
+
return if url.nil?
|
|
168
|
+
|
|
169
|
+
resolved = URI.join(project_url, url)
|
|
170
|
+
resolved.user = nil
|
|
171
|
+
resolved.password = nil
|
|
172
|
+
String(resolved)
|
|
173
|
+
rescue URI::InvalidURIError
|
|
174
|
+
invalid(context, "must be a valid URL")
|
|
175
|
+
end
|
|
176
|
+
private_class_method :resolve_url
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
end
|
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
# typed: strict
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
-
require "json"
|
|
5
|
-
require "time"
|
|
6
4
|
require "cgi/escape"
|
|
7
5
|
require "excon"
|
|
8
6
|
require "nokogiri"
|
|
@@ -13,6 +11,8 @@ require "dependabot/package/package_release"
|
|
|
13
11
|
require "dependabot/package/package_details"
|
|
14
12
|
require "dependabot/python/package/package_registry_finder"
|
|
15
13
|
require "dependabot/python/package/simple_api_parser"
|
|
14
|
+
require "dependabot/python/package/distribution"
|
|
15
|
+
require "dependabot/python/package/pypi_json_parser"
|
|
16
16
|
|
|
17
17
|
# Stores metadata for a package, including all its available versions
|
|
18
18
|
module Dependabot
|
|
@@ -111,7 +111,7 @@ module Dependabot
|
|
|
111
111
|
"Fetching release information from simple registry at #{sanitized_url(index_url)} for #{dependency.name}"
|
|
112
112
|
)
|
|
113
113
|
response = registry_response_for_dependency(index_url, accept: simple_api_accept)
|
|
114
|
-
project_url =
|
|
114
|
+
project_url = project_url_for(response, index_url)
|
|
115
115
|
check_authentication_response(response, index_url)
|
|
116
116
|
|
|
117
117
|
version_releases = if simple_json_response?(response)
|
|
@@ -120,14 +120,11 @@ module Dependabot
|
|
|
120
120
|
project_url: project_url
|
|
121
121
|
).parse(response.body)
|
|
122
122
|
else
|
|
123
|
-
|
|
123
|
+
distributions_from_html(response.body, project_url: project_url)
|
|
124
124
|
end
|
|
125
125
|
|
|
126
126
|
format_version_releases(version_releases, preserve_distributions: simple_json_response?(response))
|
|
127
127
|
.sort_by(&:version).reverse
|
|
128
|
-
rescue JSON::ParserError
|
|
129
|
-
Dependabot.logger.warn("JSON parsing error for #{sanitized_url(index_url)}.")
|
|
130
|
-
[]
|
|
131
128
|
end
|
|
132
129
|
|
|
133
130
|
# Example JSON Response Format:
|
|
@@ -190,16 +187,13 @@ module Dependabot
|
|
|
190
187
|
return nil unless response.status == 200
|
|
191
188
|
|
|
192
189
|
begin
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
version_releases = data["releases"]
|
|
190
|
+
version_releases = PypiJsonParser.new(source_url: json_url).parse(response.body)
|
|
196
191
|
|
|
197
192
|
releases = format_version_releases(version_releases)
|
|
198
193
|
|
|
199
194
|
releases.sort_by(&:version).reverse
|
|
200
|
-
rescue
|
|
201
|
-
|
|
202
|
-
nil
|
|
195
|
+
rescue Dependabot::DependencyFileNotResolvable
|
|
196
|
+
raise
|
|
203
197
|
rescue StandardError => e
|
|
204
198
|
Dependabot.logger.warn("Unexpected error while fetching JSON data: #{e.message}.")
|
|
205
199
|
nil
|
|
@@ -235,7 +229,9 @@ module Dependabot
|
|
|
235
229
|
index_response = registry_response_for_dependency(index_url, accept: APPLICATION_TEXT)
|
|
236
230
|
check_authentication_response(index_response, index_url)
|
|
237
231
|
|
|
238
|
-
version_releases =
|
|
232
|
+
version_releases = distributions_from_html(
|
|
233
|
+
index_response.body, project_url: project_url_for(index_response, index_url)
|
|
234
|
+
)
|
|
239
235
|
releases = format_version_releases(version_releases)
|
|
240
236
|
|
|
241
237
|
releases.sort_by(&:version).reverse
|
|
@@ -262,102 +258,64 @@ module Dependabot
|
|
|
262
258
|
raise PrivateSourceAuthenticationFailure, sanitized_url(index_url)
|
|
263
259
|
end
|
|
264
260
|
|
|
265
|
-
sig
|
|
266
|
-
|
|
267
|
-
|
|
261
|
+
sig { params(response: Excon::Response, index_url: String).returns(String) }
|
|
262
|
+
def project_url_for(response, index_url)
|
|
263
|
+
response.path ? Excon::Utils.request_uri(response.data) : index_url + normalised_name + "/"
|
|
268
264
|
end
|
|
269
|
-
def extract_release_details_json_from_html(html_body)
|
|
270
|
-
doc = Nokogiri::HTML(html_body)
|
|
271
|
-
|
|
272
|
-
releases = {}
|
|
273
265
|
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
266
|
+
sig { params(html_body: String, project_url: String).returns(T::Hash[String, T::Array[Distribution]]) }
|
|
267
|
+
def distributions_from_html(html_body, project_url:)
|
|
268
|
+
doc = Nokogiri::HTML(html_body)
|
|
269
|
+
context = Distribution.context("HTML index", project_url)
|
|
270
|
+
releases = T.let({}, T::Hash[String, T::Array[Distribution]])
|
|
271
|
+
|
|
272
|
+
doc.css("a").each_with_index do |link, index|
|
|
273
|
+
filename = link.content
|
|
274
|
+
url = link["href"]
|
|
275
|
+
next unless filename.match?(name_regex) || url&.match?(name_regex)
|
|
276
|
+
|
|
277
|
+
version = get_version_from_filename(filename)
|
|
278
|
+
next unless version && version_class.correct?(version)
|
|
279
|
+
|
|
280
|
+
distribution = Distribution.from_html(
|
|
281
|
+
link, version_string: version, context: "#{context}.links[#{index}]", project_url: project_url
|
|
282
|
+
)
|
|
283
|
+
releases[version] ||= []
|
|
284
|
+
T.must(releases[version]) << distribution
|
|
280
285
|
end
|
|
281
286
|
|
|
282
287
|
releases
|
|
283
288
|
end
|
|
284
289
|
|
|
285
|
-
# rubocop:disable Metrics/PerceivedComplexity
|
|
286
|
-
sig do
|
|
287
|
-
params(link: T.nilable(String))
|
|
288
|
-
.returns(T.nilable(T::Hash[String, T.untyped]))
|
|
289
|
-
end
|
|
290
|
-
def version_details_from_link(link)
|
|
291
|
-
return unless link
|
|
292
|
-
|
|
293
|
-
doc = Nokogiri::XML(link)
|
|
294
|
-
filename = doc.at_css("a")&.content
|
|
295
|
-
url = doc.at_css("a")&.attributes&.fetch("href", nil)&.value
|
|
296
|
-
|
|
297
|
-
return unless filename&.match?(name_regex) || url&.match?(name_regex)
|
|
298
|
-
|
|
299
|
-
version = get_version_from_filename(filename)
|
|
300
|
-
return unless version_class.correct?(version)
|
|
301
|
-
|
|
302
|
-
{
|
|
303
|
-
"version" => version,
|
|
304
|
-
"requires_python" => requires_python_from_link(link),
|
|
305
|
-
"yanked" => link.include?("data-yanked"),
|
|
306
|
-
"url" => link
|
|
307
|
-
}
|
|
308
|
-
end
|
|
309
|
-
# rubocop:enable Metrics/PerceivedComplexity
|
|
310
|
-
|
|
311
290
|
sig do
|
|
312
291
|
params(
|
|
313
|
-
|
|
292
|
+
distributions: T::Hash[String, T::Array[Distribution]],
|
|
314
293
|
preserve_distributions: T::Boolean
|
|
315
294
|
)
|
|
316
295
|
.returns(T::Array[Dependabot::Package::PackageRelease])
|
|
317
296
|
end
|
|
318
|
-
def format_version_releases(
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
release_data_array = [release_data_array.last].compact unless preserve_distributions
|
|
323
|
-
versions.concat(release_data_array.filter_map { |data| format_version_release(version, data) })
|
|
297
|
+
def format_version_releases(distributions, preserve_distributions: false)
|
|
298
|
+
distributions.each_value.flat_map do |files|
|
|
299
|
+
selected = preserve_distributions ? files : files.last(1)
|
|
300
|
+
selected.map { |distribution| format_version_release(distribution) }
|
|
324
301
|
end
|
|
325
302
|
end
|
|
326
303
|
|
|
327
|
-
sig
|
|
328
|
-
|
|
329
|
-
version: String,
|
|
330
|
-
release_data: T::Hash[String, T.untyped]
|
|
331
|
-
)
|
|
332
|
-
.returns(T.nilable(Dependabot::Package::PackageRelease))
|
|
333
|
-
end
|
|
334
|
-
def format_version_release(version, release_data)
|
|
335
|
-
# Skip versions that don't conform to PEP 440
|
|
336
|
-
unless Dependabot::Python::Version.correct?(version)
|
|
337
|
-
Dependabot.logger.warn("Skipping invalid version #{version}: does not match PEP 440")
|
|
338
|
-
return nil
|
|
339
|
-
end
|
|
340
|
-
|
|
341
|
-
upload_time = release_data["upload_time"]
|
|
342
|
-
released_at = Time.parse(upload_time) if upload_time
|
|
343
|
-
yanked = release_data["yanked"] || false
|
|
344
|
-
yanked_reason = release_data["yanked_reason"]
|
|
345
|
-
downloads = release_data["downloads"] || -1
|
|
346
|
-
url = release_data["url"]
|
|
347
|
-
package_type = release_data["packagetype"]
|
|
304
|
+
sig { params(distribution: Distribution).returns(Dependabot::Package::PackageRelease) }
|
|
305
|
+
def format_version_release(distribution)
|
|
348
306
|
language = package_language(
|
|
349
|
-
python_version:
|
|
350
|
-
requires_python:
|
|
307
|
+
python_version: distribution.python_version,
|
|
308
|
+
requires_python: distribution.requires_python
|
|
351
309
|
)
|
|
352
310
|
|
|
353
311
|
Dependabot::Package::PackageRelease.new(
|
|
354
|
-
version: Dependabot::Python::Version.new(
|
|
355
|
-
released_at: released_at,
|
|
356
|
-
yanked: yanked,
|
|
357
|
-
yanked_reason: yanked_reason,
|
|
358
|
-
downloads: downloads,
|
|
359
|
-
url: url,
|
|
360
|
-
package_type: package_type,
|
|
312
|
+
version: Dependabot::Python::Version.new(distribution.version_string),
|
|
313
|
+
released_at: distribution.released_at,
|
|
314
|
+
yanked: distribution.yanked,
|
|
315
|
+
yanked_reason: distribution.yanked_reason,
|
|
316
|
+
downloads: distribution.downloads,
|
|
317
|
+
url: distribution.url,
|
|
318
|
+
package_type: distribution.package_type,
|
|
361
319
|
language: language
|
|
362
320
|
)
|
|
363
321
|
end
|
|
@@ -472,18 +430,6 @@ module Dependabot
|
|
|
472
430
|
nil
|
|
473
431
|
end
|
|
474
432
|
|
|
475
|
-
sig { params(link: String).returns(T.nilable(String)) }
|
|
476
|
-
def requires_python_from_link(link)
|
|
477
|
-
raw_value = Nokogiri::XML(link)
|
|
478
|
-
.at_css("a")
|
|
479
|
-
&.attribute("data-requires-python")
|
|
480
|
-
&.content
|
|
481
|
-
|
|
482
|
-
return nil unless raw_value
|
|
483
|
-
|
|
484
|
-
CGI.unescapeHTML(raw_value) # Decodes HTML entities like >=3 → >=3
|
|
485
|
-
end
|
|
486
|
-
|
|
487
433
|
sig { returns(T.class_of(Dependabot::Version)) }
|
|
488
434
|
def version_class
|
|
489
435
|
dependency.version_class
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# typed: strong
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
require "dependabot/python/version"
|
|
6
|
+
require "dependabot/python/package/distribution"
|
|
7
|
+
|
|
8
|
+
module Dependabot
|
|
9
|
+
module Python
|
|
10
|
+
module Package
|
|
11
|
+
class PypiJsonParser
|
|
12
|
+
extend T::Sig
|
|
13
|
+
|
|
14
|
+
sig { params(source_url: String).void }
|
|
15
|
+
def initialize(source_url:)
|
|
16
|
+
@context = T.let(Distribution.context("PyPI JSON", source_url), String)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
sig { params(content: String).returns(T::Hash[String, T::Array[Distribution]]) }
|
|
20
|
+
def parse(content)
|
|
21
|
+
fields = Distribution.object(Distribution.parse_json(content, @context), @context)
|
|
22
|
+
raw_releases = fields["releases"]
|
|
23
|
+
return {} if raw_releases.nil?
|
|
24
|
+
|
|
25
|
+
releases = Distribution.object(raw_releases, "#{@context}.releases")
|
|
26
|
+
parsed = T.let({}, T::Hash[String, T::Array[Distribution]])
|
|
27
|
+
releases.each_with_index do |(version, value), release_index|
|
|
28
|
+
location = "#{@context}.releases[#{release_index}]"
|
|
29
|
+
entries = Distribution.array(value, location)
|
|
30
|
+
next if entries.empty?
|
|
31
|
+
|
|
32
|
+
unless Python::Version.correct?(version)
|
|
33
|
+
Dependabot.logger.warn("Skipping invalid version #{version}: does not match PEP 440")
|
|
34
|
+
next
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
parsed[version] = entries.each_with_index.map do |entry, index|
|
|
38
|
+
Distribution.from_pypi(
|
|
39
|
+
entry, version_string: version, context: "#{location}[#{index}]"
|
|
40
|
+
)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
parsed
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -1,11 +1,10 @@
|
|
|
1
|
-
# typed:
|
|
1
|
+
# typed: strong
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
-
require "json"
|
|
5
|
-
require "uri"
|
|
6
4
|
require "sorbet-runtime"
|
|
7
5
|
require "dependabot/errors"
|
|
8
6
|
require "dependabot/python/name_normaliser"
|
|
7
|
+
require "dependabot/python/package/distribution"
|
|
9
8
|
|
|
10
9
|
module Dependabot
|
|
11
10
|
module Python
|
|
@@ -13,8 +12,6 @@ module Dependabot
|
|
|
13
12
|
class SimpleApiParser
|
|
14
13
|
extend T::Sig
|
|
15
14
|
|
|
16
|
-
ReleaseDetail = T.type_alias { T::Hash[String, T.nilable(T.any(String, T::Boolean))] }
|
|
17
|
-
|
|
18
15
|
sig { params(dependency: Dependabot::Dependency, project_url: String).void }
|
|
19
16
|
def initialize(dependency:, project_url:)
|
|
20
17
|
@dependency = dependency
|
|
@@ -23,31 +20,32 @@ module Dependabot
|
|
|
23
20
|
|
|
24
21
|
sig do
|
|
25
22
|
params(json_body: String)
|
|
26
|
-
.returns(T::Hash[String, T::Array[
|
|
23
|
+
.returns(T::Hash[String, T::Array[Distribution]])
|
|
27
24
|
end
|
|
28
25
|
def parse(json_body)
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
26
|
+
context = Distribution.context("Simple API JSON", project_url)
|
|
27
|
+
data = Distribution.object(Distribution.parse_json(json_body, context), context)
|
|
28
|
+
meta = data["meta"]
|
|
29
|
+
metadata = meta.nil? ? {} : Distribution.object(meta, "#{context}.meta")
|
|
30
|
+
validate_api_version!(metadata.fetch("api-version", "1.0"), context)
|
|
31
|
+
|
|
32
|
+
releases = T.let({}, T::Hash[String, T::Array[Distribution]])
|
|
33
|
+
Distribution.array(data.fetch("files", []), "#{context}.files").each_with_index do |entry, index|
|
|
34
|
+
location = "#{context}.files[#{index}]"
|
|
35
|
+
file = Distribution.object(entry, location)
|
|
36
|
+
filename = Distribution.optional_string(file["filename"], "#{location}.filename")
|
|
34
37
|
next unless filename&.match?(name_regex)
|
|
35
38
|
|
|
36
39
|
version = version_from_filename(filename)
|
|
37
|
-
next unless dependency.version_class.correct?(version)
|
|
40
|
+
next unless version && dependency.version_class.correct?(version)
|
|
38
41
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
"requires_python" => file["requires-python"],
|
|
43
|
-
"yanked" => yanked != false,
|
|
44
|
-
"yanked_reason" => yanked.is_a?(String) ? yanked : nil,
|
|
45
|
-
"upload_time" => file["upload-time"],
|
|
46
|
-
"url" => resolve_url(file["url"])
|
|
47
|
-
}
|
|
42
|
+
distribution = Distribution.from_simple(
|
|
43
|
+
file, version_string: version, context: location, project_url: project_url
|
|
44
|
+
)
|
|
48
45
|
releases[version] ||= []
|
|
49
|
-
releases[version] <<
|
|
46
|
+
T.must(releases[version]) << distribution
|
|
50
47
|
end
|
|
48
|
+
releases
|
|
51
49
|
end
|
|
52
50
|
|
|
53
51
|
private
|
|
@@ -58,18 +56,11 @@ module Dependabot
|
|
|
58
56
|
sig { returns(String) }
|
|
59
57
|
attr_reader :project_url
|
|
60
58
|
|
|
61
|
-
sig { params(
|
|
62
|
-
def
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
resolved_url.user = nil
|
|
67
|
-
resolved_url.password = nil
|
|
68
|
-
resolved_url.to_s
|
|
69
|
-
end
|
|
70
|
-
|
|
71
|
-
sig { params(api_version: String).void }
|
|
72
|
-
def validate_api_version!(api_version)
|
|
59
|
+
sig { params(api_version: Object, context: String).void }
|
|
60
|
+
def validate_api_version!(api_version, context)
|
|
61
|
+
unless api_version.is_a?(String) && api_version.match?(/\A[0-9]+\.[0-9]+\z/)
|
|
62
|
+
Distribution.invalid("#{context}.meta.api-version", "must be a Major.Minor string")
|
|
63
|
+
end
|
|
73
64
|
return unless api_version.split(".").first.to_i > 1
|
|
74
65
|
|
|
75
66
|
raise Dependabot::DependencyFileNotResolvable, "Unsupported PEP 691 API version: #{api_version}"
|
|
@@ -13,6 +13,7 @@ require "dependabot/python/package/package_registry_finder"
|
|
|
13
13
|
require "dependabot/python/update_checker"
|
|
14
14
|
require "dependabot/python/update_checker/latest_version_finder"
|
|
15
15
|
require "dependabot/python/file_parser/python_requirement_parser"
|
|
16
|
+
require "dependabot/python/file_parser/pyproject_document"
|
|
16
17
|
|
|
17
18
|
module Dependabot
|
|
18
19
|
module Python
|
|
@@ -62,7 +63,7 @@ module Dependabot
|
|
|
62
63
|
@constraints_files = T.let(nil, T.nilable(T::Array[String]))
|
|
63
64
|
@constraints_file_basenames = T.let(nil, T.nilable(T::Array[String]))
|
|
64
65
|
@requirement_file_directories = T.let(nil, T.nilable(T::Array[String]))
|
|
65
|
-
@
|
|
66
|
+
@pyproject_document_cache = T.let({}, T::Hash[String, FileParser::PyprojectDocument])
|
|
66
67
|
end
|
|
67
68
|
|
|
68
69
|
sig { returns(T.nilable(Dependabot::Version)) }
|
|
@@ -201,19 +202,8 @@ module Dependabot
|
|
|
201
202
|
|
|
202
203
|
sig { params(pyproject: Dependabot::DependencyFile).returns(T::Array[[String, String]]) }
|
|
203
204
|
def pinned_pyproject_dependencies_for(pyproject)
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
return [] unless project_obj.is_a?(Hash)
|
|
207
|
-
|
|
208
|
-
project_hash = project_obj
|
|
209
|
-
dependencies_obj = T.cast(project_hash["dependencies"], T.nilable(Object))
|
|
210
|
-
return [] unless dependencies_obj.is_a?(Array)
|
|
211
|
-
|
|
212
|
-
dependencies_obj.filter_map do |entry|
|
|
213
|
-
entry_obj = T.cast(entry, T.nilable(Object))
|
|
214
|
-
next unless entry_obj.is_a?(String)
|
|
215
|
-
|
|
216
|
-
requirement_string, marker = split_requirement_and_marker(entry_obj)
|
|
205
|
+
pyproject_document_for(pyproject).pip_dependencies.filter_map do |entry|
|
|
206
|
+
requirement_string, marker = split_requirement_and_marker(entry)
|
|
217
207
|
next unless marker_satisfied_for_python?(marker)
|
|
218
208
|
next if requirement_string.nil? || requirement_string.empty?
|
|
219
209
|
|
|
@@ -432,23 +422,8 @@ module Dependabot
|
|
|
432
422
|
|
|
433
423
|
sig { params(pyproject: Dependabot::DependencyFile).returns(T::Array[String]) }
|
|
434
424
|
def constraints_for_pyproject(pyproject)
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
return [] unless tool_obj.is_a?(Hash)
|
|
438
|
-
|
|
439
|
-
pip_obj = T.cast(tool_obj["pip"], T.nilable(Object))
|
|
440
|
-
return [] unless pip_obj.is_a?(Hash)
|
|
441
|
-
|
|
442
|
-
constraints_obj = T.cast(pip_obj["constraints"], T.nilable(Object))
|
|
443
|
-
case constraints_obj
|
|
444
|
-
when String
|
|
445
|
-
[resolve_constraint_path(path: constraints_obj, declaring_file: pyproject)]
|
|
446
|
-
when Array
|
|
447
|
-
constraints_obj.grep(String).map do |path|
|
|
448
|
-
resolve_constraint_path(path: path, declaring_file: pyproject)
|
|
449
|
-
end
|
|
450
|
-
else
|
|
451
|
-
[]
|
|
425
|
+
pyproject_document_for(pyproject).pip_constraint_paths.map do |path|
|
|
426
|
+
resolve_constraint_path(path: path, declaring_file: pyproject)
|
|
452
427
|
end
|
|
453
428
|
end
|
|
454
429
|
|
|
@@ -556,23 +531,22 @@ module Dependabot
|
|
|
556
531
|
marker_evaluator.marker_satisfied?(marker: marker, python_version: python_version)
|
|
557
532
|
end
|
|
558
533
|
|
|
559
|
-
sig { params(pyproject: Dependabot::DependencyFile).returns(
|
|
560
|
-
def
|
|
534
|
+
sig { params(pyproject: Dependabot::DependencyFile).returns(FileParser::PyprojectDocument) }
|
|
535
|
+
def pyproject_document_for(pyproject)
|
|
561
536
|
cache_key = pyproject.name
|
|
562
|
-
return T.must(@
|
|
537
|
+
return T.must(@pyproject_document_cache[cache_key]) if @pyproject_document_cache.key?(cache_key)
|
|
563
538
|
|
|
564
|
-
content =
|
|
565
|
-
|
|
566
|
-
|
|
539
|
+
content = pyproject.content
|
|
540
|
+
document =
|
|
541
|
+
if content
|
|
542
|
+
FileParser::PyprojectDocument.from_content(content)
|
|
567
543
|
else
|
|
568
|
-
|
|
544
|
+
FileParser::PyprojectDocument.new({})
|
|
569
545
|
end
|
|
570
546
|
|
|
571
|
-
@
|
|
572
|
-
content
|
|
547
|
+
@pyproject_document_cache[cache_key] = document
|
|
573
548
|
rescue TomlRB::ParseError, TomlRB::ValueOverwriteError
|
|
574
|
-
@
|
|
575
|
-
T.must(@pyproject_content_cache[pyproject.name])
|
|
549
|
+
@pyproject_document_cache[pyproject.name] = FileParser::PyprojectDocument.new({})
|
|
576
550
|
end
|
|
577
551
|
end
|
|
578
552
|
end
|