dependabot-python 0.79.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. checksums.yaml +7 -0
  2. data/helpers/build +17 -0
  3. data/helpers/lib/__init__.py +0 -0
  4. data/helpers/lib/hasher.py +23 -0
  5. data/helpers/lib/parser.py +130 -0
  6. data/helpers/requirements.txt +9 -0
  7. data/helpers/run.py +18 -0
  8. data/lib/dependabot/python.rb +11 -0
  9. data/lib/dependabot/python/file_fetcher.rb +307 -0
  10. data/lib/dependabot/python/file_parser.rb +221 -0
  11. data/lib/dependabot/python/file_parser/pipfile_files_parser.rb +150 -0
  12. data/lib/dependabot/python/file_parser/poetry_files_parser.rb +139 -0
  13. data/lib/dependabot/python/file_parser/setup_file_parser.rb +158 -0
  14. data/lib/dependabot/python/file_updater.rb +149 -0
  15. data/lib/dependabot/python/file_updater/pip_compile_file_updater.rb +361 -0
  16. data/lib/dependabot/python/file_updater/pipfile_file_updater.rb +391 -0
  17. data/lib/dependabot/python/file_updater/pipfile_preparer.rb +123 -0
  18. data/lib/dependabot/python/file_updater/poetry_file_updater.rb +282 -0
  19. data/lib/dependabot/python/file_updater/pyproject_preparer.rb +103 -0
  20. data/lib/dependabot/python/file_updater/requirement_file_updater.rb +160 -0
  21. data/lib/dependabot/python/file_updater/requirement_replacer.rb +93 -0
  22. data/lib/dependabot/python/file_updater/setup_file_sanitizer.rb +89 -0
  23. data/lib/dependabot/python/metadata_finder.rb +122 -0
  24. data/lib/dependabot/python/native_helpers.rb +17 -0
  25. data/lib/dependabot/python/python_versions.rb +25 -0
  26. data/lib/dependabot/python/requirement.rb +129 -0
  27. data/lib/dependabot/python/requirement_parser.rb +38 -0
  28. data/lib/dependabot/python/update_checker.rb +229 -0
  29. data/lib/dependabot/python/update_checker/latest_version_finder.rb +250 -0
  30. data/lib/dependabot/python/update_checker/pip_compile_version_resolver.rb +379 -0
  31. data/lib/dependabot/python/update_checker/pipfile_version_resolver.rb +558 -0
  32. data/lib/dependabot/python/update_checker/poetry_version_resolver.rb +298 -0
  33. data/lib/dependabot/python/update_checker/requirements_updater.rb +365 -0
  34. data/lib/dependabot/python/version.rb +87 -0
  35. metadata +203 -0
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dependabot
4
+ module Python
5
+ class RequirementParser
6
+ NAME = /[a-zA-Z0-9\-_\.]+/.freeze
7
+ EXTRA = /[a-zA-Z0-9\-_\.]+/.freeze
8
+ COMPARISON = /===|==|>=|<=|<|>|~=|!=/.freeze
9
+ VERSION = /[0-9]+[a-zA-Z0-9\-_\.*]*(\+[0-9a-zA-Z]+(\.[0-9a-zA-Z]+)*)?/.
10
+ freeze
11
+ REQUIREMENT =
12
+ /(?<comparison>#{COMPARISON})\s*\\?\s*(?<version>#{VERSION})/.freeze
13
+ HASH = /--hash=(?<algorithm>.*?):(?<hash>.*?)(?=\s|$)/.freeze
14
+ REQUIREMENTS = /#{REQUIREMENT}(\s*,\s*\\?\s*#{REQUIREMENT})*/.freeze
15
+ HASHES = /#{HASH}(\s*\\?\s*#{HASH})*/.freeze
16
+
17
+ INSTALL_REQ_WITH_REQUIREMENT =
18
+ /\s*\\?\s*(?<name>#{NAME})
19
+ \s*\\?\s*(\[\s*(?<extras>#{EXTRA}(\s*,\s*#{EXTRA})*)\s*\])?
20
+ \s*\\?\s*(?<requirements>#{REQUIREMENTS})
21
+ \s*\\?\s*(?<hashes>#{HASHES})?
22
+ \s*#*\s*(?<comment>.+)?
23
+ /x.freeze
24
+
25
+ INSTALL_REQ_WITHOUT_REQUIREMENT =
26
+ /^\s*\\?\s*(?<name>#{NAME})
27
+ \s*\\?\s*(\[\s*(?<extras>#{EXTRA}(\s*,\s*#{EXTRA})*)\s*\])?
28
+ \s*\\?\s*(?<hashes>#{HASHES})?
29
+ \s*#*\s*(?<comment>.+)?$
30
+ /x.freeze
31
+
32
+ NAME_WITH_EXTRAS =
33
+ /\s*\\?\s*(?<name>#{NAME})
34
+ (\s*\\?\s*\[\s*(?<extras>#{EXTRA}(\s*,\s*#{EXTRA})*)\s*\])?
35
+ /x.freeze
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,229 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "excon"
4
+ require "toml-rb"
5
+
6
+ require "dependabot/update_checkers"
7
+ require "dependabot/update_checkers/base"
8
+ require "dependabot/shared_helpers"
9
+ require "dependabot/python/requirement"
10
+ require "dependabot/python/requirement_parser"
11
+
12
+ module Dependabot
13
+ module Python
14
+ class UpdateChecker < Dependabot::UpdateCheckers::Base
15
+ require_relative "update_checker/poetry_version_resolver"
16
+ require_relative "update_checker/pipfile_version_resolver"
17
+ require_relative "update_checker/pip_compile_version_resolver"
18
+ require_relative "update_checker/requirements_updater"
19
+ require_relative "update_checker/latest_version_finder"
20
+
21
+ MAIN_PYPI_INDEXES = %w(
22
+ https://pypi.python.org/simple/
23
+ https://pypi.org/simple/
24
+ ).freeze
25
+
26
+ def latest_version
27
+ @latest_version ||= fetch_latest_version
28
+ end
29
+
30
+ def latest_resolvable_version
31
+ @latest_resolvable_version ||=
32
+ case resolver_type
33
+ when :pipfile
34
+ PipfileVersionResolver.new(
35
+ resolver_args.merge(unlock_requirement: true)
36
+ ).latest_resolvable_version
37
+ when :poetry
38
+ PoetryVersionResolver.new(
39
+ resolver_args.merge(unlock_requirement: true)
40
+ ).latest_resolvable_version
41
+ when :pip_compile
42
+ PipCompileVersionResolver.new(
43
+ resolver_args.merge(unlock_requirement: true)
44
+ ).latest_resolvable_version
45
+ when :requirements
46
+ # pip doesn't (yet) do any dependency resolution, so if we don't
47
+ # have a Pipfile or a pip-compile file, we just return the latest
48
+ # version.
49
+ latest_version
50
+ else raise "Unexpected resolver type #{resolver_type}"
51
+ end
52
+ end
53
+
54
+ def latest_resolvable_version_with_no_unlock
55
+ @latest_resolvable_version_with_no_unlock ||=
56
+ case resolver_type
57
+ when :pipfile
58
+ PipfileVersionResolver.new(
59
+ resolver_args.merge(unlock_requirement: false)
60
+ ).latest_resolvable_version
61
+ when :poetry
62
+ PoetryVersionResolver.new(
63
+ resolver_args.merge(unlock_requirement: false)
64
+ ).latest_resolvable_version
65
+ when :pip_compile
66
+ PipCompileVersionResolver.new(
67
+ resolver_args.merge(unlock_requirement: false)
68
+ ).latest_resolvable_version
69
+ when :requirements
70
+ latest_pip_version_with_no_unlock
71
+ else raise "Unexpected resolver type #{resolver_type}"
72
+ end
73
+ end
74
+
75
+ def updated_requirements
76
+ RequirementsUpdater.new(
77
+ requirements: dependency.requirements,
78
+ latest_version: latest_version&.to_s,
79
+ latest_resolvable_version: latest_resolvable_version&.to_s,
80
+ update_strategy: requirements_update_strategy,
81
+ has_lockfile: !(pipfile_lock || poetry_lock || pyproject_lock).nil?
82
+ ).updated_requirements
83
+ end
84
+
85
+ def requirements_update_strategy
86
+ # If passed in as an option (in the base class) honour that option
87
+ if @requirements_update_strategy
88
+ return @requirements_update_strategy.to_sym
89
+ end
90
+
91
+ # Otherwise, check if this is a poetry library or not
92
+ poetry_library? ? :widen_ranges : :bump_versions
93
+ end
94
+
95
+ private
96
+
97
+ def latest_version_resolvable_with_full_unlock?
98
+ # Full unlock checks aren't implemented for pip because they're not
99
+ # relevant (pip doesn't have a resolver). This method always returns
100
+ # false to ensure `updated_dependencies_after_full_unlock` is never
101
+ # called.
102
+ false
103
+ end
104
+
105
+ def updated_dependencies_after_full_unlock
106
+ raise NotImplementedError
107
+ end
108
+
109
+ # rubocop:disable Metrics/PerceivedComplexity
110
+ def resolver_type
111
+ reqs = dependency.requirements
112
+ req_files = reqs.map { |r| r.fetch(:file) }
113
+
114
+ # If there are no requirements then this is a sub-dependency. It
115
+ # must come from one of Pipenv, Poetry or pip-tools, and can't come
116
+ # from the first two unless they have a lockfile.
117
+ return subdependency_resolver if reqs.none?
118
+
119
+ # Otherwise, this is a top-level dependency, and we can figure out
120
+ # which resolver to use based on the filename of its requirements
121
+ return :pipfile if req_files.any? { |f| f == "Pipfile" }
122
+ return :poetry if req_files.any? { |f| f == "pyproject.toml" }
123
+ return :pip_compile if req_files.any? { |f| f.end_with?(".in") }
124
+
125
+ if dependency.version && !exact_requirement?(reqs)
126
+ subdependency_resolver
127
+ else
128
+ :requirements
129
+ end
130
+ end
131
+ # rubocop:enable Metrics/PerceivedComplexity
132
+
133
+ def subdependency_resolver
134
+ return :pipfile if pipfile_lock
135
+ return :poetry if poetry_lock || pyproject_lock
136
+ return :pip_compile if pip_compile_files.any?
137
+
138
+ raise "Claimed to be a sub-dependency, but no lockfile exists!"
139
+ end
140
+
141
+ def exact_requirement?(reqs)
142
+ reqs = reqs.map { |r| r.fetch(:requirement) }
143
+ reqs = reqs.compact
144
+ reqs = reqs.flat_map { |r| r.split(",").map(&:strip) }
145
+ reqs.any? { |r| Python::Requirement.new(r).exact? }
146
+ end
147
+
148
+ def resolver_args
149
+ {
150
+ dependency: dependency,
151
+ dependency_files: dependency_files,
152
+ credentials: credentials,
153
+ latest_allowable_version: latest_version
154
+ }
155
+ end
156
+
157
+ def fetch_latest_version
158
+ latest_version_finder.latest_version
159
+ end
160
+
161
+ def latest_pip_version_with_no_unlock
162
+ latest_version_finder.latest_version_with_no_unlock
163
+ end
164
+
165
+ def latest_version_finder
166
+ @latest_version_finder ||= LatestVersionFinder.new(
167
+ dependency: dependency,
168
+ dependency_files: dependency_files,
169
+ credentials: credentials,
170
+ ignored_versions: ignored_versions
171
+ )
172
+ end
173
+
174
+ def poetry_library?
175
+ return false unless pyproject
176
+
177
+ # Hit PyPi and check whether there are details for a library with a
178
+ # matching name and description
179
+ details = TomlRB.parse(pyproject.content).dig("tool", "poetry")
180
+ return false unless details
181
+
182
+ index_response = Excon.get(
183
+ "https://pypi.org/pypi/#{normalised_name(details['name'])}/json",
184
+ idempotent: true,
185
+ **SharedHelpers.excon_defaults
186
+ )
187
+
188
+ return false unless index_response.status == 200
189
+
190
+ pypi_info = JSON.parse(index_response.body)["info"] || {}
191
+ pypi_info["summary"] == details["description"]
192
+ rescue URI::InvalidURIError
193
+ false
194
+ end
195
+
196
+ # See https://www.python.org/dev/peps/pep-0503/#normalized-names
197
+ def normalised_name(name)
198
+ name.downcase.gsub(/[-_.]+/, "-")
199
+ end
200
+
201
+ def pipfile
202
+ dependency_files.find { |f| f.name == "Pipfile" }
203
+ end
204
+
205
+ def pipfile_lock
206
+ dependency_files.find { |f| f.name == "Pipfile.lock" }
207
+ end
208
+
209
+ def pyproject
210
+ dependency_files.find { |f| f.name == "pyproject.toml" }
211
+ end
212
+
213
+ def pyproject_lock
214
+ dependency_files.find { |f| f.name == "pyproject.lock" }
215
+ end
216
+
217
+ def poetry_lock
218
+ dependency_files.find { |f| f.name == "poetry.lock" }
219
+ end
220
+
221
+ def pip_compile_files
222
+ dependency_files.select { |f| f.name.end_with?(".in") }
223
+ end
224
+ end
225
+ end
226
+ end
227
+
228
+ Dependabot::UpdateCheckers.
229
+ register("pip", Dependabot::Python::UpdateChecker)
@@ -0,0 +1,250 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "excon"
4
+
5
+ require "dependabot/python/update_checker"
6
+ require "dependabot/shared_helpers"
7
+
8
+ module Dependabot
9
+ module Python
10
+ class UpdateChecker
11
+ class LatestVersionFinder
12
+ def initialize(dependency:, dependency_files:, credentials:,
13
+ ignored_versions:)
14
+ @dependency = dependency
15
+ @dependency_files = dependency_files
16
+ @credentials = credentials
17
+ @ignored_versions = ignored_versions
18
+ end
19
+
20
+ def latest_version
21
+ @latest_version ||= fetch_latest_version
22
+ end
23
+
24
+ def latest_version_with_no_unlock
25
+ @latest_version_with_no_unlock ||=
26
+ fetch_latest_version_with_no_unlock
27
+ end
28
+
29
+ private
30
+
31
+ attr_reader :dependency, :dependency_files, :credentials,
32
+ :ignored_versions
33
+
34
+ def fetch_latest_version
35
+ versions = available_versions
36
+ versions.reject! { |v| ignore_reqs.any? { |r| r.satisfied_by?(v) } }
37
+ versions.reject!(&:prerelease?) unless wants_prerelease?
38
+ versions.max
39
+ end
40
+
41
+ def fetch_latest_version_with_no_unlock
42
+ versions = available_versions
43
+ reqs = dependency.requirements.map do |r|
44
+ reqs = (r.fetch(:requirement) || "").split(",").map(&:strip)
45
+ requirement_class.new(reqs)
46
+ end
47
+ versions.reject!(&:prerelease?) unless wants_prerelease?
48
+ versions.sort.reverse.
49
+ reject { |v| ignore_reqs.any? { |r| r.satisfied_by?(v) } }.
50
+ find { |v| reqs.all? { |r| r.satisfied_by?(v) } }
51
+ end
52
+
53
+ def wants_prerelease?
54
+ if dependency.version
55
+ version = version_class.new(dependency.version.tr("+", "."))
56
+ return version.prerelease?
57
+ end
58
+
59
+ dependency.requirements.any? do |req|
60
+ reqs = (req.fetch(:requirement) || "").split(",").map(&:strip)
61
+ reqs.any? { |r| r.match?(/[A-Za-z]/) }
62
+ end
63
+ end
64
+
65
+ # See https://www.python.org/dev/peps/pep-0503/ for details of the
66
+ # Simple Repository API we use here.
67
+ def available_versions
68
+ index_urls.flat_map do |index_url|
69
+ sanitized_url = index_url.gsub(%r{(?<=//).*(?=@)}, "redacted")
70
+ index_response = registry_response_for_dependency(index_url)
71
+
72
+ if [401, 403].include?(index_response.status) &&
73
+ [401, 403].include?(registry_index_response(index_url).status)
74
+ raise PrivateSourceAuthenticationFailure, sanitized_url
75
+ end
76
+
77
+ index_response.body.
78
+ scan(%r{<a\s.*?>(.*?)</a>}m).flatten.
79
+ select { |n| n.match?(name_regex) }.
80
+ map do |filename|
81
+ version =
82
+ filename.
83
+ gsub(/#{name_regex}-/i, "").
84
+ split(/-|(\.tar\.)/).
85
+ first
86
+ next unless version_class.correct?(version)
87
+
88
+ version_class.new(version)
89
+ end.compact
90
+ rescue Excon::Error::Timeout, Excon::Error::Socket
91
+ next if MAIN_PYPI_INDEXES.include?(index_url)
92
+
93
+ raise PrivateSourceAuthenticationFailure, sanitized_url
94
+ end
95
+ end
96
+
97
+ def index_urls
98
+ main_index_url =
99
+ config_variable_index_urls[:main] ||
100
+ pipfile_index_urls[:main] ||
101
+ requirement_file_index_urls[:main] ||
102
+ pip_conf_index_urls[:main] ||
103
+ "https://pypi.python.org/simple/"
104
+
105
+ if main_index_url
106
+ main_index_url = main_index_url.strip.gsub(%r{/*$}, "") + "/"
107
+ end
108
+
109
+ extra_index_urls =
110
+ config_variable_index_urls[:extra] +
111
+ pipfile_index_urls[:extra] +
112
+ requirement_file_index_urls[:extra] +
113
+ pip_conf_index_urls[:extra]
114
+
115
+ extra_index_urls =
116
+ extra_index_urls.map { |url| url.strip.gsub(%r{/*$}, "") + "/" }
117
+
118
+ [main_index_url, *extra_index_urls].uniq
119
+ end
120
+
121
+ def registry_response_for_dependency(index_url)
122
+ Excon.get(
123
+ index_url + normalised_name + "/",
124
+ idempotent: true,
125
+ **SharedHelpers.excon_defaults
126
+ )
127
+ end
128
+
129
+ def registry_index_response(index_url)
130
+ Excon.get(
131
+ index_url,
132
+ idempotent: true,
133
+ **SharedHelpers.excon_defaults
134
+ )
135
+ end
136
+
137
+ def requirement_file_index_urls
138
+ urls = { main: nil, extra: [] }
139
+
140
+ requirements_files.each do |file|
141
+ if file.content.match?(/^--index-url\s(.+)/)
142
+ urls[:main] =
143
+ file.content.match(/^--index-url\s(.+)/).captures.first
144
+ end
145
+ urls[:extra] += file.content.scan(/^--extra-index-url\s(.+)/).
146
+ flatten
147
+ end
148
+
149
+ urls
150
+ end
151
+
152
+ def pip_conf_index_urls
153
+ urls = { main: nil, extra: [] }
154
+
155
+ return urls unless pip_conf
156
+
157
+ content = pip_conf.content
158
+
159
+ if content.match?(/^index-url\s*=/x)
160
+ urls[:main] = content.match(/^index-url\s*=\s*(.+)/).
161
+ captures.first
162
+ end
163
+ urls[:extra] += content.scan(/^extra-index-url\s*=(.+)/).flatten
164
+
165
+ urls
166
+ end
167
+
168
+ def pipfile_index_urls
169
+ urls = { main: nil, extra: [] }
170
+
171
+ return urls unless pipfile
172
+
173
+ pipfile_object = TomlRB.parse(pipfile.content)
174
+
175
+ urls[:main] = pipfile_object["source"]&.first&.fetch("url", nil)
176
+
177
+ pipfile_object["source"]&.each do |source|
178
+ urls[:extra] << source.fetch("url") if source["url"]
179
+ end
180
+ urls[:extra] = urls[:extra].uniq
181
+
182
+ urls
183
+ rescue TomlRB::ParseError
184
+ urls
185
+ end
186
+
187
+ def config_variable_index_urls
188
+ urls = { main: nil, extra: [] }
189
+
190
+ index_url_creds = credentials.
191
+ select { |cred| cred["type"] == "python_index" }
192
+ urls[:main] =
193
+ index_url_creds.
194
+ find { |cred| cred["replaces-base"] }&.
195
+ fetch("index-url")
196
+ urls[:extra] =
197
+ index_url_creds.
198
+ reject { |cred| cred["replaces-base"] }.
199
+ map { |cred| cred["index-url"] }
200
+
201
+ urls
202
+ end
203
+
204
+ def ignore_reqs
205
+ ignored_versions.map { |req| requirement_class.new(req.split(",")) }
206
+ end
207
+
208
+ # See https://www.python.org/dev/peps/pep-0503/#normalized-names
209
+ def normalised_name
210
+ dependency.name.downcase.gsub(/[-_.]+/, "-")
211
+ end
212
+
213
+ def name_regex
214
+ parts = dependency.name.split(/[\s_.-]/).map { |n| Regexp.quote(n) }
215
+ /#{parts.join("[\s_.-]")}/i
216
+ end
217
+
218
+ def pip_conf
219
+ dependency_files.find { |f| f.name == "pip.conf" }
220
+ end
221
+
222
+ def pipfile
223
+ dependency_files.find { |f| f.name == "Pipfile" }
224
+ end
225
+
226
+ def pyproject
227
+ dependency_files.find { |f| f.name == "pyproject.toml" }
228
+ end
229
+
230
+ def requirements_files
231
+ dependency_files.select { |f| f.name.match?(/requirements/x) }
232
+ end
233
+
234
+ def pip_compile_files
235
+ dependency_files.select { |f| f.name.end_with?(".in") }
236
+ end
237
+
238
+ def version_class
239
+ Utils.version_class_for_package_manager(dependency.package_manager)
240
+ end
241
+
242
+ def requirement_class
243
+ Utils.requirement_class_for_package_manager(
244
+ dependency.package_manager
245
+ )
246
+ end
247
+ end
248
+ end
249
+ end
250
+ end