dependabot-python 0.236.0 → 0.237.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/helpers/requirements.txt +2 -2
- data/lib/dependabot/python/file_fetcher.rb +7 -2
- data/lib/dependabot/python/file_parser/pipfile_files_parser.rb +2 -1
- data/lib/dependabot/python/file_parser/pyproject_files_parser.rb +15 -6
- data/lib/dependabot/python/file_parser/python_requirement_parser.rb +24 -0
- data/lib/dependabot/python/file_updater/pipfile_file_updater.rb +21 -78
- data/lib/dependabot/python/file_updater/pipfile_preparer.rb +1 -67
- data/lib/dependabot/python/file_updater/poetry_file_updater.rb +1 -1
- data/lib/dependabot/python/file_updater.rb +2 -1
- data/lib/dependabot/python/pipenv_runner.rb +82 -0
- data/lib/dependabot/python/update_checker/index_finder.rb +7 -2
- data/lib/dependabot/python/update_checker/latest_version_finder.rb +2 -1
- data/lib/dependabot/python/update_checker/pip_compile_version_resolver.rb +3 -2
- data/lib/dependabot/python/update_checker/pipenv_version_resolver.rb +27 -127
- data/lib/dependabot/python/update_checker/poetry_version_resolver.rb +3 -2
- data/lib/dependabot/python/update_checker.rb +5 -4
- data/lib/dependabot/python.rb +3 -0
- metadata +22 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1cd7a4517e826fb42d55e5f504f0162c5e0850b9d1dd01ff259451a40103cf8f
|
4
|
+
data.tar.gz: 9c064acaf52f7856f60881873d7d3127623d12935936c2b324d1beb2e1b6abc4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 175a621875538a2b4c7aab6b02a61cf89d8fa0847d4568466526d2403cfb67a2944322a39ecfef17834d08892e29099d3d54e00db124d6d3a8fb26d505616706
|
7
|
+
data.tar.gz: 0f6dc93dd2598d9f1ab8daf3e5c654c05ec3488c305ccec9a651de34680fefad8892d9d508c4752b74fa69767f4fc9514b9ca931b3f5a2ca87a2d773afedc5ec
|
data/helpers/requirements.txt
CHANGED
@@ -2,6 +2,7 @@
|
|
2
2
|
# frozen_string_literal: true
|
3
3
|
|
4
4
|
require "toml-rb"
|
5
|
+
require "sorbet-runtime"
|
5
6
|
|
6
7
|
require "dependabot/file_fetchers"
|
7
8
|
require "dependabot/file_fetchers/base"
|
@@ -15,6 +16,9 @@ require "dependabot/errors"
|
|
15
16
|
module Dependabot
|
16
17
|
module Python
|
17
18
|
class FileFetcher < Dependabot::FileFetchers::Base
|
19
|
+
extend T::Sig
|
20
|
+
extend T::Helpers
|
21
|
+
|
18
22
|
CHILD_REQUIREMENT_REGEX = /^-r\s?(?<path>.*\.(?:txt|in))/
|
19
23
|
CONSTRAINT_REGEX = /^-c\s?(?<path>.*\.(?:txt|in))/
|
20
24
|
DEPENDENCY_TYPES = %w(packages dev-packages).freeze
|
@@ -63,8 +67,7 @@ module Dependabot
|
|
63
67
|
}
|
64
68
|
end
|
65
69
|
|
66
|
-
|
67
|
-
|
70
|
+
sig { override.returns(T::Array[DependencyFile]) }
|
68
71
|
def fetch_files
|
69
72
|
fetched_files = []
|
70
73
|
|
@@ -84,6 +87,8 @@ module Dependabot
|
|
84
87
|
uniq_files(fetched_files)
|
85
88
|
end
|
86
89
|
|
90
|
+
private
|
91
|
+
|
87
92
|
def uniq_files(fetched_files)
|
88
93
|
uniq_files = fetched_files.reject(&:support_file?).uniq
|
89
94
|
uniq_files += fetched_files
|
@@ -137,12 +137,21 @@ module Dependabot
|
|
137
137
|
|
138
138
|
check_requirements(requirement)
|
139
139
|
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
140
|
+
if requirement.is_a?(String)
|
141
|
+
{
|
142
|
+
requirement: requirement,
|
143
|
+
file: pyproject.name,
|
144
|
+
source: nil,
|
145
|
+
groups: [type]
|
146
|
+
}
|
147
|
+
else
|
148
|
+
{
|
149
|
+
requirement: requirement["version"],
|
150
|
+
file: pyproject.name,
|
151
|
+
source: requirement.fetch("source", nil),
|
152
|
+
groups: [type]
|
153
|
+
}
|
154
|
+
end
|
146
155
|
end
|
147
156
|
end
|
148
157
|
|
@@ -6,6 +6,7 @@ require "open3"
|
|
6
6
|
require "dependabot/errors"
|
7
7
|
require "dependabot/shared_helpers"
|
8
8
|
require "dependabot/python/file_parser"
|
9
|
+
require "dependabot/python/pip_compile_file_matcher"
|
9
10
|
require "dependabot/python/requirement"
|
10
11
|
|
11
12
|
module Dependabot
|
@@ -22,6 +23,7 @@ module Dependabot
|
|
22
23
|
[
|
23
24
|
pipfile_python_requirement,
|
24
25
|
pyproject_python_requirement,
|
26
|
+
pip_compile_python_requirement,
|
25
27
|
python_version_file_version,
|
26
28
|
runtime_file_python_version,
|
27
29
|
setup_file_requirement
|
@@ -64,6 +66,20 @@ module Dependabot
|
|
64
66
|
poetry_object&.dig("dev-dependencies", "python")
|
65
67
|
end
|
66
68
|
|
69
|
+
def pip_compile_python_requirement
|
70
|
+
requirement_files.each do |file|
|
71
|
+
next unless pip_compile_file_matcher.lockfile_for_pip_compile_file?(file)
|
72
|
+
|
73
|
+
marker = /^# This file is autogenerated by pip-compile with [pP]ython (?<version>\d+.\d+)$/m
|
74
|
+
match = marker.match(file.content)
|
75
|
+
next unless match
|
76
|
+
|
77
|
+
return match[:version]
|
78
|
+
end
|
79
|
+
|
80
|
+
nil
|
81
|
+
end
|
82
|
+
|
67
83
|
def python_version_file_version
|
68
84
|
return unless python_version_file
|
69
85
|
|
@@ -106,6 +122,10 @@ module Dependabot
|
|
106
122
|
SharedHelpers.run_shell_command(command, env: env, stderr_to_stdout: true)
|
107
123
|
end
|
108
124
|
|
125
|
+
def pip_compile_file_matcher
|
126
|
+
@pip_compile_file_matcher ||= PipCompileFileMatcher.new(pip_compile_files)
|
127
|
+
end
|
128
|
+
|
109
129
|
def requirement_class
|
110
130
|
Dependabot::Python::Requirement
|
111
131
|
end
|
@@ -144,6 +164,10 @@ module Dependabot
|
|
144
164
|
def requirement_files
|
145
165
|
dependency_files.select { |f| f.name.end_with?(".txt") }
|
146
166
|
end
|
167
|
+
|
168
|
+
def pip_compile_files
|
169
|
+
dependency_files.select { |f| f.name.end_with?(".in") }
|
170
|
+
end
|
147
171
|
end
|
148
172
|
end
|
149
173
|
end
|
@@ -1,7 +1,6 @@
|
|
1
1
|
# typed: true
|
2
2
|
# frozen_string_literal: true
|
3
3
|
|
4
|
-
require "toml-rb"
|
5
4
|
require "open3"
|
6
5
|
require "dependabot/dependency"
|
7
6
|
require "dependabot/python/requirement_parser"
|
@@ -10,7 +9,7 @@ require "dependabot/python/file_updater"
|
|
10
9
|
require "dependabot/python/language_version_manager"
|
11
10
|
require "dependabot/shared_helpers"
|
12
11
|
require "dependabot/python/native_helpers"
|
13
|
-
require "dependabot/python/
|
12
|
+
require "dependabot/python/pipenv_runner"
|
14
13
|
|
15
14
|
module Dependabot
|
16
15
|
module Python
|
@@ -22,12 +21,13 @@ module Dependabot
|
|
22
21
|
|
23
22
|
DEPENDENCY_TYPES = %w(packages dev-packages).freeze
|
24
23
|
|
25
|
-
attr_reader :dependencies, :dependency_files, :credentials
|
24
|
+
attr_reader :dependencies, :dependency_files, :credentials, :repo_contents_path
|
26
25
|
|
27
|
-
def initialize(dependencies:, dependency_files:, credentials:)
|
26
|
+
def initialize(dependencies:, dependency_files:, credentials:, repo_contents_path:)
|
28
27
|
@dependencies = dependencies
|
29
28
|
@dependency_files = dependency_files
|
30
29
|
@credentials = credentials
|
30
|
+
@repo_contents_path = repo_contents_path
|
31
31
|
end
|
32
32
|
|
33
33
|
def updated_dependency_files
|
@@ -83,7 +83,6 @@ module Dependabot
|
|
83
83
|
return [] unless lockfile
|
84
84
|
|
85
85
|
pipfile_lock_deps = parsed_lockfile[type]&.keys&.sort || []
|
86
|
-
pipfile_lock_deps = pipfile_lock_deps.map { |n| normalise(n) }
|
87
86
|
return [] unless pipfile_lock_deps.any?
|
88
87
|
|
89
88
|
regex = RequirementParser::INSTALL_REQ_WITH_REQUIREMENT
|
@@ -94,7 +93,7 @@ module Dependabot
|
|
94
93
|
requirements_files.select do |req_file|
|
95
94
|
deps = []
|
96
95
|
req_file.content.scan(regex) { deps << Regexp.last_match }
|
97
|
-
deps = deps.map { |m|
|
96
|
+
deps = deps.map { |m| m[:name] }
|
98
97
|
deps.sort == pipfile_lock_deps
|
99
98
|
end
|
100
99
|
end
|
@@ -129,61 +128,17 @@ module Dependabot
|
|
129
128
|
|
130
129
|
def prepared_pipfile_content
|
131
130
|
content = updated_pipfile_content
|
132
|
-
content = freeze_other_dependencies(content)
|
133
|
-
content = freeze_dependencies_being_updated(content)
|
134
131
|
content = add_private_sources(content)
|
135
132
|
content = update_python_requirement(content)
|
136
133
|
content
|
137
134
|
end
|
138
135
|
|
139
|
-
def freeze_other_dependencies(pipfile_content)
|
140
|
-
PipfilePreparer
|
141
|
-
.new(pipfile_content: pipfile_content, lockfile: lockfile)
|
142
|
-
.freeze_top_level_dependencies_except(dependencies)
|
143
|
-
end
|
144
|
-
|
145
136
|
def update_python_requirement(pipfile_content)
|
146
137
|
PipfilePreparer
|
147
138
|
.new(pipfile_content: pipfile_content)
|
148
139
|
.update_python_requirement(language_version_manager.python_major_minor)
|
149
140
|
end
|
150
141
|
|
151
|
-
# rubocop:disable Metrics/PerceivedComplexity
|
152
|
-
def freeze_dependencies_being_updated(pipfile_content)
|
153
|
-
pipfile_object = TomlRB.parse(pipfile_content)
|
154
|
-
|
155
|
-
dependencies.each do |dep|
|
156
|
-
DEPENDENCY_TYPES.each do |type|
|
157
|
-
names = pipfile_object[type]&.keys || []
|
158
|
-
pkg_name = names.find { |nm| normalise(nm) == dep.name }
|
159
|
-
next unless pkg_name || subdep_type?(type)
|
160
|
-
|
161
|
-
pkg_name ||= dependency.name
|
162
|
-
if pipfile_object[type][pkg_name].is_a?(Hash)
|
163
|
-
pipfile_object[type][pkg_name]["version"] =
|
164
|
-
"==#{dep.version}"
|
165
|
-
else
|
166
|
-
pipfile_object[type][pkg_name] = "==#{dep.version}"
|
167
|
-
end
|
168
|
-
end
|
169
|
-
end
|
170
|
-
|
171
|
-
TomlRB.dump(pipfile_object)
|
172
|
-
end
|
173
|
-
# rubocop:enable Metrics/PerceivedComplexity
|
174
|
-
|
175
|
-
def subdep_type?(type)
|
176
|
-
return false if dependency.top_level?
|
177
|
-
|
178
|
-
lockfile_type = Python::FileParser::DEPENDENCY_GROUP_KEYS
|
179
|
-
.find { |i| i.fetch(:pipfile) == type }
|
180
|
-
.fetch(:lockfile)
|
181
|
-
|
182
|
-
JSON.parse(lockfile.content)
|
183
|
-
.fetch(lockfile_type, {})
|
184
|
-
.keys.any? { |k| normalise(k) == dependency.name }
|
185
|
-
end
|
186
|
-
|
187
142
|
def add_private_sources(pipfile_content)
|
188
143
|
PipfilePreparer
|
189
144
|
.new(pipfile_content: pipfile_content)
|
@@ -192,14 +147,12 @@ module Dependabot
|
|
192
147
|
|
193
148
|
def updated_generated_files
|
194
149
|
@updated_generated_files ||=
|
195
|
-
SharedHelpers.
|
150
|
+
SharedHelpers.in_a_temporary_repo_directory(dependency_files.first.directory, repo_contents_path) do
|
196
151
|
SharedHelpers.with_git_configured(credentials: credentials) do
|
197
152
|
write_temporary_dependency_files(prepared_pipfile_content)
|
198
153
|
install_required_python
|
199
154
|
|
200
|
-
|
201
|
-
"pyenv exec pipenv lock"
|
202
|
-
)
|
155
|
+
pipenv_runner.run_upgrade("==#{dependency.version}")
|
203
156
|
|
204
157
|
result = { lockfile: File.read("Pipfile.lock") }
|
205
158
|
result[:lockfile] = post_process_lockfile(result[:lockfile])
|
@@ -245,17 +198,12 @@ module Dependabot
|
|
245
198
|
File.write("dev-req.txt", dev_req_content)
|
246
199
|
end
|
247
200
|
|
248
|
-
def run_command(command
|
249
|
-
SharedHelpers.run_shell_command(command
|
201
|
+
def run_command(command)
|
202
|
+
SharedHelpers.run_shell_command(command)
|
250
203
|
end
|
251
204
|
|
252
|
-
def run_pipenv_command(command
|
253
|
-
|
254
|
-
"pyenv local #{language_version_manager.python_major_minor}",
|
255
|
-
fingerprint: "pyenv local <python_major_minor>"
|
256
|
-
)
|
257
|
-
|
258
|
-
run_command(command, env: env)
|
205
|
+
def run_pipenv_command(command)
|
206
|
+
pipenv_runner.run(command)
|
259
207
|
end
|
260
208
|
|
261
209
|
def write_temporary_dependency_files(pipfile_content)
|
@@ -317,7 +265,7 @@ module Dependabot
|
|
317
265
|
SharedHelpers.run_helper_subprocess(
|
318
266
|
command: "pyenv exec python3 #{NativeHelpers.python_helper_path}",
|
319
267
|
function: "get_pipfile_hash",
|
320
|
-
args: [dir]
|
268
|
+
args: [T.cast(dir, Pathname).to_s]
|
321
269
|
)
|
322
270
|
end
|
323
271
|
end
|
@@ -328,10 +276,6 @@ module Dependabot
|
|
328
276
|
updated_file
|
329
277
|
end
|
330
278
|
|
331
|
-
def normalise(name)
|
332
|
-
NameNormaliser.normalise(name)
|
333
|
-
end
|
334
|
-
|
335
279
|
def python_requirement_parser
|
336
280
|
@python_requirement_parser ||=
|
337
281
|
FileParser::PythonRequirementParser.new(
|
@@ -346,6 +290,15 @@ module Dependabot
|
|
346
290
|
)
|
347
291
|
end
|
348
292
|
|
293
|
+
def pipenv_runner
|
294
|
+
@pipenv_runner ||=
|
295
|
+
PipenvRunner.new(
|
296
|
+
dependency: dependency,
|
297
|
+
lockfile: lockfile,
|
298
|
+
language_version_manager: language_version_manager
|
299
|
+
)
|
300
|
+
end
|
301
|
+
|
349
302
|
def parsed_lockfile
|
350
303
|
@parsed_lockfile ||= JSON.parse(lockfile.content)
|
351
304
|
end
|
@@ -369,16 +322,6 @@ module Dependabot
|
|
369
322
|
def requirements_files
|
370
323
|
dependency_files.select { |f| f.name.end_with?(".txt") }
|
371
324
|
end
|
372
|
-
|
373
|
-
def pipenv_env_variables
|
374
|
-
{
|
375
|
-
"PIPENV_YES" => "true", # Install new Python ver if needed
|
376
|
-
"PIPENV_MAX_RETRIES" => "3", # Retry timeouts
|
377
|
-
"PIPENV_NOSPIN" => "1", # Don't pollute logs with spinner
|
378
|
-
"PIPENV_TIMEOUT" => "600", # Set install timeout to 10 minutes
|
379
|
-
"PIP_DEFAULT_TIMEOUT" => "60" # Set pip timeout to 1 minute
|
380
|
-
}
|
381
|
-
end
|
382
325
|
end
|
383
326
|
end
|
384
327
|
end
|
@@ -7,15 +7,13 @@ require "dependabot/dependency"
|
|
7
7
|
require "dependabot/python/file_parser"
|
8
8
|
require "dependabot/python/file_updater"
|
9
9
|
require "dependabot/python/authed_url_builder"
|
10
|
-
require "dependabot/python/name_normaliser"
|
11
10
|
|
12
11
|
module Dependabot
|
13
12
|
module Python
|
14
13
|
class FileUpdater
|
15
14
|
class PipfilePreparer
|
16
|
-
def initialize(pipfile_content
|
15
|
+
def initialize(pipfile_content:)
|
17
16
|
@pipfile_content = pipfile_content
|
18
|
-
@lockfile = lockfile
|
19
17
|
end
|
20
18
|
|
21
19
|
def replace_sources(credentials)
|
@@ -28,45 +26,6 @@ module Dependabot
|
|
28
26
|
TomlRB.dump(pipfile_object)
|
29
27
|
end
|
30
28
|
|
31
|
-
def freeze_top_level_dependencies_except(dependencies)
|
32
|
-
return pipfile_content unless lockfile
|
33
|
-
|
34
|
-
pipfile_object = TomlRB.parse(pipfile_content)
|
35
|
-
excluded_names = dependencies.map(&:name)
|
36
|
-
|
37
|
-
Python::FileParser::DEPENDENCY_GROUP_KEYS.each do |keys|
|
38
|
-
next unless pipfile_object[keys[:pipfile]]
|
39
|
-
|
40
|
-
pipfile_object.fetch(keys[:pipfile]).each do |dep_name, _|
|
41
|
-
next if excluded_names.include?(normalise(dep_name))
|
42
|
-
|
43
|
-
freeze_dependency(dep_name, pipfile_object, keys)
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
TomlRB.dump(pipfile_object)
|
48
|
-
end
|
49
|
-
|
50
|
-
def freeze_dependency(dep_name, pipfile_object, keys)
|
51
|
-
locked_version = version_from_lockfile(
|
52
|
-
keys[:lockfile],
|
53
|
-
normalise(dep_name)
|
54
|
-
)
|
55
|
-
locked_ref = ref_from_lockfile(
|
56
|
-
keys[:lockfile],
|
57
|
-
normalise(dep_name)
|
58
|
-
)
|
59
|
-
|
60
|
-
pipfile_req = pipfile_object[keys[:pipfile]][dep_name]
|
61
|
-
if pipfile_req.is_a?(Hash) && locked_version
|
62
|
-
pipfile_req["version"] = "==#{locked_version}"
|
63
|
-
elsif pipfile_req.is_a?(Hash) && locked_ref && !pipfile_req["ref"]
|
64
|
-
pipfile_req["ref"] = locked_ref
|
65
|
-
elsif locked_version
|
66
|
-
pipfile_object[keys[:pipfile]][dep_name] = "==#{locked_version}"
|
67
|
-
end
|
68
|
-
end
|
69
|
-
|
70
29
|
def update_python_requirement(requirement)
|
71
30
|
pipfile_object = TomlRB.parse(pipfile_content)
|
72
31
|
|
@@ -84,31 +43,6 @@ module Dependabot
|
|
84
43
|
|
85
44
|
attr_reader :pipfile_content, :lockfile
|
86
45
|
|
87
|
-
def version_from_lockfile(dep_type, dep_name)
|
88
|
-
details = parsed_lockfile.dig(dep_type, normalise(dep_name))
|
89
|
-
|
90
|
-
case details
|
91
|
-
when String then details.gsub(/^==/, "")
|
92
|
-
when Hash then details["version"]&.gsub(/^==/, "")
|
93
|
-
end
|
94
|
-
end
|
95
|
-
|
96
|
-
def ref_from_lockfile(dep_type, dep_name)
|
97
|
-
details = parsed_lockfile.dig(dep_type, normalise(dep_name))
|
98
|
-
|
99
|
-
case details
|
100
|
-
when Hash then details["ref"]
|
101
|
-
end
|
102
|
-
end
|
103
|
-
|
104
|
-
def parsed_lockfile
|
105
|
-
@parsed_lockfile ||= JSON.parse(lockfile.content)
|
106
|
-
end
|
107
|
-
|
108
|
-
def normalise(name)
|
109
|
-
NameNormaliser.normalise(name)
|
110
|
-
end
|
111
|
-
|
112
46
|
def pipfile_sources
|
113
47
|
@pipfile_sources ||= TomlRB.parse(pipfile_content).fetch("source", [])
|
114
48
|
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
# typed: true
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require "dependabot/shared_helpers"
|
5
|
+
require "dependabot/python/file_parser"
|
6
|
+
require "json"
|
7
|
+
|
8
|
+
module Dependabot
|
9
|
+
module Python
|
10
|
+
class PipenvRunner
|
11
|
+
def initialize(dependency:, lockfile:, language_version_manager:)
|
12
|
+
@dependency = dependency
|
13
|
+
@lockfile = lockfile
|
14
|
+
@language_version_manager = language_version_manager
|
15
|
+
end
|
16
|
+
|
17
|
+
def run_upgrade(constraint)
|
18
|
+
command = "pyenv exec pipenv upgrade #{dependency_name}#{constraint}"
|
19
|
+
command << " --dev" if lockfile_section == "develop"
|
20
|
+
|
21
|
+
run(command, fingerprint: "pyenv exec pipenv upgrade <dependency_name><constraint>")
|
22
|
+
end
|
23
|
+
|
24
|
+
def run_upgrade_and_fetch_version(constraint)
|
25
|
+
run_upgrade(constraint)
|
26
|
+
|
27
|
+
updated_lockfile = JSON.parse(File.read("Pipfile.lock"))
|
28
|
+
|
29
|
+
fetch_version_from_parsed_lockfile(updated_lockfile)
|
30
|
+
end
|
31
|
+
|
32
|
+
def run(command, fingerprint: nil)
|
33
|
+
run_command(
|
34
|
+
"pyenv local #{language_version_manager.python_major_minor}",
|
35
|
+
fingerprint: "pyenv local <python_major_minor>"
|
36
|
+
)
|
37
|
+
|
38
|
+
run_command(command, fingerprint: fingerprint)
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
|
43
|
+
attr_reader :dependency, :lockfile, :language_version_manager
|
44
|
+
|
45
|
+
def fetch_version_from_parsed_lockfile(updated_lockfile)
|
46
|
+
deps = updated_lockfile[lockfile_section] || {}
|
47
|
+
|
48
|
+
deps.dig(dependency_name, "version")
|
49
|
+
&.gsub(/^==/, "")
|
50
|
+
end
|
51
|
+
|
52
|
+
def run_command(command, fingerprint: nil)
|
53
|
+
SharedHelpers.run_shell_command(command, env: pipenv_env_variables, fingerprint: fingerprint)
|
54
|
+
end
|
55
|
+
|
56
|
+
def lockfile_section
|
57
|
+
if dependency.requirements.any?
|
58
|
+
dependency.requirements.first[:groups].first
|
59
|
+
else
|
60
|
+
Python::FileParser::DEPENDENCY_GROUP_KEYS.each do |keys|
|
61
|
+
section = keys.fetch(:lockfile)
|
62
|
+
return section if JSON.parse(lockfile.content)[section].keys.any?(dependency_name)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def dependency_name
|
68
|
+
dependency.metadata[:original_name] || dependency.name
|
69
|
+
end
|
70
|
+
|
71
|
+
def pipenv_env_variables
|
72
|
+
{
|
73
|
+
"PIPENV_YES" => "true", # Install new Python ver if needed
|
74
|
+
"PIPENV_MAX_RETRIES" => "3", # Retry timeouts
|
75
|
+
"PIPENV_NOSPIN" => "1", # Don't pollute logs with spinner
|
76
|
+
"PIPENV_TIMEOUT" => "600", # Set install timeout to 10 minutes
|
77
|
+
"PIP_DEFAULT_TIMEOUT" => "60" # Set pip timeout to 1 minute
|
78
|
+
}
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
@@ -12,9 +12,10 @@ module Dependabot
|
|
12
12
|
PYPI_BASE_URL = "https://pypi.org/simple/"
|
13
13
|
ENVIRONMENT_VARIABLE_REGEX = /\$\{.+\}/
|
14
14
|
|
15
|
-
def initialize(dependency_files:, credentials:)
|
15
|
+
def initialize(dependency_files:, credentials:, dependency:)
|
16
16
|
@dependency_files = dependency_files
|
17
17
|
@credentials = credentials
|
18
|
+
@dependency = dependency
|
18
19
|
end
|
19
20
|
|
20
21
|
def index_urls
|
@@ -124,7 +125,11 @@ module Dependabot
|
|
124
125
|
|
125
126
|
if source["default"]
|
126
127
|
urls[:main] = source["url"]
|
127
|
-
|
128
|
+
elsif source["priority"] != "explicit"
|
129
|
+
# if source is not explicit, add it to extra
|
130
|
+
urls[:extra] << source["url"]
|
131
|
+
elsif @dependency.all_sources.include?(source["name"])
|
132
|
+
# if source is explicit, and dependency has specified it as a source, add it to extra
|
128
133
|
urls[:extra] << source["url"]
|
129
134
|
end
|
130
135
|
end
|
@@ -33,12 +33,13 @@ module Dependabot
|
|
33
33
|
RESOLUTION_IMPOSSIBLE_ERROR = "ResolutionImpossible"
|
34
34
|
ERROR_REGEX = /(?<=ERROR\:\W).*$/
|
35
35
|
|
36
|
-
attr_reader :dependency, :dependency_files, :credentials
|
36
|
+
attr_reader :dependency, :dependency_files, :credentials, :repo_contents_path
|
37
37
|
|
38
|
-
def initialize(dependency:, dependency_files:, credentials:)
|
38
|
+
def initialize(dependency:, dependency_files:, credentials:, repo_contents_path:)
|
39
39
|
@dependency = dependency
|
40
40
|
@dependency_files = dependency_files
|
41
41
|
@credentials = credentials
|
42
|
+
@repo_contents_path = repo_contents_path
|
42
43
|
@build_isolation = true
|
43
44
|
end
|
44
45
|
|
@@ -2,7 +2,6 @@
|
|
2
2
|
# frozen_string_literal: true
|
3
3
|
|
4
4
|
require "excon"
|
5
|
-
require "toml-rb"
|
6
5
|
require "open3"
|
7
6
|
require "dependabot/dependency"
|
8
7
|
require "dependabot/errors"
|
@@ -13,21 +12,12 @@ require "dependabot/python/file_updater/pipfile_preparer"
|
|
13
12
|
require "dependabot/python/file_updater/setup_file_sanitizer"
|
14
13
|
require "dependabot/python/update_checker"
|
15
14
|
require "dependabot/python/native_helpers"
|
16
|
-
require "dependabot/python/
|
15
|
+
require "dependabot/python/pipenv_runner"
|
17
16
|
require "dependabot/python/version"
|
18
17
|
|
19
18
|
module Dependabot
|
20
19
|
module Python
|
21
20
|
class UpdateChecker
|
22
|
-
# This class does version resolution for Pipfiles. Its current approach
|
23
|
-
# is somewhat crude:
|
24
|
-
# - Unlock the dependency we're checking in the Pipfile
|
25
|
-
# - Freeze all of the other dependencies in the Pipfile
|
26
|
-
# - Run `pipenv lock` and see what the result is
|
27
|
-
#
|
28
|
-
# Unfortunately, Pipenv doesn't resolve how we'd expect - it appears to
|
29
|
-
# just raise if the latest version can't be resolved. Knowing that is
|
30
|
-
# still better than nothing, though.
|
31
21
|
class PipenvVersionResolver
|
32
22
|
GIT_DEPENDENCY_UNREACHABLE_REGEX = /git clone --filter=blob:none (?<url>[^\s]+).*/
|
33
23
|
GIT_REFERENCE_NOT_FOUND_REGEX = /git checkout -q (?<tag>[^\s]+).*/
|
@@ -37,14 +27,13 @@ module Dependabot
|
|
37
27
|
|
38
28
|
PIPENV_RANGE_WARNING = /Warning:\sPython\s[<>].* was not found/
|
39
29
|
|
40
|
-
|
30
|
+
attr_reader :dependency, :dependency_files, :credentials, :repo_contents_path
|
41
31
|
|
42
|
-
|
43
|
-
|
44
|
-
def initialize(dependency:, dependency_files:, credentials:)
|
32
|
+
def initialize(dependency:, dependency_files:, credentials:, repo_contents_path:)
|
45
33
|
@dependency = dependency
|
46
34
|
@dependency_files = dependency_files
|
47
35
|
@credentials = credentials
|
36
|
+
@repo_contents_path = repo_contents_path
|
48
37
|
end
|
49
38
|
|
50
39
|
def latest_resolvable_version(requirement: nil)
|
@@ -68,53 +57,18 @@ module Dependabot
|
|
68
57
|
return @latest_resolvable_version_string[requirement] if @latest_resolvable_version_string.key?(requirement)
|
69
58
|
|
70
59
|
@latest_resolvable_version_string[requirement] ||=
|
71
|
-
SharedHelpers.
|
60
|
+
SharedHelpers.in_a_temporary_repo_directory(base_directory, repo_contents_path) do
|
72
61
|
SharedHelpers.with_git_configured(credentials: credentials) do
|
73
|
-
write_temporary_dependency_files
|
62
|
+
write_temporary_dependency_files
|
74
63
|
install_required_python
|
75
64
|
|
76
|
-
|
77
|
-
# Whilst calling `lock` avoids doing an install as part of the
|
78
|
-
# pipenv flow, an install is still done by pip-tools in order
|
79
|
-
# to resolve the dependencies. That means this is slow.
|
80
|
-
run_pipenv_command("pyenv exec pipenv lock")
|
81
|
-
|
82
|
-
updated_lockfile = JSON.parse(File.read("Pipfile.lock"))
|
83
|
-
|
84
|
-
fetch_version_from_parsed_lockfile(updated_lockfile)
|
65
|
+
pipenv_runner.run_upgrade_and_fetch_version(requirement)
|
85
66
|
end
|
86
67
|
rescue SharedHelpers::HelperSubprocessFailed => e
|
87
68
|
handle_pipenv_errors(e)
|
88
69
|
end
|
89
70
|
end
|
90
71
|
|
91
|
-
def fetch_version_from_parsed_lockfile(updated_lockfile)
|
92
|
-
if dependency.requirements.any?
|
93
|
-
group = dependency.requirements.first[:groups].first
|
94
|
-
deps = updated_lockfile[group] || {}
|
95
|
-
|
96
|
-
version =
|
97
|
-
deps.transform_keys { |k| normalise(k) }
|
98
|
-
.dig(dependency.name, "version")
|
99
|
-
&.gsub(/^==/, "")
|
100
|
-
|
101
|
-
return version
|
102
|
-
end
|
103
|
-
|
104
|
-
Python::FileParser::DEPENDENCY_GROUP_KEYS.each do |keys|
|
105
|
-
deps = updated_lockfile[keys.fetch(:lockfile)] || {}
|
106
|
-
version =
|
107
|
-
deps.transform_keys { |k| normalise(k) }
|
108
|
-
.dig(dependency.name, "version")
|
109
|
-
&.gsub(/^==/, "")
|
110
|
-
|
111
|
-
return version if version
|
112
|
-
end
|
113
|
-
|
114
|
-
# If the sub-dependency no longer appears in the lockfile return nil
|
115
|
-
nil
|
116
|
-
end
|
117
|
-
|
118
72
|
# rubocop:disable Metrics/CyclomaticComplexity
|
119
73
|
# rubocop:disable Metrics/PerceivedComplexity
|
120
74
|
# rubocop:disable Metrics/AbcSize
|
@@ -190,10 +144,10 @@ module Dependabot
|
|
190
144
|
# boolean, so that all deps for this repo will raise identical
|
191
145
|
# errors when failing to update
|
192
146
|
def check_original_requirements_resolvable
|
193
|
-
SharedHelpers.
|
147
|
+
SharedHelpers.in_a_temporary_repo_directory(base_directory, repo_contents_path) do
|
194
148
|
write_temporary_dependency_files(update_pipfile: false)
|
195
149
|
|
196
|
-
|
150
|
+
pipenv_runner.run_upgrade("==#{dependency.version}")
|
197
151
|
|
198
152
|
true
|
199
153
|
rescue SharedHelpers::HelperSubprocessFailed => e
|
@@ -201,6 +155,10 @@ module Dependabot
|
|
201
155
|
end
|
202
156
|
end
|
203
157
|
|
158
|
+
def base_directory
|
159
|
+
dependency_files.first.directory
|
160
|
+
end
|
161
|
+
|
204
162
|
def handle_pipenv_errors_resolving_original_reqs(error)
|
205
163
|
if error.message.include?("Could not find a version") ||
|
206
164
|
error.message.include?("package versions have conflicting dependencies")
|
@@ -264,8 +222,7 @@ module Dependabot
|
|
264
222
|
raise DependencyFileNotResolvable, msg
|
265
223
|
end
|
266
224
|
|
267
|
-
def write_temporary_dependency_files(
|
268
|
-
update_pipfile: true)
|
225
|
+
def write_temporary_dependency_files(update_pipfile: true)
|
269
226
|
dependency_files.each do |file|
|
270
227
|
path = file.name
|
271
228
|
FileUtils.mkdir_p(Pathname.new(path).dirname)
|
@@ -291,7 +248,7 @@ module Dependabot
|
|
291
248
|
# Overwrite the pipfile with updated content
|
292
249
|
File.write(
|
293
250
|
"Pipfile",
|
294
|
-
pipfile_content
|
251
|
+
pipfile_content
|
295
252
|
)
|
296
253
|
end
|
297
254
|
|
@@ -319,93 +276,27 @@ module Dependabot
|
|
319
276
|
dependency_files.find { |f| f.name == config_name }
|
320
277
|
end
|
321
278
|
|
322
|
-
def pipfile_content
|
279
|
+
def pipfile_content
|
323
280
|
content = pipfile.content
|
324
|
-
content = freeze_other_dependencies(content)
|
325
|
-
content = set_target_dependency_req(content, updated_requirement)
|
326
281
|
content = add_private_sources(content)
|
327
282
|
content = update_python_requirement(content)
|
328
283
|
content
|
329
284
|
end
|
330
285
|
|
331
|
-
def freeze_other_dependencies(pipfile_content)
|
332
|
-
Python::FileUpdater::PipfilePreparer
|
333
|
-
.new(pipfile_content: pipfile_content, lockfile: lockfile)
|
334
|
-
.freeze_top_level_dependencies_except([dependency])
|
335
|
-
end
|
336
|
-
|
337
286
|
def update_python_requirement(pipfile_content)
|
338
287
|
Python::FileUpdater::PipfilePreparer
|
339
288
|
.new(pipfile_content: pipfile_content)
|
340
289
|
.update_python_requirement(language_version_manager.python_major_minor)
|
341
290
|
end
|
342
291
|
|
343
|
-
# rubocop:disable Metrics/PerceivedComplexity
|
344
|
-
def set_target_dependency_req(pipfile_content, updated_requirement)
|
345
|
-
return pipfile_content unless updated_requirement
|
346
|
-
|
347
|
-
pipfile_object = TomlRB.parse(pipfile_content)
|
348
|
-
|
349
|
-
DEPENDENCY_TYPES.each do |type|
|
350
|
-
names = pipfile_object[type]&.keys || []
|
351
|
-
pkg_name = names.find { |nm| normalise(nm) == dependency.name }
|
352
|
-
next unless pkg_name || subdep_type?(type)
|
353
|
-
|
354
|
-
pkg_name ||= dependency.name
|
355
|
-
if pipfile_object.dig(type, pkg_name).is_a?(Hash)
|
356
|
-
pipfile_object[type][pkg_name]["version"] = updated_requirement
|
357
|
-
else
|
358
|
-
pipfile_object[type][pkg_name] = updated_requirement
|
359
|
-
end
|
360
|
-
end
|
361
|
-
|
362
|
-
TomlRB.dump(pipfile_object)
|
363
|
-
end
|
364
|
-
# rubocop:enable Metrics/PerceivedComplexity
|
365
|
-
|
366
|
-
def subdep_type?(type)
|
367
|
-
return false if dependency.top_level?
|
368
|
-
|
369
|
-
lockfile_type = Python::FileParser::DEPENDENCY_GROUP_KEYS
|
370
|
-
.find { |i| i.fetch(:pipfile) == type }
|
371
|
-
.fetch(:lockfile)
|
372
|
-
|
373
|
-
JSON.parse(lockfile.content)
|
374
|
-
.fetch(lockfile_type, {})
|
375
|
-
.keys.any? { |k| normalise(k) == dependency.name }
|
376
|
-
end
|
377
|
-
|
378
292
|
def add_private_sources(pipfile_content)
|
379
293
|
Python::FileUpdater::PipfilePreparer
|
380
294
|
.new(pipfile_content: pipfile_content)
|
381
295
|
.replace_sources(credentials)
|
382
296
|
end
|
383
297
|
|
384
|
-
def run_command(command
|
385
|
-
SharedHelpers.run_shell_command(command,
|
386
|
-
end
|
387
|
-
|
388
|
-
def run_pipenv_command(command, env: pipenv_env_variables)
|
389
|
-
run_command(
|
390
|
-
"pyenv local #{language_version_manager.python_major_minor}",
|
391
|
-
fingerprint: "pyenv local <python_major_minor>"
|
392
|
-
)
|
393
|
-
|
394
|
-
run_command(command, env: env)
|
395
|
-
end
|
396
|
-
|
397
|
-
def pipenv_env_variables
|
398
|
-
{
|
399
|
-
"PIPENV_YES" => "true", # Install new Python ver if needed
|
400
|
-
"PIPENV_MAX_RETRIES" => "3", # Retry timeouts
|
401
|
-
"PIPENV_NOSPIN" => "1", # Don't pollute logs with spinner
|
402
|
-
"PIPENV_TIMEOUT" => "600", # Set install timeout to 10 minutes
|
403
|
-
"PIP_DEFAULT_TIMEOUT" => "60" # Set pip timeout to 1 minute
|
404
|
-
}
|
405
|
-
end
|
406
|
-
|
407
|
-
def normalise(name)
|
408
|
-
NameNormaliser.normalise(name)
|
298
|
+
def run_command(command)
|
299
|
+
SharedHelpers.run_shell_command(command, stderr_to_stdout: true)
|
409
300
|
end
|
410
301
|
|
411
302
|
def python_requirement_parser
|
@@ -422,6 +313,15 @@ module Dependabot
|
|
422
313
|
)
|
423
314
|
end
|
424
315
|
|
316
|
+
def pipenv_runner
|
317
|
+
@pipenv_runner ||=
|
318
|
+
PipenvRunner.new(
|
319
|
+
dependency: dependency,
|
320
|
+
lockfile: lockfile,
|
321
|
+
language_version_manager: language_version_manager
|
322
|
+
)
|
323
|
+
end
|
324
|
+
|
425
325
|
def pipfile
|
426
326
|
dependency_files.find { |f| f.name == "Pipfile" }
|
427
327
|
end
|
@@ -38,12 +38,13 @@ module Dependabot
|
|
38
38
|
\s+check\syour\sgit\sconfiguration
|
39
39
|
/mx
|
40
40
|
|
41
|
-
attr_reader :dependency, :dependency_files, :credentials
|
41
|
+
attr_reader :dependency, :dependency_files, :credentials, :repo_contents_path
|
42
42
|
|
43
|
-
def initialize(dependency:, dependency_files:, credentials:)
|
43
|
+
def initialize(dependency:, dependency_files:, credentials:, repo_contents_path:)
|
44
44
|
@dependency = dependency
|
45
45
|
@dependency_files = dependency_files
|
46
46
|
@credentials = credentials
|
47
|
+
@repo_contents_path = repo_contents_path
|
47
48
|
end
|
48
49
|
|
49
50
|
def latest_resolvable_version(requirement: nil)
|
@@ -191,7 +191,8 @@ module Dependabot
|
|
191
191
|
{
|
192
192
|
dependency: dependency,
|
193
193
|
dependency_files: dependency_files,
|
194
|
-
credentials: credentials
|
194
|
+
credentials: credentials,
|
195
|
+
repo_contents_path: repo_contents_path
|
195
196
|
}
|
196
197
|
end
|
197
198
|
|
@@ -221,11 +222,11 @@ module Dependabot
|
|
221
222
|
return lower_bound_req if latest_version.nil?
|
222
223
|
return lower_bound_req unless Python::Version.correct?(latest_version)
|
223
224
|
|
224
|
-
lower_bound_req + "
|
225
|
+
lower_bound_req + ",<=#{latest_version}"
|
225
226
|
end
|
226
227
|
|
227
228
|
def updated_version_req_lower_bound
|
228
|
-
return "
|
229
|
+
return ">=#{dependency.version}" if dependency.version
|
229
230
|
|
230
231
|
version_for_requirement =
|
231
232
|
requirements.filter_map { |r| r[:requirement] }
|
@@ -235,7 +236,7 @@ module Dependabot
|
|
235
236
|
.select { |version| Gem::Version.correct?(version) }
|
236
237
|
.max_by { |version| Gem::Version.new(version) }
|
237
238
|
|
238
|
-
"
|
239
|
+
">=#{version_for_requirement || 0}"
|
239
240
|
end
|
240
241
|
|
241
242
|
def fetch_latest_version
|
data/lib/dependabot/python.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dependabot-python
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.237.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Dependabot
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
11
|
+
date: 2023-11-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: dependabot-common
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - '='
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 0.
|
19
|
+
version: 0.237.0
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - '='
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 0.
|
26
|
+
version: 0.237.0
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: debug
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -94,20 +94,34 @@ dependencies:
|
|
94
94
|
- - "~>"
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '1.3'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rspec-sorbet
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: 1.9.2
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: 1.9.2
|
97
111
|
- !ruby/object:Gem::Dependency
|
98
112
|
name: rubocop
|
99
113
|
requirement: !ruby/object:Gem::Requirement
|
100
114
|
requirements:
|
101
115
|
- - "~>"
|
102
116
|
- !ruby/object:Gem::Version
|
103
|
-
version: 1.
|
117
|
+
version: 1.57.2
|
104
118
|
type: :development
|
105
119
|
prerelease: false
|
106
120
|
version_requirements: !ruby/object:Gem::Requirement
|
107
121
|
requirements:
|
108
122
|
- - "~>"
|
109
123
|
- !ruby/object:Gem::Version
|
110
|
-
version: 1.
|
124
|
+
version: 1.57.2
|
111
125
|
- !ruby/object:Gem::Dependency
|
112
126
|
name: rubocop-performance
|
113
127
|
requirement: !ruby/object:Gem::Requirement
|
@@ -229,6 +243,7 @@ files:
|
|
229
243
|
- lib/dependabot/python/name_normaliser.rb
|
230
244
|
- lib/dependabot/python/native_helpers.rb
|
231
245
|
- lib/dependabot/python/pip_compile_file_matcher.rb
|
246
|
+
- lib/dependabot/python/pipenv_runner.rb
|
232
247
|
- lib/dependabot/python/requirement.rb
|
233
248
|
- lib/dependabot/python/requirement_parser.rb
|
234
249
|
- lib/dependabot/python/update_checker.rb
|
@@ -245,7 +260,7 @@ licenses:
|
|
245
260
|
- Nonstandard
|
246
261
|
metadata:
|
247
262
|
bug_tracker_uri: https://github.com/dependabot/dependabot-core/issues
|
248
|
-
changelog_uri: https://github.com/dependabot/dependabot-core/releases/tag/v0.
|
263
|
+
changelog_uri: https://github.com/dependabot/dependabot-core/releases/tag/v0.237.0
|
249
264
|
post_install_message:
|
250
265
|
rdoc_options: []
|
251
266
|
require_paths:
|