dependabot-python 0.211.0 → 0.213.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/helpers/build +1 -6
- data/helpers/lib/parser.py +52 -0
- data/helpers/requirements.txt +3 -3
- data/helpers/run.py +2 -0
- data/lib/dependabot/python/file_fetcher.rb +24 -14
- data/lib/dependabot/python/file_parser/{poetry_files_parser.rb → pyproject_files_parser.rb} +87 -5
- data/lib/dependabot/python/file_parser/python_requirement_parser.rb +1 -2
- data/lib/dependabot/python/file_parser/setup_file_parser.rb +5 -5
- data/lib/dependabot/python/file_parser.rb +5 -29
- data/lib/dependabot/python/file_updater/pip_compile_file_updater.rb +14 -29
- data/lib/dependabot/python/file_updater/pipfile_file_updater.rb +7 -9
- data/lib/dependabot/python/file_updater/poetry_file_updater.rb +7 -6
- data/lib/dependabot/python/file_updater/pyproject_preparer.rb +3 -2
- data/lib/dependabot/python/file_updater/requirement_file_updater.rb +2 -2
- data/lib/dependabot/python/file_updater/requirement_replacer.rb +2 -2
- data/lib/dependabot/python/file_updater/setup_file_sanitizer.rb +8 -8
- data/lib/dependabot/python/file_updater.rb +15 -2
- data/lib/dependabot/python/helpers.rb +20 -0
- data/lib/dependabot/python/metadata_finder.rb +2 -0
- data/lib/dependabot/python/native_helpers.rb +1 -1
- data/lib/dependabot/python/python_versions.rb +5 -5
- data/lib/dependabot/python/requirement.rb +7 -4
- data/lib/dependabot/python/requirement_parser.rb +20 -23
- data/lib/dependabot/python/update_checker/index_finder.rb +2 -2
- data/lib/dependabot/python/update_checker/latest_version_finder.rb +10 -7
- data/lib/dependabot/python/update_checker/pip_compile_version_resolver.rb +17 -19
- data/lib/dependabot/python/update_checker/pipenv_version_resolver.rb +29 -34
- data/lib/dependabot/python/update_checker/poetry_version_resolver.rb +45 -26
- data/lib/dependabot/python/update_checker/requirements_updater.rb +18 -5
- data/lib/dependabot/python/update_checker.rb +82 -27
- data/lib/dependabot/python/version.rb +2 -2
- metadata +16 -43
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6cb23890c79504e40e7e4962485003c76c07179574ac89b210b6529d15d2c216
|
4
|
+
data.tar.gz: b96523f9cf991cbffc38fc2831221c43450c74ef560e8e80ff2d2bbf73c889c4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5beeac4ec63193ce095e6a5d7223c11e4e9c2ace55b3ef5c94f0011d8cb0c70fc7b364108b68cd68f656a5ed79d712b64eecbb75998714325a0f3b101169592c
|
7
|
+
data.tar.gz: 17ec5483c750fe4bc35490feebf418a8bd7eaf0eb2d14e2bcfc811e06f94c02f69eca71bafb589cb2d51b8a73155d3d358b97fcc6125a8fb7229b54f06a42fbe
|
data/helpers/build
CHANGED
@@ -18,9 +18,4 @@ cp -r \
|
|
18
18
|
"$install_dir"
|
19
19
|
|
20
20
|
cd "$install_dir"
|
21
|
-
PYENV_VERSION=3.10.
|
22
|
-
|
23
|
-
# Workaround of https://github.com/python-poetry/poetry/issues/3010
|
24
|
-
# By default poetry config file is stored under ~/.config/pypoetry
|
25
|
-
# and is not bound to any specific Python version
|
26
|
-
PYENV_VERSION=3.10.5 pyenv exec poetry config experimental.new-installer false
|
21
|
+
PYENV_VERSION=3.10.7 pyenv exec pip --disable-pip-version-check install --use-pep517 -r "requirements.txt"
|
data/helpers/lib/parser.py
CHANGED
@@ -11,11 +11,63 @@ from pip._internal.req.constructors import (
|
|
11
11
|
install_req_from_line,
|
12
12
|
install_req_from_parsed_requirement,
|
13
13
|
)
|
14
|
+
|
15
|
+
from packaging.requirements import InvalidRequirement, Requirement
|
16
|
+
import toml
|
17
|
+
|
14
18
|
# Inspired by pips internal check:
|
15
19
|
# https://github.com/pypa/pip/blob/0bb3ac87f5bb149bd75cceac000844128b574385/src/pip/_internal/req/req_file.py#L35
|
16
20
|
COMMENT_RE = re.compile(r'(^|\s+)#.*$')
|
17
21
|
|
18
22
|
|
23
|
+
def parse_pep621_dependencies(pyproject_path):
|
24
|
+
project_toml = toml.load(pyproject_path)['project']
|
25
|
+
|
26
|
+
def parse_toml_section_pep621_dependencies(pyproject_path, dependencies):
|
27
|
+
requirement_packages = []
|
28
|
+
|
29
|
+
def version_from_req(specifier_set):
|
30
|
+
if (len(specifier_set) == 1 and
|
31
|
+
next(iter(specifier_set)).operator in {"==", "==="}):
|
32
|
+
return next(iter(specifier_set)).version
|
33
|
+
|
34
|
+
for dependency in dependencies:
|
35
|
+
try:
|
36
|
+
req = Requirement(dependency)
|
37
|
+
except InvalidRequirement as e:
|
38
|
+
print(json.dumps({"error": repr(e)}))
|
39
|
+
exit(1)
|
40
|
+
else:
|
41
|
+
requirement_packages.append({
|
42
|
+
"name": req.name,
|
43
|
+
"version": version_from_req(req.specifier),
|
44
|
+
"markers": str(req.marker) or None,
|
45
|
+
"file": pyproject_path,
|
46
|
+
"requirement": str(req.specifier),
|
47
|
+
"extras": sorted(list(req.extras))
|
48
|
+
})
|
49
|
+
|
50
|
+
return requirement_packages
|
51
|
+
|
52
|
+
dependencies = parse_toml_section_pep621_dependencies(
|
53
|
+
pyproject_path,
|
54
|
+
project_toml['dependencies']
|
55
|
+
)
|
56
|
+
|
57
|
+
if 'optional-dependencies' in project_toml:
|
58
|
+
optional_dependencies_toml = project_toml['optional-dependencies']
|
59
|
+
|
60
|
+
for group in optional_dependencies_toml:
|
61
|
+
group_dependencies = parse_toml_section_pep621_dependencies(
|
62
|
+
pyproject_path,
|
63
|
+
optional_dependencies_toml[group]
|
64
|
+
)
|
65
|
+
|
66
|
+
dependencies.extend(group_dependencies)
|
67
|
+
|
68
|
+
return json.dumps({"result": dependencies})
|
69
|
+
|
70
|
+
|
19
71
|
def parse_requirements(directory):
|
20
72
|
# Parse the requirements.txt
|
21
73
|
requirement_packages = []
|
data/helpers/requirements.txt
CHANGED
@@ -1,10 +1,10 @@
|
|
1
|
-
pip>=21.3.1,<22.
|
2
|
-
pip-tools>=6.4.0,<6.
|
1
|
+
pip>=21.3.1,<22.4.0 # Range maintains py36 support TODO: Review python 3.6 support in April 2023 (eol ubuntu 18.04)
|
2
|
+
pip-tools>=6.4.0,<6.9.1 # Range maintains py36 support TODO: Review python 3.6 support in April 2023 (eol ubuntu 18.04)
|
3
3
|
flake8==5.0.4
|
4
4
|
hashin==0.17.0
|
5
5
|
pipenv==2022.4.8
|
6
6
|
pipfile==0.0.2
|
7
|
-
poetry
|
7
|
+
poetry>=1.1.15,<1.3.0
|
8
8
|
wheel==0.37.1
|
9
9
|
|
10
10
|
# Some dependencies will only install if Cython is present
|
data/helpers/run.py
CHANGED
@@ -10,6 +10,8 @@ if __name__ == "__main__":
|
|
10
10
|
print(parser.parse_requirements(args["args"][0]))
|
11
11
|
elif args["function"] == "parse_setup":
|
12
12
|
print(parser.parse_setup(args["args"][0]))
|
13
|
+
elif args["function"] == "parse_pep621_dependencies":
|
14
|
+
print(parser.parse_pep621_dependencies(args["args"][0]))
|
13
15
|
elif args["function"] == "get_dependency_hash":
|
14
16
|
print(hasher.get_dependency_hash(*args["args"]))
|
15
17
|
elif args["function"] == "get_pipfile_hash":
|
@@ -5,14 +5,15 @@ require "toml-rb"
|
|
5
5
|
require "dependabot/file_fetchers"
|
6
6
|
require "dependabot/file_fetchers/base"
|
7
7
|
require "dependabot/python/requirement_parser"
|
8
|
-
require "dependabot/python/file_parser/
|
8
|
+
require "dependabot/python/file_parser/pyproject_files_parser"
|
9
9
|
require "dependabot/errors"
|
10
10
|
|
11
11
|
module Dependabot
|
12
12
|
module Python
|
13
13
|
class FileFetcher < Dependabot::FileFetchers::Base
|
14
|
-
CHILD_REQUIREMENT_REGEX = /^-r\s?(?<path>.*\.(?:txt|in))
|
15
|
-
CONSTRAINT_REGEX = /^-c\s?(?<path>.*\.(?:txt|in))
|
14
|
+
CHILD_REQUIREMENT_REGEX = /^-r\s?(?<path>.*\.(?:txt|in))/
|
15
|
+
CONSTRAINT_REGEX = /^-c\s?(?<path>.*\.(?:txt|in))/
|
16
|
+
DEPENDENCY_TYPES = %w(packages dev-packages)
|
16
17
|
|
17
18
|
def self.required_files_in?(filenames)
|
18
19
|
return true if filenames.any? { |name| name.end_with?(".txt", ".in") }
|
@@ -23,7 +24,7 @@ module Dependabot
|
|
23
24
|
# If this repo is using a Pipfile return true
|
24
25
|
return true if filenames.include?("Pipfile")
|
25
26
|
|
26
|
-
# If this repo is using
|
27
|
+
# If this repo is using pyproject.toml return true
|
27
28
|
return true if filenames.include?("pyproject.toml")
|
28
29
|
|
29
30
|
return true if filenames.include?("setup.py")
|
@@ -32,8 +33,8 @@ module Dependabot
|
|
32
33
|
end
|
33
34
|
|
34
35
|
def self.required_files_message
|
35
|
-
"Repo must contain a requirements.txt, setup.py, setup.cfg, pyproject.toml, "\
|
36
|
-
|
36
|
+
"Repo must contain a requirements.txt, setup.py, setup.cfg, pyproject.toml, " \
|
37
|
+
"or a Pipfile."
|
37
38
|
end
|
38
39
|
|
39
40
|
private
|
@@ -68,7 +69,7 @@ module Dependabot
|
|
68
69
|
end
|
69
70
|
|
70
71
|
def pyproject_files
|
71
|
-
[pyproject, pyproject_lock, poetry_lock].compact
|
72
|
+
[pyproject, pyproject_lock, poetry_lock, pdm_lock].compact
|
72
73
|
end
|
73
74
|
|
74
75
|
def requirement_files
|
@@ -80,7 +81,12 @@ module Dependabot
|
|
80
81
|
end
|
81
82
|
|
82
83
|
def check_required_files_present
|
83
|
-
return if requirements_txt_files.any? ||
|
84
|
+
return if requirements_txt_files.any? ||
|
85
|
+
requirements_in_files.any? ||
|
86
|
+
setup_file ||
|
87
|
+
setup_cfg_file ||
|
88
|
+
pipfile ||
|
89
|
+
pyproject
|
84
90
|
|
85
91
|
path = Pathname.new(File.join(directory, "requirements.txt")).
|
86
92
|
cleanpath.to_path
|
@@ -135,6 +141,10 @@ module Dependabot
|
|
135
141
|
@poetry_lock ||= fetch_file_if_present("poetry.lock")
|
136
142
|
end
|
137
143
|
|
144
|
+
def pdm_lock
|
145
|
+
@pdm_lock ||= fetch_file_if_present("pdm.lock")
|
146
|
+
end
|
147
|
+
|
138
148
|
def requirements_txt_files
|
139
149
|
req_txt_and_in_files.select { |f| f.name.end_with?(".txt") }
|
140
150
|
end
|
@@ -168,7 +178,7 @@ module Dependabot
|
|
168
178
|
repo_contents.
|
169
179
|
select { |f| f.type == "file" }.
|
170
180
|
select { |f| f.name.end_with?(".txt", ".in") }.
|
171
|
-
reject { |f| f.size >
|
181
|
+
reject { |f| f.size > 500_000 }.
|
172
182
|
map { |f| fetch_file_from_host(f.name) }.
|
173
183
|
select { |f| requirements_file?(f) }.
|
174
184
|
each { |f| @req_txt_and_in_files << f }
|
@@ -188,7 +198,7 @@ module Dependabot
|
|
188
198
|
repo_contents(dir: relative_reqs_dir).
|
189
199
|
select { |f| f.type == "file" }.
|
190
200
|
select { |f| f.name.end_with?(".txt", ".in") }.
|
191
|
-
reject { |f| f.size >
|
201
|
+
reject { |f| f.size > 500_000 }.
|
192
202
|
map { |f| fetch_file_from_host("#{relative_reqs_dir}/#{f.name}") }.
|
193
203
|
select { |f| requirements_file?(f) }
|
194
204
|
end
|
@@ -290,8 +300,8 @@ module Dependabot
|
|
290
300
|
fetch_submodules: true
|
291
301
|
).tap { |f| f.support_file = true }
|
292
302
|
rescue Dependabot::DependencyFileNotFound
|
293
|
-
# For
|
294
|
-
# given path instead of a setup.py. We do not require a
|
303
|
+
# For projects with pyproject.toml attempt to fetch a pyproject.toml
|
304
|
+
# at the given path instead of a setup.py. We do not require a
|
295
305
|
# setup.py to be present, so if none can be found, simply return
|
296
306
|
return [] unless allow_pyproject
|
297
307
|
|
@@ -372,7 +382,7 @@ module Dependabot
|
|
372
382
|
return [] unless pipfile
|
373
383
|
|
374
384
|
paths = []
|
375
|
-
|
385
|
+
DEPENDENCY_TYPES.each do |dep_type|
|
376
386
|
next unless parsed_pipfile[dep_type]
|
377
387
|
|
378
388
|
parsed_pipfile[dep_type].each do |_, req|
|
@@ -389,7 +399,7 @@ module Dependabot
|
|
389
399
|
return [] unless pyproject
|
390
400
|
|
391
401
|
paths = []
|
392
|
-
Dependabot::Python::FileParser::
|
402
|
+
Dependabot::Python::FileParser::PyprojectFilesParser::POETRY_DEPENDENCY_TYPES.each do |dep_type|
|
393
403
|
next unless parsed_pyproject.dig("tool", "poetry", dep_type)
|
394
404
|
|
395
405
|
parsed_pyproject.dig("tool", "poetry", dep_type).each do |_, req|
|
@@ -12,7 +12,7 @@ require "dependabot/python/name_normaliser"
|
|
12
12
|
module Dependabot
|
13
13
|
module Python
|
14
14
|
class FileParser
|
15
|
-
class
|
15
|
+
class PyprojectFilesParser
|
16
16
|
POETRY_DEPENDENCY_TYPES = %w(dependencies dev-dependencies).freeze
|
17
17
|
|
18
18
|
# https://python-poetry.org/docs/dependency-specification/
|
@@ -25,7 +25,7 @@ module Dependabot
|
|
25
25
|
def dependency_set
|
26
26
|
dependency_set = Dependabot::FileParsers::Base::DependencySet.new
|
27
27
|
|
28
|
-
dependency_set += pyproject_dependencies
|
28
|
+
dependency_set += pyproject_dependencies if using_poetry? || using_pep621?
|
29
29
|
dependency_set += lockfile_dependencies if lockfile
|
30
30
|
|
31
31
|
dependency_set
|
@@ -36,6 +36,14 @@ module Dependabot
|
|
36
36
|
attr_reader :dependency_files
|
37
37
|
|
38
38
|
def pyproject_dependencies
|
39
|
+
if using_poetry?
|
40
|
+
poetry_dependencies
|
41
|
+
else
|
42
|
+
pep621_dependencies
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def poetry_dependencies
|
39
47
|
dependencies = Dependabot::FileParsers::Base::DependencySet.new
|
40
48
|
|
41
49
|
POETRY_DEPENDENCY_TYPES.each do |type|
|
@@ -59,9 +67,47 @@ module Dependabot
|
|
59
67
|
dependencies
|
60
68
|
end
|
61
69
|
|
70
|
+
def pep621_dependencies
|
71
|
+
dependencies = Dependabot::FileParsers::Base::DependencySet.new
|
72
|
+
|
73
|
+
# PDM is not yet supported, so we want to ignore it for now because in
|
74
|
+
# the current state of things, going on would result in updating
|
75
|
+
# pyproject.toml but leaving pdm.lock out of sync, which is
|
76
|
+
# undesirable. Leave PDM alone until properly supported
|
77
|
+
return dependencies if using_pdm?
|
78
|
+
|
79
|
+
parsed_pep621_dependencies.each do |dep|
|
80
|
+
# If a requirement has a `<` or `<=` marker then updating it is
|
81
|
+
# probably blocked. Ignore it.
|
82
|
+
next if dep["markers"].include?("<")
|
83
|
+
|
84
|
+
# If no requirement, don't add it
|
85
|
+
next if dep["requirement"].empty?
|
86
|
+
|
87
|
+
dependencies <<
|
88
|
+
Dependency.new(
|
89
|
+
name: normalised_name(dep["name"], dep["extras"]),
|
90
|
+
version: dep["version"]&.include?("*") ? nil : dep["version"],
|
91
|
+
requirements: [{
|
92
|
+
requirement: dep["requirement"],
|
93
|
+
file: Pathname.new(dep["file"]).cleanpath.to_path,
|
94
|
+
source: nil,
|
95
|
+
groups: [dep["requirement_type"]]
|
96
|
+
}],
|
97
|
+
package_manager: "pip"
|
98
|
+
)
|
99
|
+
end
|
100
|
+
|
101
|
+
dependencies
|
102
|
+
end
|
103
|
+
|
104
|
+
def normalised_name(name, extras)
|
105
|
+
NameNormaliser.normalise_including_extras(name, extras)
|
106
|
+
end
|
107
|
+
|
62
108
|
# @param req can be an Array, Hash or String that represents the constraints for a dependency
|
63
109
|
def parse_requirements_from(req, type)
|
64
|
-
[req].flatten.compact.
|
110
|
+
[req].flatten.compact.filter_map do |requirement|
|
65
111
|
next if requirement.is_a?(Hash) && (UNSUPPORTED_DEPENDENCY_TYPES & requirement.keys).any?
|
66
112
|
|
67
113
|
check_requirements(requirement)
|
@@ -72,7 +118,19 @@ module Dependabot
|
|
72
118
|
source: nil,
|
73
119
|
groups: [type]
|
74
120
|
}
|
75
|
-
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def using_poetry?
|
125
|
+
!parsed_pyproject.dig("tool", "poetry").nil?
|
126
|
+
end
|
127
|
+
|
128
|
+
def using_pep621?
|
129
|
+
!parsed_pyproject.dig("project", "dependencies").nil?
|
130
|
+
end
|
131
|
+
|
132
|
+
def using_pdm?
|
133
|
+
using_pep621? && pdm_lock
|
76
134
|
end
|
77
135
|
|
78
136
|
# Create a DependencySet where each element has no requirement. Any
|
@@ -81,8 +139,9 @@ module Dependabot
|
|
81
139
|
def lockfile_dependencies
|
82
140
|
dependencies = Dependabot::FileParsers::Base::DependencySet.new
|
83
141
|
|
142
|
+
source_types = %w(directory git url)
|
84
143
|
parsed_lockfile.fetch("package", []).each do |details|
|
85
|
-
next if
|
144
|
+
next if source_types.include?(details.dig("source", "type"))
|
86
145
|
|
87
146
|
dependencies <<
|
88
147
|
Dependency.new(
|
@@ -145,6 +204,24 @@ module Dependabot
|
|
145
204
|
poetry_lock || pyproject_lock
|
146
205
|
end
|
147
206
|
|
207
|
+
def parsed_pep621_dependencies
|
208
|
+
SharedHelpers.in_a_temporary_directory do
|
209
|
+
write_temporary_pyproject
|
210
|
+
|
211
|
+
SharedHelpers.run_helper_subprocess(
|
212
|
+
command: "pyenv exec python #{NativeHelpers.python_helper_path}",
|
213
|
+
function: "parse_pep621_dependencies",
|
214
|
+
args: [pyproject.name]
|
215
|
+
)
|
216
|
+
end
|
217
|
+
end
|
218
|
+
|
219
|
+
def write_temporary_pyproject
|
220
|
+
path = pyproject.name
|
221
|
+
FileUtils.mkdir_p(Pathname.new(path).dirname)
|
222
|
+
File.write(path, pyproject.content)
|
223
|
+
end
|
224
|
+
|
148
225
|
def parsed_lockfile
|
149
226
|
return parsed_poetry_lock if poetry_lock
|
150
227
|
return parsed_pyproject_lock if pyproject_lock
|
@@ -159,6 +236,11 @@ module Dependabot
|
|
159
236
|
@poetry_lock ||=
|
160
237
|
dependency_files.find { |f| f.name == "poetry.lock" }
|
161
238
|
end
|
239
|
+
|
240
|
+
def pdm_lock
|
241
|
+
@pdm_lock ||=
|
242
|
+
dependency_files.find { |f| f.name == "pdm.lock" }
|
243
|
+
end
|
162
244
|
end
|
163
245
|
end
|
164
246
|
end
|
@@ -33,8 +33,7 @@ module Dependabot
|
|
33
33
|
requirement_files.flat_map do |file|
|
34
34
|
file.content.lines.
|
35
35
|
select { |l| l.include?(";") && l.include?("python") }.
|
36
|
-
|
37
|
-
compact.
|
36
|
+
filter_map { |l| l.match(/python_version(?<req>.*?["'].*?['"])/) }.
|
38
37
|
map { |re| re.named_captures.fetch("req").gsub(/['"]/, "") }.
|
39
38
|
select { |r| valid_requirement?(r) }
|
40
39
|
end
|
@@ -12,10 +12,10 @@ module Dependabot
|
|
12
12
|
module Python
|
13
13
|
class FileParser
|
14
14
|
class SetupFileParser
|
15
|
-
INSTALL_REQUIRES_REGEX = /install_requires\s*=\s*\[/m
|
16
|
-
SETUP_REQUIRES_REGEX = /setup_requires\s*=\s*\[/m
|
17
|
-
TESTS_REQUIRE_REGEX = /tests_require\s*=\s*\[/m
|
18
|
-
EXTRAS_REQUIRE_REGEX = /extras_require\s*=\s*\{/m
|
15
|
+
INSTALL_REQUIRES_REGEX = /install_requires\s*=\s*\[/m
|
16
|
+
SETUP_REQUIRES_REGEX = /setup_requires\s*=\s*\[/m
|
17
|
+
TESTS_REQUIRE_REGEX = /tests_require\s*=\s*\[/m
|
18
|
+
EXTRAS_REQUIRE_REGEX = /extras_require\s*=\s*\{/m
|
19
19
|
|
20
20
|
CLOSING_BRACKET = { "[" => "]", "{" => "}" }.freeze
|
21
21
|
|
@@ -128,7 +128,7 @@ module Dependabot
|
|
128
128
|
tests_require = get_regexed_req_array(TESTS_REQUIRE_REGEX)
|
129
129
|
extras_require = get_regexed_req_dict(EXTRAS_REQUIRE_REGEX)
|
130
130
|
|
131
|
-
tmp = "from setuptools import setup\n\n"\
|
131
|
+
tmp = "from setuptools import setup\n\n" \
|
132
132
|
"setup(name=\"sanitized-package\",version=\"0.0.1\","
|
133
133
|
|
134
134
|
tmp += "install_requires=#{install_requires}," if install_requires
|
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "toml-rb"
|
4
3
|
require "dependabot/dependency"
|
5
4
|
require "dependabot/file_parsers"
|
6
5
|
require "dependabot/file_parsers/base"
|
@@ -15,11 +14,9 @@ module Dependabot
|
|
15
14
|
module Python
|
16
15
|
class FileParser < Dependabot::FileParsers::Base
|
17
16
|
require_relative "file_parser/pipfile_files_parser"
|
18
|
-
require_relative "file_parser/
|
17
|
+
require_relative "file_parser/pyproject_files_parser"
|
19
18
|
require_relative "file_parser/setup_file_parser"
|
20
19
|
|
21
|
-
POETRY_DEPENDENCY_TYPES =
|
22
|
-
%w(tool.poetry.dependencies tool.poetry.dev-dependencies).freeze
|
23
20
|
DEPENDENCY_GROUP_KEYS = [
|
24
21
|
{
|
25
22
|
pipfile: "packages",
|
@@ -42,7 +39,7 @@ module Dependabot
|
|
42
39
|
dependency_set = DependencySet.new
|
43
40
|
|
44
41
|
dependency_set += pipenv_dependencies if pipfile
|
45
|
-
dependency_set +=
|
42
|
+
dependency_set += pyproject_file_dependencies if pyproject
|
46
43
|
dependency_set += requirement_dependencies if requirement_files.any?
|
47
44
|
dependency_set += setup_file_dependencies if setup_file || setup_cfg_file
|
48
45
|
|
@@ -62,9 +59,9 @@ module Dependabot
|
|
62
59
|
dependency_set
|
63
60
|
end
|
64
61
|
|
65
|
-
def
|
66
|
-
@
|
67
|
-
|
62
|
+
def pyproject_file_dependencies
|
63
|
+
@pyproject_file_dependencies ||=
|
64
|
+
PyprojectFilesParser.
|
68
65
|
new(dependency_files: dependency_files).
|
69
66
|
dependency_set
|
70
67
|
end
|
@@ -105,18 +102,6 @@ module Dependabot
|
|
105
102
|
end
|
106
103
|
end
|
107
104
|
|
108
|
-
def included_in_pipenv_deps?(dep_name)
|
109
|
-
return false unless pipfile
|
110
|
-
|
111
|
-
pipenv_dependencies.dependencies.map(&:name).include?(dep_name)
|
112
|
-
end
|
113
|
-
|
114
|
-
def included_in_poetry_deps?(dep_name)
|
115
|
-
return false unless using_poetry?
|
116
|
-
|
117
|
-
poetry_dependencies.dependencies.map(&:name).include?(dep_name)
|
118
|
-
end
|
119
|
-
|
120
105
|
def blocking_marker?(dep)
|
121
106
|
return false if dep["markers"] == "None"
|
122
107
|
return true if dep["markers"].include?("<")
|
@@ -215,15 +200,6 @@ module Dependabot
|
|
215
200
|
@pipfile_lock ||= get_original_file("Pipfile.lock")
|
216
201
|
end
|
217
202
|
|
218
|
-
def using_poetry?
|
219
|
-
return false unless pyproject
|
220
|
-
return true if poetry_lock || pyproject_lock
|
221
|
-
|
222
|
-
!TomlRB.parse(pyproject.content).dig("tool", "poetry").nil?
|
223
|
-
rescue TomlRB::ParseError, TomlRB::ValueOverwriteError
|
224
|
-
raise Dependabot::DependencyFileNotParseable, pyproject.path
|
225
|
-
end
|
226
|
-
|
227
203
|
def output_file_regex(filename)
|
228
204
|
"--output-file[=\s]+#{Regexp.escape(filename)}(?:\s|$)"
|
229
205
|
end
|
@@ -7,6 +7,7 @@ require "dependabot/python/file_fetcher"
|
|
7
7
|
require "dependabot/python/file_parser/python_requirement_parser"
|
8
8
|
require "dependabot/python/file_updater"
|
9
9
|
require "dependabot/shared_helpers"
|
10
|
+
require "dependabot/python/helpers"
|
10
11
|
require "dependabot/python/native_helpers"
|
11
12
|
require "dependabot/python/python_versions"
|
12
13
|
require "dependabot/python/name_normaliser"
|
@@ -22,10 +23,9 @@ module Dependabot
|
|
22
23
|
require_relative "setup_file_sanitizer"
|
23
24
|
|
24
25
|
UNSAFE_PACKAGES = %w(setuptools distribute pip).freeze
|
25
|
-
INCOMPATIBLE_VERSIONS_REGEX = /There are incompatible versions in the resolved dependencies:.*\z/m
|
26
|
-
WARNINGS = /\s*# WARNING:.*\Z/m
|
27
|
-
UNSAFE_NOTE =
|
28
|
-
/\s*# The following packages are considered to be unsafe.*\Z/m.freeze
|
26
|
+
INCOMPATIBLE_VERSIONS_REGEX = /There are incompatible versions in the resolved dependencies:.*\z/m
|
27
|
+
WARNINGS = /\s*# WARNING:.*\Z/m
|
28
|
+
UNSAFE_NOTE = /\s*# The following packages are considered to be unsafe.*\Z/m
|
29
29
|
|
30
30
|
attr_reader :dependencies, :dependency_files, :credentials
|
31
31
|
|
@@ -66,33 +66,27 @@ module Dependabot
|
|
66
66
|
def compile_new_requirement_files
|
67
67
|
SharedHelpers.in_a_temporary_directory do
|
68
68
|
write_updated_dependency_files
|
69
|
-
install_required_python
|
69
|
+
Helpers.install_required_python(python_version)
|
70
70
|
|
71
71
|
filenames_to_compile.each do |filename|
|
72
72
|
# Shell out to pip-compile, generate a new set of requirements.
|
73
73
|
# This is slow, as pip-compile needs to do installs.
|
74
|
-
name_part = "pyenv exec pip-compile "\
|
75
|
-
"#{pip_compile_options(filename)} -P "\
|
74
|
+
name_part = "pyenv exec pip-compile " \
|
75
|
+
"#{pip_compile_options(filename)} -P " \
|
76
76
|
"#{dependency.name}"
|
77
77
|
version_part = "#{dependency.version} #{filename}"
|
78
78
|
# Don't escape pyenv `dep-name==version` syntax
|
79
79
|
run_pip_compile_command(
|
80
|
-
"#{SharedHelpers.escape_command(name_part)}=="\
|
80
|
+
"#{SharedHelpers.escape_command(name_part)}==" \
|
81
81
|
"#{SharedHelpers.escape_command(version_part)}",
|
82
82
|
allow_unsafe_shell_command: true
|
83
83
|
)
|
84
|
-
# Run pip-compile a second time, without an update argument, to
|
85
|
-
# ensure it resets the right comments.
|
86
|
-
run_pip_compile_command(
|
87
|
-
"pyenv exec pip-compile #{pip_compile_options(filename)} "\
|
88
|
-
"#{filename}"
|
89
|
-
)
|
90
84
|
end
|
91
85
|
|
92
86
|
# Remove any .python-version file before parsing the reqs
|
93
87
|
FileUtils.remove_entry(".python-version", true)
|
94
88
|
|
95
|
-
dependency_files.
|
89
|
+
dependency_files.filter_map do |file|
|
96
90
|
next unless file.name.end_with?(".txt")
|
97
91
|
|
98
92
|
updated_content = File.read(file.name)
|
@@ -102,12 +96,12 @@ module Dependabot
|
|
102
96
|
next if updated_content == file.content
|
103
97
|
|
104
98
|
file.dup.tap { |f| f.content = updated_content }
|
105
|
-
end
|
99
|
+
end
|
106
100
|
end
|
107
101
|
end
|
108
102
|
|
109
103
|
def update_manifest_files
|
110
|
-
dependency_files.
|
104
|
+
dependency_files.filter_map do |file|
|
111
105
|
next unless file.name.end_with?(".in")
|
112
106
|
|
113
107
|
file = file.dup
|
@@ -116,7 +110,7 @@ module Dependabot
|
|
116
110
|
|
117
111
|
file.content = updated_content
|
118
112
|
file
|
119
|
-
end
|
113
|
+
end
|
120
114
|
end
|
121
115
|
|
122
116
|
def update_uncompiled_files(updated_files)
|
@@ -132,7 +126,7 @@ module Dependabot
|
|
132
126
|
reject { |file| updated_filenames.include?(file.name) }
|
133
127
|
|
134
128
|
args = dependency.to_h
|
135
|
-
args = args.keys.
|
129
|
+
args = args.keys.to_h { |k| [k.to_sym, args[k]] }
|
136
130
|
args[:requirements] = new_reqs
|
137
131
|
args[:previous_requirements] = old_reqs
|
138
132
|
|
@@ -219,15 +213,6 @@ module Dependabot
|
|
219
213
|
end
|
220
214
|
end
|
221
215
|
|
222
|
-
def install_required_python
|
223
|
-
return if run_command("pyenv versions").include?("#{python_version}\n")
|
224
|
-
|
225
|
-
run_command("pyenv install -s #{python_version}")
|
226
|
-
run_command("pyenv exec pip install --upgrade pip")
|
227
|
-
run_command("pyenv exec pip install -r "\
|
228
|
-
"#{NativeHelpers.python_requirements_path}")
|
229
|
-
end
|
230
|
-
|
231
216
|
def sanitized_setup_file_content(file)
|
232
217
|
@sanitized_setup_file_content ||= {}
|
233
218
|
return @sanitized_setup_file_content[file.name] if @sanitized_setup_file_content[file.name]
|
@@ -352,7 +337,7 @@ module Dependabot
|
|
352
337
|
end
|
353
338
|
|
354
339
|
def deps_to_augment_hashes_for(updated_content, original_content)
|
355
|
-
regex = /^#{RequirementParser::INSTALL_REQ_WITH_REQUIREMENT}/
|
340
|
+
regex = /^#{RequirementParser::INSTALL_REQ_WITH_REQUIREMENT}/o
|
356
341
|
|
357
342
|
new_matches = []
|
358
343
|
updated_content.scan(regex) { new_matches << Regexp.last_match }
|
@@ -18,6 +18,8 @@ module Dependabot
|
|
18
18
|
require_relative "pipfile_manifest_updater"
|
19
19
|
require_relative "setup_file_sanitizer"
|
20
20
|
|
21
|
+
DEPENDENCY_TYPES = %w(packages dev-packages).freeze
|
22
|
+
|
21
23
|
attr_reader :dependencies, :dependency_files, :credentials
|
22
24
|
|
23
25
|
def initialize(dependencies:, dependency_files:, credentials:)
|
@@ -145,7 +147,7 @@ module Dependabot
|
|
145
147
|
pipfile_object = TomlRB.parse(pipfile_content)
|
146
148
|
|
147
149
|
dependencies.each do |dep|
|
148
|
-
|
150
|
+
DEPENDENCY_TYPES.each do |type|
|
149
151
|
names = pipfile_object[type]&.keys || []
|
150
152
|
pkg_name = names.find { |nm| normalise(nm) == dep.name }
|
151
153
|
next unless pkg_name || subdep_type?(type)
|
@@ -300,11 +302,7 @@ module Dependabot
|
|
300
302
|
nil
|
301
303
|
end
|
302
304
|
|
303
|
-
|
304
|
-
|
305
|
-
requirements_path = NativeHelpers.python_requirements_path
|
306
|
-
run_command("pyenv install -s #{python_version}")
|
307
|
-
run_command("pyenv exec pip install -r #{requirements_path}")
|
305
|
+
Helpers.install_required_python(python_version)
|
308
306
|
end
|
309
307
|
|
310
308
|
def sanitized_setup_file_content(file)
|
@@ -350,9 +348,9 @@ module Dependabot
|
|
350
348
|
|
351
349
|
# Otherwise we have to raise, giving details of the Python versions
|
352
350
|
# that Dependabot supports
|
353
|
-
msg = "Dependabot detected the following Python requirement "\
|
354
|
-
"for your project: '#{requirement_string}'.\n\nCurrently, the "\
|
355
|
-
"following Python versions are supported in Dependabot: "\
|
351
|
+
msg = "Dependabot detected the following Python requirement " \
|
352
|
+
"for your project: '#{requirement_string}'.\n\nCurrently, the " \
|
353
|
+
"following Python versions are supported in Dependabot: " \
|
356
354
|
"#{PythonVersions::SUPPORTED_VERSIONS.join(', ')}."
|
357
355
|
raise DependencyFileNotResolvable, msg
|
358
356
|
end
|