dependabot-python 0.404.0 → 0.405.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,195 @@
1
+ # typed: strong
2
+ # frozen_string_literal: true
3
+
4
+ require "dependabot/python/file_parser/pyproject_document"
5
+ require "dependabot/python/file_parser/poetry_lock"
6
+ require "dependabot/python/name_normaliser"
7
+ require "dependabot/python/requirement"
8
+
9
+ module Dependabot
10
+ module Python
11
+ class FileParser < Dependabot::FileParsers::Base
12
+ class PyprojectDocument
13
+ class AuthSource
14
+ extend T::Sig
15
+
16
+ sig { params(data: Object).void }
17
+ def initialize(data)
18
+ @data = T.let(PyprojectValueParser.object_hash(data, "Poetry source"), PyprojectValueParser::ObjectHash)
19
+ end
20
+
21
+ sig { returns(T.nilable(String)) }
22
+ def name
23
+ PyprojectValueParser.optional_string(@data["name"], "Poetry source name")
24
+ end
25
+
26
+ sig { returns(T.nilable(String)) }
27
+ def url
28
+ PyprojectValueParser.optional_string(@data["url"], "Poetry source url")
29
+ end
30
+ end
31
+
32
+ sig { returns(String) }
33
+ def to_toml
34
+ T.cast(TomlRB.dump(@data), String)
35
+ end
36
+
37
+ sig { returns(T::Array[AuthSource]) }
38
+ def poetry_auth_sources
39
+ value = poetry_root&.[]("source")
40
+ return [] if value.nil?
41
+
42
+ PyprojectValueParser.array(value, "tool.poetry.source").map { |entry| AuthSource.new(entry) }
43
+ end
44
+
45
+ sig { params(version: String).void }
46
+ def update_python_requirement(version)
47
+ root = mutable_poetry_root
48
+ return unless root
49
+
50
+ dependencies = mutable_section(root, "dependencies", "tool.poetry.dependencies")
51
+ specification = dependencies&.[]("python")
52
+ return unless specification
53
+
54
+ constraint = PyprojectValueParser.string(specification, "tool.poetry.dependencies.python")
55
+ requirements = Requirement.requirements_array(constraint)
56
+ return if requirements.empty? || requirements.any? { |requirement| requirement.satisfied_by?(version) }
57
+
58
+ dependencies["python"] = "~#{version}"
59
+ end
60
+
61
+ sig { params(name: String, requirement: String).void }
62
+ def set_poetry_requirement(name:, requirement:)
63
+ rewrite_poetry_requirement(name, requirement, preserve_enrichment: false)
64
+ end
65
+
66
+ sig { params(name: String, requirement: String).void }
67
+ def pin_poetry_requirement(name:, requirement:)
68
+ rewrite_poetry_requirement(name, requirement, preserve_enrichment: true)
69
+ end
70
+
71
+ sig { params(name: String, requirement: String, production: T::Boolean).void }
72
+ def add_poetry_dependency(name:, requirement:, production:)
73
+ root = mutable_poetry_root
74
+ return unless root
75
+
76
+ type = production ? "dependencies" : "dev-dependencies"
77
+ table = T.must(mutable_section(root, type, "tool.poetry.#{type}", create: true))
78
+ table[name] = requirement
79
+ end
80
+
81
+ sig { params(lock: PoetryLock, excluding: T::Array[String]).void }
82
+ def freeze_poetry_dependencies(lock:, excluding:)
83
+ root = mutable_poetry_root
84
+ return unless root
85
+
86
+ PyprojectFilesParser::POETRY_DEPENDENCY_TYPES.each do |type|
87
+ next unless root[type]
88
+
89
+ table = T.must(mutable_section(root, type, "tool.poetry.#{type}"))
90
+ table.each_key do |name|
91
+ normalized = NameNormaliser.normalise(name)
92
+ next if excluding.include?(normalized)
93
+
94
+ record = lock.record_for(normalized, &:itself)
95
+ freeze_poetry_entry(table, name, record) if record
96
+ end
97
+ end
98
+ end
99
+
100
+ private
101
+
102
+ sig do
103
+ params(
104
+ parent: PyprojectValueParser::ObjectHash, key: String, context: String, create: T::Boolean
105
+ ).returns(T.nilable(PyprojectValueParser::ObjectHash))
106
+ end
107
+ def mutable_section(parent, key, context, create: false)
108
+ value = parent[key]
109
+ return if value.nil? && !create
110
+
111
+ table = value.nil? ? {} : PyprojectValueParser.object_hash(value, context)
112
+ parent[key] = table
113
+ table
114
+ end
115
+
116
+ sig { returns(T.nilable(PyprojectValueParser::ObjectHash)) }
117
+ def mutable_poetry_root
118
+ tool = mutable_section(@data, "tool", "tool")
119
+ tool && mutable_section(tool, "poetry", "tool.poetry")
120
+ end
121
+
122
+ sig do
123
+ params(_block: T.proc.params(table: PyprojectValueParser::ObjectHash).void).void
124
+ end
125
+ def each_poetry_table(&_block)
126
+ root = mutable_poetry_root
127
+ return unless root
128
+
129
+ PyprojectFilesParser::POETRY_DEPENDENCY_TYPES.each do |type|
130
+ table = mutable_section(root, type, "tool.poetry.#{type}")
131
+ yield(table) if table
132
+ end
133
+ groups = mutable_section(root, "group", "tool.poetry.group")
134
+ groups&.each_key do |name|
135
+ group = T.must(mutable_section(groups, name, "tool.poetry.group.#{name}"))
136
+ table = mutable_section(group, "dependencies", "tool.poetry.group.#{name}.dependencies")
137
+ yield(table) if table
138
+ end
139
+ end
140
+
141
+ sig { params(name: String, requirement: String, preserve_enrichment: T::Boolean).void }
142
+ def rewrite_poetry_requirement(name, requirement, preserve_enrichment:)
143
+ each_poetry_table do |table|
144
+ key = table.keys.find { |entry| NameNormaliser.normalise(entry) == name }
145
+ next unless key
146
+
147
+ value = table[key]
148
+ if value.is_a?(Hash)
149
+ entry = PyprojectValueParser.object_hash(value, "Poetry dependency #{key}")
150
+ next if preserve_enrichment && !entry.key?("version")
151
+
152
+ entry["version"] = requirement
153
+ table[key] = entry
154
+ else
155
+ table[key] = requirement
156
+ end
157
+ end
158
+ end
159
+
160
+ sig { params(table: PyprojectValueParser::ObjectHash, name: String, record: PoetryLock::Record).void }
161
+ def freeze_poetry_entry(table, name, record)
162
+ version = record.version
163
+ source_type = record.source_type
164
+ return if %w(directory file url).include?(source_type)
165
+
166
+ value = table[name]
167
+ if source_type == "git"
168
+ table[name] = frozen_git_declaration(value, name, record.git_source)
169
+ elsif value.is_a?(Hash)
170
+ entry = PyprojectValueParser.object_hash(value, "Poetry dependency #{name}")
171
+ entry["version"] = version
172
+ table[name] = entry
173
+ elsif !value.is_a?(Array)
174
+ table[name] = version
175
+ end
176
+ end
177
+
178
+ sig do
179
+ params(value: Object, name: String, source: PoetryLock::GitSource).returns(PyprojectValueParser::ObjectHash)
180
+ end
181
+ def frozen_git_declaration(value, name, source)
182
+ declaration = T.let({ "git" => source.url, "rev" => source.reference }, PyprojectValueParser::ObjectHash)
183
+ declaration["subdirectory"] = source.subdirectory if source.subdirectory
184
+ if value.is_a?(Hash)
185
+ extras = PyprojectValueParser.object_hash(value, "Poetry dependency #{name}")["extras"]
186
+ if extras
187
+ declaration["extras"] = PyprojectValueParser.string_array(extras, "Poetry dependency #{name} extras")
188
+ end
189
+ end
190
+ declaration
191
+ end
192
+ end
193
+ end
194
+ end
195
+ end
@@ -90,6 +90,9 @@ module Dependabot
90
90
  class PyprojectDocument
91
91
  extend T::Sig
92
92
 
93
+ require_relative "pyproject_document/poetry_mutations"
94
+ require_relative "pyproject_document/pep621_mutations"
95
+
93
96
  PoetryRequirement = T.type_alias { PyprojectValueParser::PoetryRequirement }
94
97
  PoetryDependencyMap = T.type_alias { PyprojectValueParser::PoetryDependencyMap }
95
98
 
@@ -16,6 +16,7 @@ require "dependabot/python/pip_compile_file_matcher"
16
16
  require "dependabot/python/language_version_manager"
17
17
  require "dependabot/python/package_manager"
18
18
  require "dependabot/python/marker_evaluator"
19
+ require "dependabot/python/requirement_references"
19
20
 
20
21
  module Dependabot
21
22
  module Python
@@ -293,14 +294,16 @@ module Dependabot
293
294
  sig { returns(DependencySet) }
294
295
  def requirement_dependencies
295
296
  dependencies = DependencySet.new
296
- parsed_requirement_files.each do |dep|
297
- # If a requirement has a `<`, `<=` or '==' marker then updating it is
298
- # probably blocked. Ignore it.
299
- next if blocking_marker?(dep)
297
+ # If a requirement has a `<`, `<=` or '==' marker then updating it is
298
+ # probably blocked. Ignore it.
299
+ unblocked = parsed_requirement_files.reject { |dep| blocking_marker?(dep) }
300
+ entries, constraints = split_constraints(unblocked)
301
+ pins = constraint_pins(constraints)
300
302
 
303
+ entries.each do |dep|
301
304
  name = dep.name
302
305
  file = dep.file
303
- version = dep.version
306
+ version = dep.version || pins[NameNormaliser.normalise(name)]
304
307
  original_file = get_original_file(file)
305
308
 
306
309
  requirements =
@@ -329,6 +332,51 @@ module Dependabot
329
332
  dependencies
330
333
  end
331
334
 
335
+ # Without roots every requirements file is parsed as its own manifest, so `-c` entries are kept as
336
+ # requirements of the constraints file. From roots, they follow pip and only pin versions.
337
+ sig do
338
+ params(parsed: T::Array[PepDependency])
339
+ .returns([T::Array[PepDependency], T::Array[PepDependency]])
340
+ end
341
+ def split_constraints(parsed)
342
+ return [parsed, []] unless requirement_roots
343
+
344
+ parsed.partition { |dep| !dep.constraint }
345
+ end
346
+
347
+ sig { params(constraints: T::Array[PepDependency]).returns(T::Hash[String, String]) }
348
+ def constraint_pins(constraints)
349
+ constraints.each_with_object({}) do |dep, pins|
350
+ version = dep.version
351
+ pins[NameNormaliser.normalise(dep.name)] ||= version if version
352
+ end
353
+ end
354
+
355
+ # Updates need every file parsed as a manifest so they can update pins in constraints files, so parsing
356
+ # from roots is opt-in. A root is any requirements file not pulled in by another via `-r`/`-c`, as pip
357
+ # would only read those through the reference. A reference loop leaves files no root reaches; parse
358
+ # everything so pip reports it.
359
+ sig { returns(T.nilable(T::Array[String])) }
360
+ def requirement_roots
361
+ return unless options[:requirement_roots]
362
+
363
+ references = requirement_files.to_h do |file|
364
+ [Pathname.new(file.name).cleanpath.to_path, RequirementReferences.paths(file)]
365
+ end
366
+ roots = references.keys - references.values.flatten
367
+ # A root pip-compile input may constrain itself to its own output, which is still installed directly.
368
+ roots |= roots.filter_map { |name| name.sub(/\.in\z/, ".txt") if name.end_with?(".in") } & references.keys
369
+ roots if reaches_every_file?(roots, references)
370
+ end
371
+
372
+ sig { params(roots: T::Array[String], references: T::Hash[String, T::Array[String]]).returns(T::Boolean) }
373
+ def reaches_every_file?(roots, references)
374
+ reached = T.let(Set.new(roots), T::Set[String])
375
+ queue = roots.dup
376
+ references.fetch(T.must(queue.shift), []).each { |path| queue << path if reached.add?(path) } until queue.empty?
377
+ references.keys.all? { |name| reached.include?(name) }
378
+ end
379
+
332
380
  sig { params(name: T.nilable(String), version: T.nilable(String)).returns(T::Boolean) }
333
381
  def old_pyyaml?(name, version)
334
382
  major_version = version&.split(".")&.first
@@ -364,7 +412,10 @@ module Dependabot
364
412
 
365
413
  sig { returns(MarkerEvaluator) }
366
414
  def marker_evaluator
367
- @marker_evaluator ||= T.let(MarkerEvaluator.new, T.nilable(MarkerEvaluator))
415
+ @marker_evaluator ||= T.let(
416
+ MarkerEvaluator.new(non_python_markers_satisfied: true),
417
+ T.nilable(MarkerEvaluator)
418
+ )
368
419
  end
369
420
 
370
421
  sig { returns(DependencySet) }
@@ -384,7 +435,7 @@ module Dependabot
384
435
  result = SharedHelpers.run_helper_subprocess(
385
436
  command: "pyenv exec python3 #{NativeHelpers.python_helper_path}",
386
437
  function: "parse_requirements",
387
- args: [Dir.pwd]
438
+ args: [Dir.pwd, requirement_roots].compact
388
439
  )
389
440
 
390
441
  requirements = PepDependency.from_requirements_helper_result(result)
@@ -1,8 +1,7 @@
1
- # typed: strict
1
+ # typed: strong
2
2
  # frozen_string_literal: true
3
3
 
4
4
  require "sorbet-runtime"
5
- require "toml-rb"
6
5
  require "open3"
7
6
  require "dependabot/dependency"
8
7
  require "dependabot/shared_helpers"
@@ -12,6 +11,7 @@ require "dependabot/python/requirement"
12
11
  require "dependabot/python/file_parser/python_requirement_parser"
13
12
  require "dependabot/python/file_updater"
14
13
  require "dependabot/python/file_parser/pyproject_document"
14
+ require "dependabot/python/file_parser/poetry_lock"
15
15
  require "dependabot/python/native_helpers"
16
16
  require "dependabot/python/name_normaliser"
17
17
  require "dependabot/python/poetry_plugin_installer"
@@ -20,18 +20,7 @@ require "dependabot/package/release_cooldown_options"
20
20
 
21
21
  module Dependabot
22
22
  module Python
23
- PoetryPyprojectHashResult = T.type_alias do
24
- T.nilable(
25
- T.any(
26
- T::Hash[String, T.untyped],
27
- String,
28
- T::Array[T::Hash[String, T.untyped]]
29
- )
30
- )
31
- end
32
-
33
23
  class FileUpdater
34
- # rubocop:disable-next Metrics/ClassLength
35
24
  class PoetryFileUpdater
36
25
  require_relative "pyproject_preparer"
37
26
  require_relative "poetry_file_updater/pep621_updater"
@@ -200,7 +189,8 @@ module Dependabot
200
189
  begin
201
190
  new_lockfile = updated_lockfile_content_for(prepared_pyproject)
202
191
 
203
- original_locked_python = TomlRB.parse(T.must(lockfile).content)["metadata"]["python-versions"]
192
+ original_locked_python =
193
+ FileParser::PoetryLock.from_content(T.must(T.must(lockfile).content)).python_versions
204
194
 
205
195
  new_lockfile.gsub!(/\[metadata\](?:\r?\n).*python-versions[^\r\n]+(?:\r?\n)/m) do |match|
206
196
  # Detect the line ending style from the match (CRLF or LF)
@@ -208,11 +198,10 @@ module Dependabot
208
198
  match.gsub(/(["']).*\1(?:\r?\n)\Z/, '\1' + original_locked_python + '\1' + line_ending)
209
199
  end
210
200
 
211
- tmp_hash =
212
- TomlRB.parse(new_lockfile)["metadata"]["content-hash"]
213
- correct_hash = pyproject_hash_for(updated_pyproject_content.to_s)
201
+ tmp_hash = FileParser::PoetryLock.from_content(new_lockfile).content_hash
202
+ correct_hash = pyproject_hash_for(T.must(updated_pyproject_content))
214
203
 
215
- new_lockfile.gsub(tmp_hash, T.must(correct_hash).to_s)
204
+ new_lockfile.gsub(tmp_hash, correct_hash)
216
205
  end
217
206
  end
218
207
 
@@ -237,42 +226,53 @@ module Dependabot
237
226
 
238
227
  sig { params(pyproject_content: String).returns(String) }
239
228
  def freeze_dependencies_being_updated(pyproject_content)
240
- pyproject_object = TomlRB.parse(pyproject_content)
229
+ document = FileParser::PyprojectDocument.from_content(pyproject_content)
241
230
 
242
- poetry_object = pyproject_object.dig("tool", "poetry")
231
+ if document.poetry?
232
+ pin_updated_poetry_dependencies(document)
233
+ elsif document.poetry_build_backend?
234
+ pin_pep621_subdependencies!(document)
235
+ end
243
236
 
244
- if poetry_object
245
- dependencies.each do |dep|
246
- # Skip Git dependencies - they use tags/refs, not versions
247
- next if git_dependency_being_updated?(dep)
237
+ versions = T.let({}, T::Hash[String, String])
238
+ dependencies.each do |dep|
239
+ next if git_dependency_being_updated?(dep)
248
240
 
249
- if dep.requirements.find { |r| r.file == pyproject&.name }
250
- lock_declaration_to_new_version!(poetry_object, dep)
251
- else
252
- create_declaration_at_new_version!(poetry_object, dep)
253
- end
254
- end
255
- elsif FileParser::PyprojectDocument.from_content(pyproject_content).poetry_build_backend?
256
- pin_pep621_subdependencies!(pyproject_object)
241
+ version = dep.version
242
+ versions[NameNormaliser.normalise(dep.name)] = version if version
257
243
  end
244
+ document.freeze_pep621_versions(versions)
258
245
 
259
- # Freeze PEP 621 project.dependencies and project.optional-dependencies
260
- PyprojectPreparer.freeze_pep621_deps!(pyproject_object, dependencies) do |dep|
261
- !git_dependency_being_updated?(dep)
262
- end
246
+ document.to_toml
247
+ end
263
248
 
264
- TomlRB.dump(pyproject_object)
249
+ sig { params(document: FileParser::PyprojectDocument).void }
250
+ def pin_updated_poetry_dependencies(document)
251
+ dependencies.each do |dep|
252
+ next if git_dependency_being_updated?(dep)
253
+
254
+ pinned_version = exact_poetry_requirement(dep.version)
255
+ next unless pinned_version
256
+
257
+ if dep.requirements.find { |r| r.file == pyproject&.name }
258
+ document.pin_poetry_requirement(name: dep.name, requirement: pinned_version)
259
+ else
260
+ document.add_poetry_dependency(
261
+ name: dep.name, requirement: pinned_version, production: dep.production?
262
+ )
263
+ end
264
+ end
265
265
  end
266
266
 
267
- # freeze_pep621_deps! skips dependency groups and a sub-dependency has no declaration, so without
267
+ # Version freezing skips dependency groups and a sub-dependency has no declaration, so without
268
268
  # this Poetry could lock either to a newer release than the one selected.
269
- sig { params(pyproject_object: T::Hash[String, T.untyped]).void }
270
- def pin_pep621_subdependencies!(pyproject_object)
269
+ sig { params(document: FileParser::PyprojectDocument).void }
270
+ def pin_pep621_subdependencies!(document)
271
271
  dependencies.each do |dep|
272
272
  next if git_dependency_being_updated?(dep)
273
273
  next unless dep.version
274
274
 
275
- PoetrySubdependencyPin.new(lockfile, dep.name).set_requirement!(pyproject_object, "==#{dep.version}")
275
+ PoetrySubdependencyPin.new(lockfile, dep.name).set_requirement!(document, "==#{dep.version}")
276
276
  rescue PoetrySubdependencyPin::ScopeNotReproducible
277
277
  # The update checker declines these; skipping keeps a direct run from promoting the package.
278
278
  next
@@ -285,34 +285,6 @@ module Dependabot
285
285
  .update_python_requirement(language_version_manager.python_version)
286
286
  end
287
287
 
288
- sig { params(poetry_object: T::Hash[String, T.untyped], dep: Dependabot::Dependency).void }
289
- def lock_declaration_to_new_version!(poetry_object, dep)
290
- pinned_version = exact_poetry_requirement(dep.version)
291
- return unless pinned_version
292
-
293
- poetry_dependency_tables(poetry_object).each do |deps_hash|
294
- pkg_name = deps_hash.keys.find { |nm| normalise(nm) == dep.name }
295
- next unless pkg_name
296
-
297
- entry = deps_hash[pkg_name]
298
- if entry.is_a?(Hash)
299
- entry["version"] = pinned_version if entry.key?("version") # skip enrichment-only entries
300
- else
301
- deps_hash[pkg_name] = pinned_version
302
- end
303
- end
304
- end
305
-
306
- sig { params(poetry_object: T::Hash[String, T.untyped], dep: Dependabot::Dependency).void }
307
- def create_declaration_at_new_version!(poetry_object, dep)
308
- pinned_version = exact_poetry_requirement(dep.version)
309
- return unless pinned_version
310
-
311
- subdep_type = dep.production? ? "dependencies" : "dev-dependencies"
312
- poetry_object[subdep_type] ||= {}
313
- poetry_object[subdep_type][dep.name] = pinned_version
314
- end
315
-
316
288
  sig { params(version: T.nilable(String)).returns(T.nilable(String)) }
317
289
  def exact_poetry_requirement(version)
318
290
  return nil unless version
@@ -321,15 +293,6 @@ module Dependabot
321
293
  @cooldown ? "==#{version}" : version
322
294
  end
323
295
 
324
- sig { params(poetry_object: T::Hash[String, T.untyped]).returns(T::Array[T::Hash[String, T.untyped]]) }
325
- def poetry_dependency_tables(poetry_object)
326
- types = Dependabot::Python::FileParser::PyprojectFilesParser::POETRY_DEPENDENCY_TYPES
327
- groups = poetry_object["group"].is_a?(Hash) ? poetry_object["group"].values : []
328
- candidates = types.map { |type| poetry_object[type] } +
329
- groups.map { |group_spec| group_spec["dependencies"] if group_spec.is_a?(Hash) }
330
- candidates.grep(Hash)
331
- end
332
-
333
296
  sig { params(dep: Dependabot::Dependency).returns(T::Boolean) }
334
297
  def git_dependency_being_updated?(dep)
335
298
  dep.requirements.any? { |r| r.source_string("type") == "git" }
@@ -375,7 +338,7 @@ module Dependabot
375
338
  SharedHelpers.run_shell_command(command, fingerprint: fingerprint)
376
339
  end
377
340
 
378
- sig { params(pyproject_content: Object).returns(Integer) }
341
+ sig { params(pyproject_content: String).returns(Integer) }
379
342
  def write_temporary_dependency_files(pyproject_content)
380
343
  dependency_files.each do |file|
381
344
  path = file.name
@@ -397,7 +360,7 @@ module Dependabot
397
360
  .add_auth_env_vars(credentials)
398
361
  end
399
362
 
400
- sig { params(pyproject_content: String).returns(PoetryPyprojectHashResult) }
363
+ sig { params(pyproject_content: String).returns(String) }
401
364
  def pyproject_hash_for(pyproject_content)
402
365
  SharedHelpers.in_a_temporary_directory do |dir|
403
366
  SharedHelpers.with_git_configured(credentials: credentials) do
@@ -408,7 +371,9 @@ module Dependabot
408
371
  function: "get_pyproject_hash",
409
372
  args: [T.cast(dir, Pathname).to_s]
410
373
  )
411
- T.cast(result, PoetryPyprojectHashResult)
374
+ raise TypeError, "Poetry content hash must be a string" unless result.is_a?(String)
375
+
376
+ result
412
377
  end
413
378
  end
414
379
  end
@@ -451,11 +416,6 @@ module Dependabot
451
416
  updated_file
452
417
  end
453
418
 
454
- sig { params(name: String).returns(String) }
455
- def normalise(name)
456
- NameNormaliser.normalise(name)
457
- end
458
-
459
419
  sig { returns(FileParser::PythonRequirementParser) }
460
420
  def python_requirement_parser
461
421
  @python_requirement_parser ||=