dependabot-core 0.78.0 → 0.79.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +5 -0
  3. data/helpers/npm/lib/updater.js +11 -5
  4. data/helpers/npm/package.json +2 -2
  5. data/helpers/npm/yarn.lock +26 -28
  6. data/helpers/yarn/lib/replace-lockfile-declaration.js +15 -3
  7. data/helpers/yarn/lib/updater.js +17 -5
  8. data/helpers/yarn/package.json +2 -2
  9. data/helpers/yarn/yarn.lock +24 -31
  10. data/lib/dependabot/file_fetchers.rb +0 -2
  11. data/lib/dependabot/file_parsers.rb +0 -2
  12. data/lib/dependabot/file_updaters.rb +0 -2
  13. data/lib/dependabot/metadata_finders.rb +0 -2
  14. data/lib/dependabot/update_checkers.rb +0 -2
  15. data/lib/dependabot/utils.rb +0 -4
  16. data/lib/dependabot/version.rb +1 -1
  17. metadata +3 -34
  18. data/helpers/python/lib/__init__.py +0 -0
  19. data/helpers/python/lib/hasher.py +0 -23
  20. data/helpers/python/lib/parser.py +0 -130
  21. data/helpers/python/requirements.txt +0 -9
  22. data/helpers/python/run.py +0 -18
  23. data/lib/dependabot/file_fetchers/python/pip.rb +0 -305
  24. data/lib/dependabot/file_parsers/python/pip.rb +0 -223
  25. data/lib/dependabot/file_parsers/python/pip/pipfile_files_parser.rb +0 -154
  26. data/lib/dependabot/file_parsers/python/pip/poetry_files_parser.rb +0 -141
  27. data/lib/dependabot/file_parsers/python/pip/setup_file_parser.rb +0 -164
  28. data/lib/dependabot/file_updaters/python/pip.rb +0 -147
  29. data/lib/dependabot/file_updaters/python/pip/pip_compile_file_updater.rb +0 -363
  30. data/lib/dependabot/file_updaters/python/pip/pipfile_file_updater.rb +0 -397
  31. data/lib/dependabot/file_updaters/python/pip/pipfile_preparer.rb +0 -125
  32. data/lib/dependabot/file_updaters/python/pip/poetry_file_updater.rb +0 -289
  33. data/lib/dependabot/file_updaters/python/pip/pyproject_preparer.rb +0 -105
  34. data/lib/dependabot/file_updaters/python/pip/requirement_file_updater.rb +0 -166
  35. data/lib/dependabot/file_updaters/python/pip/requirement_replacer.rb +0 -95
  36. data/lib/dependabot/file_updaters/python/pip/setup_file_sanitizer.rb +0 -91
  37. data/lib/dependabot/file_updaters/ruby/.DS_Store +0 -0
  38. data/lib/dependabot/metadata_finders/python/pip.rb +0 -120
  39. data/lib/dependabot/update_checkers/python/pip.rb +0 -227
  40. data/lib/dependabot/update_checkers/python/pip/latest_version_finder.rb +0 -252
  41. data/lib/dependabot/update_checkers/python/pip/pip_compile_version_resolver.rb +0 -380
  42. data/lib/dependabot/update_checkers/python/pip/pipfile_version_resolver.rb +0 -559
  43. data/lib/dependabot/update_checkers/python/pip/poetry_version_resolver.rb +0 -300
  44. data/lib/dependabot/update_checkers/python/pip/requirements_updater.rb +0 -367
  45. data/lib/dependabot/utils/python/requirement.rb +0 -130
  46. data/lib/dependabot/utils/python/version.rb +0 -88
  47. data/lib/python_requirement_parser.rb +0 -33
  48. data/lib/python_versions.rb +0 -21
@@ -1,559 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "excon"
4
- require "toml-rb"
5
-
6
- require "dependabot/file_parsers/python/pip"
7
- require "dependabot/file_updaters/python/pip/pipfile_preparer"
8
- require "dependabot/file_updaters/python/pip/setup_file_sanitizer"
9
- require "dependabot/update_checkers/python/pip"
10
- require "dependabot/shared_helpers"
11
- require "dependabot/utils/python/version"
12
- require "dependabot/errors"
13
-
14
- # rubocop:disable Metrics/ClassLength
15
- module Dependabot
16
- module UpdateCheckers
17
- module Python
18
- class Pip
19
- # This class does version resolution for Pipfiles. Its current approach
20
- # is somewhat crude:
21
- # - Unlock the dependency we're checking in the Pipfile
22
- # - Freeze all of the other dependencies in the Pipfile
23
- # - Run `pipenv lock` and see what the result is
24
- #
25
- # Unfortunately, Pipenv doesn't resolve how we'd expect - it appears to
26
- # just raise if the latest version can't be resolved. Knowing that is
27
- # still better than nothing, though.
28
- class PipfileVersionResolver
29
- VERSION_REGEX = /[0-9]+(?:\.[A-Za-z0-9\-_]+)*/.freeze
30
- GIT_DEPENDENCY_UNREACHABLE_REGEX =
31
- /Command "git clone -q (?<url>[^\s]+).*" failed/.freeze
32
- GIT_REFERENCE_NOT_FOUND_REGEX =
33
- %r{"git checkout -q (?<tag>[^"]+)" .*/(?<name>.*?)(\\n'\]|$)}.
34
- freeze
35
- UNSUPPORTED_DEPS = %w(pyobjc).freeze
36
- UNSUPPORTED_DEP_REGEX =
37
- /"python setup\.py egg_info".*(?:#{UNSUPPORTED_DEPS.join("|")})/.
38
- freeze
39
-
40
- attr_reader :dependency, :dependency_files, :credentials
41
-
42
- def initialize(dependency:, dependency_files:, credentials:,
43
- unlock_requirement:, latest_allowable_version:)
44
- @dependency = dependency
45
- @dependency_files = dependency_files
46
- @credentials = credentials
47
- @latest_allowable_version = latest_allowable_version
48
- @unlock_requirement = unlock_requirement
49
-
50
- check_private_sources_are_reachable
51
- end
52
-
53
- def latest_resolvable_version
54
- return @latest_resolvable_version if @resolution_already_attempted
55
-
56
- @resolution_already_attempted = true
57
- @latest_resolvable_version ||= fetch_latest_resolvable_version
58
- end
59
-
60
- private
61
-
62
- attr_reader :latest_allowable_version
63
-
64
- def unlock_requirement?
65
- @unlock_requirement
66
- end
67
-
68
- def fetch_latest_resolvable_version
69
- @latest_resolvable_version_string ||=
70
- SharedHelpers.in_a_temporary_directory do
71
- SharedHelpers.with_git_configured(credentials: credentials) do
72
- write_temporary_dependency_files
73
-
74
- # Shell out to Pipenv, which handles everything for us.
75
- # Whilst calling `lock` avoids doing an install as part of the
76
- # pipenv flow, an install is still done by pip-tools in order
77
- # to resolve the dependencies. That means this is slow.
78
- run_pipenv_command(
79
- pipenv_environment_variables + "pyenv exec pipenv lock"
80
- )
81
-
82
- updated_lockfile = JSON.parse(File.read("Pipfile.lock"))
83
-
84
- fetch_version_from_parsed_lockfile(updated_lockfile)
85
- end
86
- rescue SharedHelpers::HelperSubprocessFailed => error
87
- handle_pipenv_errors(error)
88
- end
89
- return unless @latest_resolvable_version_string
90
-
91
- Utils::Python::Version.new(@latest_resolvable_version_string)
92
- end
93
-
94
- def fetch_version_from_parsed_lockfile(updated_lockfile)
95
- if dependency.requirements.any?
96
- group = dependency.requirements.first[:groups].first
97
- deps = updated_lockfile[group] || {}
98
-
99
- version =
100
- deps.transform_keys { |k| normalise(k) }.
101
- dig(dependency.name, "version")&.
102
- gsub(/^==/, "")
103
-
104
- return version
105
- end
106
-
107
- FileParsers::Python::Pip::DEPENDENCY_GROUP_KEYS.each do |keys|
108
- deps = updated_lockfile[keys.fetch(:lockfile)] || {}
109
- version =
110
- deps.transform_keys { |k| normalise(k) }.
111
- dig(dependency.name, "version")&.
112
- gsub(/^==/, "")
113
-
114
- return version if version
115
- end
116
-
117
- # If the sub-dependency no longer appears in the lockfile return nil
118
- nil
119
- end
120
-
121
- # rubocop:disable Metrics/CyclomaticComplexity
122
- # rubocop:disable Metrics/PerceivedComplexity
123
- # rubocop:disable Metrics/AbcSize
124
- # rubocop:disable Metrics/MethodLength
125
- def handle_pipenv_errors(error)
126
- if error.message.include?("no version found at all") ||
127
- error.message.include?("Invalid specifier:") ||
128
- error.message.include?("Max retries exceeded")
129
- msg = clean_error_message(error.message)
130
- raise if msg.empty?
131
-
132
- raise DependencyFileNotResolvable, msg
133
- end
134
-
135
- if error.message.match?(UNSUPPORTED_DEP_REGEX)
136
- msg = "Dependabot detected a dependency that can't be built on "\
137
- "linux. Currently, all Dependabot builds happen on linux "\
138
- "boxes, so there is no way for Dependabot to resolve your "\
139
- "dependency files.\n\n"\
140
- "Unless you think Dependabot has made a mistake (please "\
141
- "tag us if so) you may wish to disable Dependabot on this "\
142
- "repo."
143
- raise DependencyFileNotResolvable, msg
144
- end
145
-
146
- if error.message.include?("Could not find a version") ||
147
- error.message.include?("is not a python version")
148
- check_original_requirements_resolvable
149
- end
150
-
151
- if error.message.include?('Command "python setup.py egg_info"') &&
152
- error.message.include?(dependency.name)
153
- # The latest version of the dependency we're updating is borked
154
- # (because it has an unevaluatable setup.py). Skip the update.
155
- return nil
156
- end
157
-
158
- if error.message.match?(GIT_DEPENDENCY_UNREACHABLE_REGEX)
159
- url = error.message.match(GIT_DEPENDENCY_UNREACHABLE_REGEX).
160
- named_captures.fetch("url")
161
- raise GitDependenciesNotReachable, url
162
- end
163
-
164
- if error.message.match?(GIT_REFERENCE_NOT_FOUND_REGEX)
165
- name = error.message.match(GIT_REFERENCE_NOT_FOUND_REGEX).
166
- named_captures.fetch("name")
167
- raise GitDependencyReferenceNotFound, name
168
- end
169
-
170
- raise unless error.message.include?("could not be resolved")
171
- end
172
- # rubocop:enable Metrics/CyclomaticComplexity
173
- # rubocop:enable Metrics/PerceivedComplexity
174
- # rubocop:enable Metrics/AbcSize
175
- # rubocop:enable Metrics/MethodLength
176
-
177
- # Needed because Pipenv's resolver isn't perfect.
178
- # Note: We raise errors from this method, rather than returning a
179
- # boolean, so that all deps for this repo will raise identical
180
- # errors when failing to update
181
- def check_original_requirements_resolvable
182
- SharedHelpers.in_a_temporary_directory do
183
- SharedHelpers.with_git_configured(credentials: credentials) do
184
- write_temporary_dependency_files(update_pipfile: false)
185
-
186
- run_pipenv_command(
187
- pipenv_environment_variables + "pyenv exec pipenv lock"
188
- )
189
-
190
- true
191
- rescue SharedHelpers::HelperSubprocessFailed => error
192
- if error.message.include?("Could not find a version")
193
- msg = clean_error_message(error.message)
194
- msg.gsub!(/\s+\(from .*$/, "")
195
- raise if msg.empty?
196
-
197
- raise DependencyFileNotResolvable, msg
198
- end
199
-
200
- if error.message.include?("is not a python version")
201
- msg = "Pipenv does not support specifying Python ranges "\
202
- "(see https://github.com/pypa/pipenv/issues/1050 for more "\
203
- "details)."
204
- raise DependencyFileNotResolvable, msg
205
- end
206
-
207
- raise
208
- end
209
- end
210
- end
211
-
212
- def clean_error_message(message)
213
- # Pipenv outputs a lot of things to STDERR, so we need to clean
214
- # up the error message
215
- msg_lines = message.lines
216
- msg = msg_lines.
217
- take_while { |l| !l.start_with?("During handling of") }.
218
- drop_while do |l|
219
- next false if l.start_with?("CRITICAL:")
220
- next false if l.start_with?("ERROR:")
221
- next false if l.start_with?("packaging.specifiers")
222
- next false if l.include?("Max retries exceeded")
223
-
224
- true
225
- end.join.strip
226
-
227
- # We also need to redact any URLs, as they may include credentials
228
- msg.gsub(/http.*?(?=\s)/, "<redacted>")
229
- end
230
-
231
- def write_temporary_dependency_files(update_pipfile: true)
232
- dependency_files.each do |file|
233
- next if file.name == ".python-version"
234
-
235
- path = file.name
236
- FileUtils.mkdir_p(Pathname.new(path).dirname)
237
- File.write(path, file.content)
238
- end
239
-
240
- setup_files.each do |file|
241
- path = file.name
242
- FileUtils.mkdir_p(Pathname.new(path).dirname)
243
- File.write(path, sanitized_setup_file_content(file))
244
- end
245
-
246
- setup_cfg_files.each do |file|
247
- path = file.name
248
- FileUtils.mkdir_p(Pathname.new(path).dirname)
249
- File.write(path, "[metadata]\nname = sanitized-package\n")
250
- end
251
-
252
- # Overwrite the pipfile with updated content
253
- File.write(
254
- "Pipfile",
255
- pipfile_content(update_pipfile: update_pipfile)
256
- )
257
- end
258
-
259
- def sanitized_setup_file_content(file)
260
- @sanitized_setup_file_content ||= {}
261
- @sanitized_setup_file_content[file.name] ||=
262
- FileUpdaters::Python::Pip::SetupFileSanitizer.
263
- new(setup_file: file, setup_cfg: setup_cfg(file)).
264
- sanitized_content
265
- end
266
-
267
- def setup_cfg(file)
268
- config_name = file.name.sub(/\.py$/, ".cfg")
269
- dependency_files.find { |f| f.name == config_name }
270
- end
271
-
272
- def pipfile_content(update_pipfile: true)
273
- content = pipfile.content
274
- return content unless update_pipfile
275
-
276
- content = freeze_other_dependencies(content)
277
- content = unlock_target_dependency(content) if unlock_requirement?
278
- content = add_private_sources(content)
279
- content
280
- end
281
-
282
- def freeze_other_dependencies(pipfile_content)
283
- FileUpdaters::Python::Pip::PipfilePreparer.
284
- new(pipfile_content: pipfile_content).
285
- freeze_top_level_dependencies_except([dependency], lockfile)
286
- end
287
-
288
- def unlock_target_dependency(pipfile_content)
289
- pipfile_object = TomlRB.parse(pipfile_content)
290
-
291
- %w(packages dev-packages).each do |type|
292
- names = pipfile_object[type]&.keys || []
293
- pkg_name = names.find { |nm| normalise(nm) == dependency.name }
294
- next unless pkg_name
295
-
296
- if pipfile_object.dig(type, pkg_name).is_a?(Hash)
297
- pipfile_object[type][pkg_name]["version"] =
298
- updated_version_requirement_string
299
- else
300
- pipfile_object[type][pkg_name] =
301
- updated_version_requirement_string
302
- end
303
- end
304
-
305
- TomlRB.dump(pipfile_object)
306
- end
307
-
308
- def add_private_sources(pipfile_content)
309
- FileUpdaters::Python::Pip::PipfilePreparer.
310
- new(pipfile_content: pipfile_content).
311
- replace_sources(credentials)
312
- end
313
-
314
- def add_python_two_requirement_to_pipfile
315
- content = File.read("Pipfile")
316
-
317
- updated_content =
318
- FileUpdaters::Python::Pip::PipfilePreparer.
319
- new(pipfile_content: content).
320
- update_python_requirement("2.7.15")
321
-
322
- File.write("Pipfile", updated_content)
323
- end
324
-
325
- def pipfile_python_requirement
326
- parsed_pipfile = TomlRB.parse(pipfile.content)
327
-
328
- parsed_pipfile.dig("requires", "python_full_version") ||
329
- parsed_pipfile.dig("requires", "python_version")
330
- end
331
-
332
- def python_requirement_specified?
333
- return true if pipfile_python_requirement
334
-
335
- !python_version_file.nil?
336
- end
337
-
338
- def set_up_python_environment
339
- # Initialize a git repo to appease pip-tools
340
- begin
341
- SharedHelpers.run_shell_command("git init") if setup_files.any?
342
- rescue Dependabot::SharedHelpers::HelperSubprocessFailed
343
- nil
344
- end
345
-
346
- SharedHelpers.run_shell_command(
347
- "pyenv install -s #{python_version}"
348
- )
349
- SharedHelpers.run_shell_command("pyenv local #{python_version}")
350
- return if pre_installed_python?(python_version)
351
-
352
- SharedHelpers.run_shell_command(
353
- "pyenv exec pip install -r #{python_requirements_path}"
354
- )
355
- end
356
-
357
- def python_version
358
- requirement =
359
- if @using_python_two
360
- "2.7.*"
361
- elsif pipfile_python_requirement&.match?(/^\d/)
362
- parts = pipfile_python_requirement.split(".")
363
- parts.fill("*", (parts.length)..2).join(".")
364
- elsif python_version_file
365
- python_version_file.content
366
- else
367
- PythonVersions::PRE_INSTALLED_PYTHON_VERSIONS.first
368
- end
369
-
370
- requirement = Utils::Python::Requirement.new(requirement)
371
-
372
- PythonVersions::PYTHON_VERSIONS.find do |version|
373
- requirement.satisfied_by?(Utils::Python::Version.new(version))
374
- end
375
- end
376
-
377
- def pre_installed_python?(version)
378
- PythonVersions::PRE_INSTALLED_PYTHON_VERSIONS.include?(version)
379
- end
380
-
381
- def check_private_sources_are_reachable
382
- env_sources = pipfile_sources.select { |h| h["url"].include?("${") }
383
-
384
- check_env_sources_included_in_config_variables(env_sources)
385
-
386
- sources_to_check =
387
- pipfile_sources.reject { |h| h["url"].include?("${") } +
388
- config_variable_sources
389
-
390
- sources_to_check.
391
- map { |details| details["url"] }.
392
- reject { |url| MAIN_PYPI_INDEXES.include?(url) }.
393
- each do |url|
394
- sanitized_url = url.gsub(%r{(?<=//).*(?=@)}, "redacted")
395
-
396
- response = Excon.get(
397
- url,
398
- idempotent: true,
399
- **SharedHelpers.excon_defaults
400
- )
401
-
402
- if response.status == 401 || response.status == 403
403
- raise PrivateSourceAuthenticationFailure, sanitized_url
404
- end
405
- rescue Excon::Error::Timeout, Excon::Error::Socket
406
- raise PrivateSourceTimedOut, sanitized_url
407
- end
408
- end
409
-
410
- def updated_version_requirement_string
411
- lower_bound_req = updated_version_req_lower_bound
412
-
413
- # Add the latest_allowable_version as an upper bound. This means
414
- # ignore conditions are considered when checking for the latest
415
- # resolvable version.
416
- #
417
- # NOTE: This isn't perfect. If v2.x is ignored and v3 is out but
418
- # unresolvable then the `latest_allowable_version` will be v3, and
419
- # we won't be ignoring v2.x releases like we should be.
420
- return lower_bound_req if latest_allowable_version.nil?
421
- unless Utils::Python::Version.correct?(latest_allowable_version)
422
- return lower_bound_req
423
- end
424
-
425
- lower_bound_req + ", <= #{latest_allowable_version}"
426
- end
427
-
428
- def updated_version_req_lower_bound
429
- if dependency.version
430
- ">= #{dependency.version}"
431
- else
432
- version_for_requirement =
433
- dependency.requirements.map { |r| r[:requirement] }.compact.
434
- reject { |req_string| req_string.start_with?("<") }.
435
- select { |req_string| req_string.match?(VERSION_REGEX) }.
436
- map { |req_string| req_string.match(VERSION_REGEX) }.
437
- select { |version| Gem::Version.correct?(version) }.
438
- max_by { |version| Gem::Version.new(version) }
439
-
440
- ">= #{version_for_requirement || 0}"
441
- end
442
- end
443
-
444
- def run_pipenv_command(cmd)
445
- set_up_python_environment
446
-
447
- raw_response = nil
448
- IO.popen(cmd, err: %i(child out)) { |p| raw_response = p.read }
449
-
450
- # Raise an error with the output from the shell session if Pipenv
451
- # returns a non-zero status
452
- return if $CHILD_STATUS.success?
453
-
454
- raise SharedHelpers::HelperSubprocessFailed.new(raw_response, cmd)
455
- rescue SharedHelpers::HelperSubprocessFailed => error
456
- original_error ||= error
457
- msg = error.message
458
-
459
- relevant_error =
460
- if may_be_using_wrong_python_version?(msg) then original_error
461
- else error
462
- end
463
-
464
- raise relevant_error unless may_be_using_wrong_python_version?(msg)
465
- raise relevant_error if @using_python_two
466
-
467
- @using_python_two = true
468
- add_python_two_requirement_to_pipfile
469
- cmd = cmd.gsub("pipenv ", "pipenv --two ")
470
- retry
471
- ensure
472
- @using_python_two = nil
473
- end
474
-
475
- def may_be_using_wrong_python_version?(error_message)
476
- return false if python_requirement_specified?
477
- return true if error_message.include?("UnsupportedPythonVersion")
478
-
479
- error_message.include?('Command "python setup.py egg_info" failed')
480
- end
481
-
482
- def check_env_sources_included_in_config_variables(env_sources)
483
- config_variable_source_urls =
484
- config_variable_sources.map { |s| s["url"] }
485
-
486
- env_sources.each do |source|
487
- url = source["url"]
488
- known_parts = url.split(/\$\{.*?\}/).reject(&:empty?).compact
489
-
490
- # If the whole URL is an environment variable we can't do a check
491
- next if known_parts.none?
492
-
493
- regex = known_parts.map { |p| Regexp.quote(p) }.join(".*?")
494
- next if config_variable_source_urls.any? { |s| s.match?(regex) }
495
-
496
- raise PrivateSourceAuthenticationFailure, url
497
- end
498
- end
499
-
500
- def config_variable_sources
501
- @config_variable_sources ||=
502
- credentials.
503
- select { |cred| cred["type"] == "python_index" }.
504
- map { |h| { "url" => h["index-url"].gsub(%r{/*$}, "") + "/" } }
505
- end
506
-
507
- def pipfile_sources
508
- @pipfile_sources ||=
509
- TomlRB.parse(pipfile.content).fetch("source", []).
510
- map { |h| h.dup.merge("url" => h["url"].gsub(%r{/*$}, "") + "/") }
511
- end
512
-
513
- def pipenv_environment_variables
514
- environment_variables = [
515
- "PIPENV_YES=true", # Install new Python versions if needed
516
- "PIPENV_MAX_RETRIES=3", # Retry timeouts
517
- "PIPENV_NOSPIN=1", # Don't pollute logs with spinner
518
- "PIPENV_TIMEOUT=600", # Set install timeout to 10 minutes
519
- "PIP_DEFAULT_TIMEOUT=60" # Set pip timeout to 1 minute
520
- ]
521
-
522
- environment_variables.join(" ") + " "
523
- end
524
-
525
- def python_requirements_path
526
- project_root = File.join(File.dirname(__FILE__), "../../../../..")
527
- File.join(project_root, "helpers/python/requirements.txt")
528
- end
529
-
530
- # See https://www.python.org/dev/peps/pep-0503/#normalized-names
531
- def normalise(name)
532
- name.downcase.gsub(/[-_.]+/, "-")
533
- end
534
-
535
- def pipfile
536
- dependency_files.find { |f| f.name == "Pipfile" }
537
- end
538
-
539
- def lockfile
540
- dependency_files.find { |f| f.name == "Pipfile.lock" }
541
- end
542
-
543
- def setup_files
544
- dependency_files.select { |f| f.name.end_with?("setup.py") }
545
- end
546
-
547
- def setup_cfg_files
548
- dependency_files.select { |f| f.name.end_with?("setup.cfg") }
549
- end
550
-
551
- def python_version_file
552
- dependency_files.find { |f| f.name == ".python-version" }
553
- end
554
- end
555
- end
556
- end
557
- end
558
- end
559
- # rubocop:enable Metrics/ClassLength