dependabot-python 0.79.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: '0996037472fe577fcff80293f9fe7c5a1a3579f5bfc74b74aafdbdb05ab2940d'
+  data.tar.gz: 7e90b75ec5540d68b384052354fbf216b59fc6ec8839ec82352f743af0112490
+SHA512:
+  metadata.gz: 6c275bd0828455b6aeaa98fe936570d179201992229d0994de5f586843dccd73187278686736736023a8839ea9bf588754e8c4e087ec9341ede5a89f93440f37
+  data.tar.gz: 60e5e5b0f05fa66bf2d675de732bc16c4341f1f7188f33b3903323e59189be8ad4d6cbc0568529220f4ff32751916d62c4834e7869c1ea59e77df87decceface

data/helpers/build

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+
+install_dir=$1
+if [ -z "$install_dir" ]; then
+  echo "usage: $0 INSTALL_DIR"
+  exit 1
+fi
+
+if [ ! -d "$install_dir/bin" ]; then
+  mkdir -p "$install_dir/bin"
+fi
+
+helpers_dir="$(dirname "${BASH_SOURCE[0]}")"
+PYENV_VERSION=2.7.15 pyenv exec pip install -r "${helpers_dir}/requirements.txt"
+PYENV_VERSION=3.6.7  pyenv exec pip install -r "${helpers_dir}/requirements.txt"

data/helpers/lib/hasher.py

@@ -0,0 +1,23 @@
+import hashin
+import json
+import pipfile
+from poetry.poetry import Poetry
+
+def get_dependency_hash(dependency_name, dependency_version, algorithm):
+    hashes = hashin.get_package_hashes(
+        dependency_name,
+        version=dependency_version,
+        algorithm=algorithm
+    )
+
+    return json.dumps({ "result": hashes["hashes"] })
+
+def get_pipfile_hash(directory):
+    p = pipfile.load(directory + '/Pipfile')
+
+    return json.dumps({ "result": p.hash })
+
+def get_pyproject_hash(directory):
+    p = Poetry.create(directory)
+
+    return json.dumps({ "result": p.locker._get_content_hash() })

data/helpers/lib/parser.py

@@ -0,0 +1,130 @@
+from itertools import chain
+import glob
+import io
+import json
+import os.path
+import re
+
+import setuptools
+import pip._internal.req.req_file
+from pip._internal.download import PipSession
+from pip._internal.req.constructors import install_req_from_line
+
+def parse_requirements(directory):
+    # Parse the requirements.txt
+    requirement_packages = []
+
+    requirement_files = glob.glob(os.path.join(directory, '*.txt')) \
+                        + glob.glob(os.path.join(directory, '**', '*.txt'))
+
+    pip_compile_files = glob.glob(os.path.join(directory, '*.in')) \
+                        + glob.glob(os.path.join(directory, '**', '*.in'))
+
+    for reqs_file in requirement_files + pip_compile_files:
+        try:
+            requirements = pip._internal.req.req_file.parse_requirements(
+                reqs_file,
+                session=PipSession()
+            )
+            for install_req in requirements:
+                if install_req.original_link:
+                    continue
+                if install_req.is_pinned:
+                    version = next(iter(install_req.specifier)).version
+                else:
+                    version = None
+
+                pattern = r"-[cr] (.*) \(line \d+\)"
+                abs_path = re.search(pattern, install_req.comes_from).group(1)
+                rel_path = os.path.relpath(abs_path, directory)
+
+                requirement_packages.append({
+                    "name": install_req.req.name,
+                    "version": version,
+                    "markers": str(install_req.markers) or None,
+                    "file": rel_path,
+                    "requirement": str(install_req.specifier) or None
+                })
+        except Exception as e:
+            print(json.dumps({ "error": repr(e) }))
+            exit(1)
+
+    return json.dumps({ "result": requirement_packages })
+
+def parse_setup(directory):
+    # Parse the setup.py
+    setup_packages = []
+    if os.path.isfile(directory + '/setup.py'):
+        def parse_requirement(req, req_type):
+            install_req = install_req_from_line(req)
+            if install_req.original_link:
+                return
+            if install_req.is_pinned:
+                version = next(iter(install_req.specifier)).version
+            else:
+                version = None
+            setup_packages.append({
+                "name": install_req.req.name,
+                "version": version,
+                "markers": str(install_req.markers) or None,
+                "file": "setup.py",
+                "requirement": str(install_req.specifier) or None,
+                "requirement_type": req_type
+            })
+
+        def setup(*args, **kwargs):
+            for arg in ['setup_requires', 'install_requires', 'tests_require']:
+                if not kwargs.get(arg):
+                    continue
+                for req in kwargs.get(arg):
+                    parse_requirement(req, arg)
+            extras_require_dict = kwargs.get('extras_require', {})
+            for key in extras_require_dict:
+                for req in extras_require_dict[key]:
+                    parse_requirement(req, 'extras_require:{}'.format(key))
+        setuptools.setup = setup
+
+        def noop(*args, **kwargs):
+            pass
+
+        def fake_parse(*args, **kwargs):
+            return []
+
+        global fake_open
+        def fake_open(*args, **kwargs):
+            content = ("VERSION = ('0', '0', '1+dependabot')\n"
+                       "__version__ = '0.0.1+dependabot'\n"
+                       "__author__ = 'someone'\n"
+                       "__title__ = 'something'\n"
+                       "__description__ = 'something'\n"
+                       "__author_email__ = 'something'\n"
+                       "__license__ = 'something'\n"
+                       "__url__ = 'something'\n")
+            return io.StringIO(content)
+
+        content = open(directory + '/setup.py', 'r').read()
+
+        # Remove `print`, `open`, `log` and import statements
+        content = re.sub(r"print\s*\(", "noop(", content)
+        content = re.sub(r"log\s*(\.\w+)*\(", "noop(", content)
+        content = re.sub(r"\b(\w+\.)*(open|file)\s*\(", "fake_open(", content)
+        content = content.replace("parse_requirements(", "fake_parse(")
+        version_re = re.compile(r"^.*import.*__version__.*$", re.MULTILINE)
+        content = re.sub(version_re, "", content)
+
+        # Set variables likely to be imported
+        __version__ = '0.0.1+dependabot'
+        __author__ = 'someone'
+        __title__ = 'something'
+        __description__ = 'something'
+        __author_email__ = 'something'
+        __license__ = 'something'
+        __url__ = 'something'
+
+        # Run as main (since setup.py is a script)
+        __name__ = '__main__'
+
+        # Exec the setup.py
+        exec(content) in globals(), locals()
+
+    return json.dumps({ "result": setup_packages })

data/helpers/requirements.txt

@@ -0,0 +1,9 @@
+pip==18.1
+pip-tools==3.1.0
+hashin==0.14.0
+pipenv==2018.11.26
+pipfile==0.0.2
+poetry==0.12.10
+
+# Some dependencies will only install if Cython is present
+Cython==0.29.1

data/helpers/run.py

@@ -0,0 +1,18 @@
+import sys
+import json
+
+from lib import parser, hasher
+
+if __name__ == "__main__":
+    args = json.loads(sys.stdin.read())
+
+    if args["function"] == "parse_requirements":
+        print(parser.parse_requirements(args["args"][0]))
+    if args["function"] == "parse_setup":
+        print(parser.parse_setup(args["args"][0]))
+    elif args["function"] == "get_dependency_hash":
+        print(hasher.get_dependency_hash(*args["args"]))
+    elif args["function"] == "get_pipfile_hash":
+        print(hasher.get_pipfile_hash(*args["args"]))
+    elif args["function"] == "get_pyproject_hash":
+        print(hasher.get_pyproject_hash(*args["args"]))

data/lib/dependabot/python.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# These all need to be required so the various classes can be registered in a
+# lookup table of package manager names to concrete classes.
+require "dependabot/python/file_fetcher"
+require "dependabot/python/file_parser"
+require "dependabot/python/update_checker"
+require "dependabot/python/file_updater"
+require "dependabot/python/metadata_finder"
+require "dependabot/python/requirement"
+require "dependabot/python/version"

data/lib/dependabot/python/file_fetcher.rb

@@ -0,0 +1,307 @@
+# frozen_string_literal: true
+
+require "toml-rb"
+
+require "dependabot/file_fetchers"
+require "dependabot/file_fetchers/base"
+require "dependabot/python/file_parser"
+require "dependabot/errors"
+
+module Dependabot
+  module Python
+    class FileFetcher < Dependabot::FileFetchers::Base
+      CHILD_REQUIREMENT_REGEX = /^-r\s?(?<path>.*\.txt)/.freeze
+      CONSTRAINT_REGEX = /^-c\s?(?<path>\..*)/.freeze
+
+      def self.required_files_in?(filenames)
+        return true if filenames.any? { |name| name.end_with?(".txt", ".in") }
+
+        # If there is a directory of requirements return true
+        return true if filenames.include?("requirements")
+
+        # If this repo is using a Pipfile return true
+        return true if filenames.include?("Pipfile")
+
+        # If this repo is using Poetry return true
+        return true if filenames.include?("pyproject.toml")
+
+        filenames.include?("setup.py")
+      end
+
+      def self.required_files_message
+        "Repo must contain a requirements.txt, setup.py, pyproject.toml, "\
+        "or a Pipfile."
+      end
+
+      private
+
+      def fetch_files
+        fetched_files = []
+
+        fetched_files += pipenv_files
+        fetched_files += pyproject_files
+
+        fetched_files += requirements_in_files
+        fetched_files += requirement_files if requirements_txt_files.any?
+
+        fetched_files << setup_file if setup_file
+        fetched_files << setup_cfg if setup_cfg
+        fetched_files += path_setup_files
+        fetched_files << pip_conf if pip_conf
+        fetched_files << python_version if python_version
+
+        check_required_files_present
+        fetched_files.uniq
+      end
+
+      def pipenv_files
+        [pipfile, pipfile_lock].compact
+      end
+
+      def pyproject_files
+        [pyproject, pyproject_lock, poetry_lock].compact
+      end
+
+      def requirement_files
+        [
+          *requirements_txt_files,
+          *child_requirement_files,
+          *constraints_files
+        ]
+      end
+
+      def check_required_files_present
+        if requirements_txt_files.any? || setup_file || pipfile || pyproject
+          return
+        end
+
+        path = Pathname.new(File.join(directory, "requirements.txt")).
+               cleanpath.to_path
+        raise Dependabot::DependencyFileNotFound, path
+      end
+
+      def setup_file
+        @setup_file ||= fetch_file_if_present("setup.py")
+      end
+
+      def setup_cfg
+        @setup_cfg ||= fetch_file_if_present("setup.cfg")
+      end
+
+      def pip_conf
+        @pip_conf ||= fetch_file_if_present("pip.conf")&.
+                      tap { |f| f.support_file = true }
+      end
+
+      def python_version
+        @python_version ||= fetch_file_if_present(".python-version")&.
+                            tap { |f| f.support_file = true }
+      end
+
+      def pipfile
+        @pipfile ||= fetch_file_if_present("Pipfile")
+      end
+
+      def pipfile_lock
+        @pipfile_lock ||= fetch_file_if_present("Pipfile.lock")
+      end
+
+      def pyproject
+        @pyproject ||= fetch_file_if_present("pyproject.toml")
+      end
+
+      def pyproject_lock
+        @pyproject_lock ||= fetch_file_if_present("pyproject.lock")
+      end
+
+      def poetry_lock
+        @poetry_lock ||= fetch_file_if_present("poetry.lock")
+      end
+
+      def requirements_txt_files
+        req_txt_and_in_files.select { |f| f.name.end_with?(".txt") }
+      end
+
+      def requirements_in_files
+        req_txt_and_in_files.select { |f| f.name.end_with?(".in") }
+      end
+
+      def parsed_pipfile
+        raise "No Pipfile" unless pipfile
+
+        @parsed_pipfile ||= TomlRB.parse(pipfile.content)
+      rescue TomlRB::ParseError
+        raise Dependabot::DependencyFileNotParseable, pipfile.path
+      end
+
+      def req_txt_and_in_files
+        return @req_txt_and_in_files if @req_txt_and_in_files
+
+        @req_txt_and_in_files = []
+
+        repo_contents.
+          select { |f| f.type == "file" }.
+          select { |f| f.name.end_with?(".txt", ".in") }.
+          map { |f| fetch_file_from_host(f.name) }.
+          select { |f| requirements_file?(f) }.
+          each { |f| @req_txt_and_in_files << f }
+
+        repo_contents.
+          select { |f| f.type == "dir" }.
+          each { |f| @req_txt_and_in_files += req_files_for_dir(f) }
+
+        @req_txt_and_in_files
+      end
+
+      def req_files_for_dir(requirements_dir)
+        dir = directory.gsub(%r{(^/|/$)}, "")
+        relative_reqs_dir =
+          requirements_dir.path.gsub(%r{^/?#{Regexp.escape(dir)}/?}, "")
+
+        repo_contents(dir: relative_reqs_dir).
+          select { |f| f.type == "file" }.
+          select { |f| f.name.end_with?(".txt", ".in") }.
+          map { |f| fetch_file_from_host("#{relative_reqs_dir}/#{f.name}") }.
+          select { |f| requirements_file?(f) }
+      end
+
+      def child_requirement_files
+        @child_requirement_files ||=
+          begin
+            fetched_files = requirements_txt_files.dup
+            requirements_txt_files.flat_map do |requirement_file|
+              child_files = fetch_child_requirement_files(
+                file: requirement_file,
+                previously_fetched_files: fetched_files
+              )
+
+              fetched_files += child_files
+              child_files
+            end
+          end
+      end
+
+      def fetch_child_requirement_files(file:, previously_fetched_files:)
+        paths = file.content.scan(CHILD_REQUIREMENT_REGEX).flatten
+        current_dir = File.dirname(file.name)
+
+        paths.flat_map do |path|
+          path = File.join(current_dir, path) unless current_dir == "."
+          path = Pathname.new(path).cleanpath.to_path
+
+          next if previously_fetched_files.map(&:name).include?(path)
+          next if file.name == path
+
+          fetched_file = fetch_file_from_host(path)
+          grandchild_requirement_files = fetch_child_requirement_files(
+            file: fetched_file,
+            previously_fetched_files: previously_fetched_files + [file]
+          )
+          [fetched_file, *grandchild_requirement_files]
+        end.compact
+      end
+
+      def constraints_files
+        all_requirement_files = requirements_txt_files +
+                                child_requirement_files
+
+        constraints_paths = all_requirement_files.map do |req_file|
+          req_file.content.scan(CONSTRAINT_REGEX).flatten
+        end.flatten.uniq
+
+        constraints_paths.map { |path| fetch_file_from_host(path) }
+      end
+
+      def path_setup_files
+        path_setup_files = []
+        unfetchable_files = []
+
+        path_setup_file_paths.each do |path|
+          path = Pathname.new(File.join(path, "setup.py")).cleanpath.to_path
+          next if path == "setup.py" && setup_file
+
+          begin
+            path_setup_files << fetch_file_from_host(path).
+                                tap { |f| f.support_file = true }
+          rescue Dependabot::DependencyFileNotFound
+            unfetchable_files << path
+          end
+
+          begin
+            cfg_path = path.gsub(/\.py$/, ".cfg")
+            path_setup_files << fetch_file_from_host(cfg_path).
+                                tap { |f| f.support_file = true }
+          rescue Dependabot::DependencyFileNotFound
+            # Ignore lack of a setup.cfg
+            nil
+          end
+        end
+
+        if unfetchable_files.any?
+          raise Dependabot::PathDependenciesNotReachable, unfetchable_files
+        end
+
+        path_setup_files
+      end
+
+      def requirements_file?(file)
+        return true if file.name.match?(/requirements/x)
+
+        content = file.content.
+                  gsub(CONSTRAINT_REGEX, "").
+                  gsub(CHILD_REQUIREMENT_REGEX, "")
+
+        tmp_file = DependencyFile.new(name: file.name, content: content)
+        Dependabot::Python::FileParser.
+          new(dependency_files: [tmp_file], source: source).
+          parse.any?
+      rescue Dependabot::DependencyFileNotEvaluatable
+        false
+      end
+
+      def path_setup_file_paths
+        requirement_txt_path_setup_file_paths + pipfile_path_setup_file_paths
+      end
+
+      def requirement_txt_path_setup_file_paths
+        (requirements_txt_files + child_requirement_files).map do |req_file|
+          uneditable_reqs =
+            req_file.content.
+            scan(/^['"]?(?<path>\..*?)(?=\[|#|'|"|$)/).
+            flatten.
+            map(&:strip).
+            reject { |p| p.include?("://") }
+
+          editable_reqs =
+            req_file.content.
+            scan(/^(?:-e)\s+['"]?(?<path>.*?)(?=\[|#|'|"|$)/).
+            flatten.
+            map(&:strip).
+            reject { |p| p.include?("://") }
+
+          uneditable_reqs + editable_reqs
+        end.flatten.uniq
+      end
+
+      def pipfile_path_setup_file_paths
+        return [] unless pipfile
+
+        paths = []
+        %w(packages dev-packages).each do |dep_type|
+          next unless parsed_pipfile[dep_type]
+
+          parsed_pipfile[dep_type].each do |_, req|
+            next unless req.is_a?(Hash) && req["path"]
+
+            paths << req["path"]
+          end
+        end
+
+        paths
+      end
+    end
+  end
+end
+
+Dependabot::FileFetchers.
+  register("pip", Dependabot::Python::FileFetcher)