dependabot-core 0.76.1

data/lib/dependabot/file_fetchers/java_script/npm_and_yarn.rb

@@ -0,0 +1,330 @@
+# frozen_string_literal: true
+
+require "json"
+require "dependabot/file_fetchers/base"
+require "dependabot/file_parsers/java_script/npm_and_yarn"
+
+module Dependabot
+  module FileFetchers
+    module JavaScript
+      class NpmAndYarn < Dependabot::FileFetchers::Base
+        require_relative "npm_and_yarn/path_dependency_builder"
+
+        def self.required_files_in?(filenames)
+          filenames.include?("package.json")
+        end
+
+        def self.required_files_message
+          "Repo must contain a package.json."
+        end
+
+        private
+
+        # rubocop:disable Metrics/CyclomaticComplexity
+        # rubocop:disable Metrics/PerceivedComplexity
+        def fetch_files
+          fetched_files = []
+          fetched_files << package_json
+          fetched_files << package_lock if package_lock && !ignore_package_lock?
+          fetched_files << yarn_lock if yarn_lock
+          fetched_files << shrinkwrap if shrinkwrap
+          fetched_files << lerna_json if lerna_json
+          fetched_files << npmrc if npmrc
+          fetched_files << yarnrc if yarnrc
+          fetched_files += workspace_package_jsons
+          fetched_files += lerna_packages
+          fetched_files += path_dependencies(fetched_files)
+
+          fetched_files.uniq
+        end
+        # rubocop:enable Metrics/CyclomaticComplexity
+        # rubocop:enable Metrics/PerceivedComplexity
+
+        def package_json
+          @package_json ||= fetch_file_from_host("package.json")
+        end
+
+        def package_lock
+          @package_lock ||= fetch_file_if_present("package-lock.json")
+        end
+
+        def yarn_lock
+          @yarn_lock ||= fetch_file_if_present("yarn.lock")
+        end
+
+        def shrinkwrap
+          @shrinkwrap ||= fetch_file_if_present("npm-shrinkwrap.json")
+        end
+
+        def npmrc
+          @npmrc ||= fetch_file_if_present(".npmrc")&.
+                     tap { |f| f.support_file = true }
+
+          return @npmrc if @npmrc || directory == "/"
+
+          # Loop through parent directories looking for an npmrc
+          (1..directory.split("/").count).each do |i|
+            @npmrc = fetch_file_from_host("../" * i + ".npmrc")&.
+                     tap { |f| f.support_file = true }
+            break if @npmrc
+          rescue Dependabot::DependencyFileNotFound
+            # Ignore errors (.npmrc may not be present)
+            nil
+          end
+
+          @npmrc
+        end
+
+        def yarnrc
+          @yarnrc ||= fetch_file_if_present(".yarnrc")&.
+                     tap { |f| f.support_file = true }
+
+          return @yarnrc if @yarnrc || directory == "/"
+
+          # Loop through parent directories looking for an yarnrc
+          (1..directory.split("/").count).each do |i|
+            @yarnrc = fetch_file_from_host("../" * i + ".yarnrc")&.
+                     tap { |f| f.support_file = true }
+            break if @yarnrc
+          rescue Dependabot::DependencyFileNotFound
+            # Ignore errors (.yarnrc may not be present)
+            nil
+          end
+
+          @yarnrc
+        end
+
+        def lerna_json
+          @lerna_json ||= fetch_file_if_present("lerna.json")&.
+                          tap { |f| f.support_file = true }
+        end
+
+        def workspace_package_jsons
+          @workspace_package_jsons ||= fetch_workspace_package_jsons
+        end
+
+        def lerna_packages
+          @lerna_packages ||= fetch_lerna_packages
+        end
+
+        def path_dependencies(fetched_files)
+          package_json_files = []
+          unfetchable_deps = []
+
+          path_dependency_details(fetched_files).each do |name, path|
+            path = path.sub(/^file:/, "").sub(/^link:/, "")
+            filename = File.join(path, "package.json")
+            cleaned_name = Pathname.new(filename).cleanpath.to_path
+            next if fetched_files.map(&:name).include?(cleaned_name)
+
+            begin
+              file = fetch_file_from_host(filename, type: "path_dependency")
+              package_json_files << file
+            rescue Dependabot::DependencyFileNotFound
+              unfetchable_deps << [name, path]
+            end
+          end
+
+          package_json_files += build_unfetchable_deps(unfetchable_deps)
+
+          if package_json_files.any?
+            package_json_files +=
+              path_dependencies(fetched_files + package_json_files)
+          end
+
+          package_json_files.tap { |fs| fs.each { |f| f.support_file = true } }
+        end
+
+        def path_dependency_details(fetched_files)
+          package_json_path_deps = []
+
+          fetched_files.each do |file|
+            package_json_path_deps +=
+              path_dependency_details_from_manifest(file)
+          end
+
+          path_starts = %w(file: link:.)
+
+          package_lock_path_deps =
+            parsed_package_lock.fetch("dependencies", []).to_a.
+            select { |_, v| v.fetch("version", "").start_with?(*path_starts) }.
+            map { |k, v| [k, v.fetch("version")] }
+
+          shrinkwrap_path_deps =
+            parsed_shrinkwrap.fetch("dependencies", []).to_a.
+            select { |_, v| v.fetch("version", "").start_with?(*path_starts) }.
+            map { |k, v| [k, v.fetch("version")] }
+
+          [
+            *package_json_path_deps,
+            *package_lock_path_deps,
+            *shrinkwrap_path_deps
+          ].uniq
+        end
+
+        def path_dependency_details_from_manifest(file)
+          return [] unless file.name.end_with?("package.json")
+
+          current_dir = file.name.rpartition("/").first
+          current_dir = nil if current_dir == ""
+          path_dep_starts = %w(file: / ./ ../ ~/ link:.)
+
+          JSON.parse(file.content).
+            values_at(*FileParsers::JavaScript::NpmAndYarn::DEPENDENCY_TYPES).
+            compact.flat_map(&:to_a).
+            select { |_, v| v.start_with?(*path_dep_starts) }.
+            map do |name, path|
+              path = path.sub(/^file:/, "").sub(/^link:/, "")
+              path = File.join(current_dir, path) unless current_dir.nil?
+              [name, Pathname.new(path).cleanpath.to_path]
+            end
+        rescue JSON::ParserError
+          raise Dependabot::DependencyFileNotParseable, file.path
+        end
+
+        def fetch_workspace_package_jsons
+          return [] unless parsed_package_json["workspaces"]
+
+          package_json_files = []
+
+          workspace_paths(parsed_package_json["workspaces"]).each do |workspace|
+            file = File.join(workspace, "package.json")
+
+            begin
+              package_json_files << fetch_file_from_host(file)
+            rescue Dependabot::DependencyFileNotFound
+              nil
+            end
+          end
+
+          package_json_files
+        end
+
+        def fetch_lerna_packages
+          return [] unless parsed_lerna_json["packages"]
+
+          dependency_files = []
+
+          workspace_paths(parsed_lerna_json["packages"]).each do |workspace|
+            dependency_files += fetch_lerna_packages_from_path(workspace)
+          end
+
+          dependency_files
+        end
+
+        def fetch_lerna_packages_from_path(path, nested = false)
+          dependency_files = []
+
+          package_json_path = File.join(path, "package.json")
+
+          begin
+            dependency_files << fetch_file_from_host(package_json_path)
+            dependency_files += [
+              fetch_file_if_present(File.join(path, "package-lock.json")),
+              fetch_file_if_present(File.join(path, "yarn.lock")),
+              fetch_file_if_present(File.join(path, "npm-shrinkwrap.json"))
+            ].compact
+          rescue Dependabot::DependencyFileNotFound
+            matches_double_glob =
+              parsed_lerna_json["packages"].any? do |globbed_path|
+                next false unless globbed_path.include?("**")
+
+                File.fnmatch?(globbed_path, path)
+              end
+
+            if matches_double_glob && !nested
+              dependency_files +=
+                expanded_paths(File.join(path, "*")).flat_map do |nested_path|
+                  fetch_lerna_packages_from_path(nested_path, true)
+                end
+            end
+          end
+
+          dependency_files
+        end
+
+        def workspace_paths(workspace_object)
+          paths_array =
+            if workspace_object.is_a?(Hash)
+              workspace_object.values_at("packages", "nohoist").flatten.compact
+            elsif workspace_object.is_a?(Array) then workspace_object
+            else raise "Unexpected workspace object"
+            end
+
+          paths_array.flat_map do |path|
+            # The packages/!(not-this-package) syntax is unique to Yarn
+            if path.include?("*") || path.include?("!(")
+              expanded_paths(path)
+            else path
+            end
+          end
+        end
+
+        def expanded_paths(path)
+          ignored_paths = path.scan(/!\((.*?)\)/).flatten
+
+          dir = directory.gsub(%r{(^/|/$)}, "")
+          path = path.gsub(%r{^\./}, "").gsub(/!\(.*?\)/, "*")
+          unglobbed_path = path.split("*").first&.gsub(%r{(?<=/)[^/]*$}, "") ||
+                           "."
+
+          repo_contents(dir: unglobbed_path, raise_errors: false).
+            select { |file| file.type == "dir" }.
+            map { |f| f.path.gsub(%r{^/?#{Regexp.escape(dir)}/?}, "") }.
+            select { |filename| File.fnmatch?(path, filename) }.
+            reject { |fn| ignored_paths.any? { |p| fn.include?(p) } }
+        end
+
+        def parsed_package_json
+          JSON.parse(package_json.content)
+        rescue JSON::ParserError
+          raise Dependabot::DependencyFileNotParseable, package_json.path
+        end
+
+        def parsed_package_lock
+          return {} unless package_lock
+
+          JSON.parse(package_lock.content)
+        rescue JSON::ParserError
+          {}
+        end
+
+        def parsed_shrinkwrap
+          return {} unless shrinkwrap
+
+          JSON.parse(shrinkwrap.content)
+        rescue JSON::ParserError
+          {}
+        end
+
+        def ignore_package_lock?
+          return false unless npmrc
+
+          npmrc.content.match?(/^package-lock\s*=\s*false/)
+        end
+
+        def build_unfetchable_deps(unfetchable_deps)
+          return [] unless package_lock || yarn_lock
+
+          unfetchable_deps.map do |name, path|
+            PathDependencyBuilder.new(
+              dependency_name: name,
+              path: path,
+              directory: directory,
+              package_lock: package_lock,
+              yarn_lock: yarn_lock
+            ).dependency_file
+          end
+        end
+
+        def parsed_lerna_json
+          return {} unless lerna_json
+
+          JSON.parse(lerna_json.content)
+        rescue JSON::ParserError
+          raise Dependabot::DependencyFileNotParseable, lerna_json.path
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/file_fetchers/java_script/npm_and_yarn/path_dependency_builder.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "json"
+require "dependabot/dependency_file"
+require "dependabot/errors"
+require "dependabot/file_fetchers/java_script/npm_and_yarn"
+
+module Dependabot
+  module FileFetchers
+    module JavaScript
+      class NpmAndYarn
+        class PathDependencyBuilder
+          def initialize(dependency_name:, path:, directory:, package_lock:,
+                         yarn_lock:)
+            @dependency_name = dependency_name
+            @path = path
+            @directory = directory
+            @package_lock = package_lock
+            @yarn_lock = yarn_lock
+          end
+
+          def dependency_file
+            filename = File.join(path, "package.json")
+
+            DependencyFile.new(
+              name: Pathname.new(filename).cleanpath.to_path,
+              content: build_path_dep_content(dependency_name),
+              directory: directory,
+              type: "path_dependency"
+            )
+          end
+
+          private
+
+          attr_reader :dependency_name, :path, :package_lock, :yarn_lock,
+                      :directory
+
+          def details_from_yarn_lock
+            parsed_yarn_lock.to_a.
+              find do |n, _|
+                next false unless n.split(/(?<=\w)\@/).first == dependency_name
+
+                n.split(/(?<=\w)\@/).last.start_with?("file:")
+              end&.last
+          end
+
+          def details_from_npm_lock
+            parsed_package_lock.fetch("dependencies", []).to_a.
+              select { |_, v| v.fetch("version", "").start_with?("file:") }.
+              find { |n, _| n == dependency_name }&.
+              last
+          end
+
+          def build_path_dep_content(dependency_name)
+            unless details_from_yarn_lock || details_from_npm_lock
+              raise Dependabot::PathDependenciesNotReachable, [dependency_name]
+            end
+
+            if details_from_yarn_lock
+              {
+                name: dependency_name,
+                version: "0.0.1",
+                dependencies: details_from_yarn_lock["dependencies"],
+                optionalDependencies:
+                  details_from_yarn_lock["optionalDependencies"]
+              }.compact.to_json
+            else
+              {
+                name: dependency_name,
+                version: "0.0.1",
+                dependencies: details_from_npm_lock["requires"]
+              }.compact.to_json
+            end
+          end
+
+          def parsed_package_lock
+            return {} unless package_lock
+
+            JSON.parse(package_lock.content)
+          rescue JSON::ParserError
+            {}
+          end
+
+          def parsed_yarn_lock
+            return {} unless yarn_lock
+
+            @parsed_yarn_lock ||=
+              SharedHelpers.in_a_temporary_directory do
+                File.write("yarn.lock", yarn_lock.content)
+
+                SharedHelpers.run_helper_subprocess(
+                  command: "node #{yarn_helper_path}",
+                  function: "parseLockfile",
+                  args: [Dir.pwd]
+                )
+              end
+          end
+
+          def yarn_helper_path
+            project_root = File.join(File.dirname(__FILE__), "../../../../..")
+            File.join(project_root, "helpers/yarn/bin/run.js")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/file_fetchers/php/composer.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require "dependabot/file_fetchers/base"
+
+module Dependabot
+  module FileFetchers
+    module Php
+      class Composer < Dependabot::FileFetchers::Base
+        def self.required_files_in?(filenames)
+          filenames.include?("composer.json")
+        end
+
+        def self.required_files_message
+          "Repo must contain a composer.json."
+        end
+
+        private
+
+        def fetch_files
+          fetched_files = []
+          fetched_files << composer_json
+          fetched_files << composer_lock if composer_lock
+          fetched_files << auth_json if auth_json
+          fetched_files += path_dependencies
+          fetched_files
+        end
+
+        def composer_json
+          @composer_json ||= fetch_file_from_host("composer.json")
+        end
+
+        def composer_lock
+          return @composer_lock if @composer_lock_lookup_attempted
+
+          @composer_lock_lookup_attempted = true
+          @composer_lock ||= fetch_file_if_present("composer.lock")
+        end
+
+        # Note: This is fetched but currently unused
+        def auth_json
+          @auth_json ||= fetch_file_if_present("auth.json")&.
+                         tap { |f| f.support_file = true }
+        end
+
+        def path_dependencies
+          @path_dependencies ||=
+            begin
+              composer_json_files = []
+              unfetchable_deps = []
+
+              path_sources.each do |path|
+                directories = path.end_with?("*") ? expand_path(path) : [path]
+
+                directories.each do |dir|
+                  file = File.join(dir, "composer.json")
+
+                  begin
+                    composer_json_files << fetch_file_with_root_fallback(file)
+                  rescue Dependabot::DependencyFileNotFound
+                    # Collected, but currently ignored
+                    unfetchable_deps << file
+                  end
+                end
+              end
+
+              # Mark the path dependencies as support files - we don't currently
+              # parse or update them.
+              composer_json_files.tap do |files|
+                files.each { |f| f.support_file = true }
+              end
+            end
+        end
+
+        def path_sources
+          @path_sources ||=
+            JSON.parse(composer_json.content).
+            fetch("repositories", []).
+            select { |details| details["type"] == "path" }.
+            map { |details| details["url"] }
+        rescue JSON::ParserError
+          raise Dependabot::DependencyFileNotParseable, composer_json.path
+        end
+
+        def expand_path(path)
+          repo_contents(dir: path.gsub(/\*$/, "")).
+            select { |file| file.type == "dir" }.
+            map { |f| path.gsub(/\*$/, f.name) }
+        rescue Octokit::NotFound, Gitlab::Error::NotFound
+          # If there's no lockfile, or if none of the dependencies are path
+          # dependencies, then we can ignore failures to find path deps
+          return [] unless composer_lock&.content&.include?('"path"')
+
+          # Otherwise, we don't know what to do. For now, just raise. If we see
+          # this in the wild we can make a call on the correct handling
+          raise if directory == "/"
+
+          # If the directory isn't found at the full path, try looking for it
+          # at the root of the repository.
+          depth = directory.gsub(%r{^/}, "").gsub(%r{/$}, "").split("/").count
+          dir = "../" * depth + path.gsub(/\*$/, "")
+
+          repo_contents(dir: dir).
+            select { |file| file.type == "dir" }.
+            map { |f| path.gsub(/\*$/, f.name) }
+        end
+
+        def fetch_file_with_root_fallback(filename, type: "file")
+          path = Pathname.new(File.join(directory, filename)).cleanpath.to_path
+
+          begin
+            fetch_file_from_host(filename, type: type)
+          rescue Dependabot::DependencyFileNotFound
+            # If the file isn't found at the full path, try looking for it
+            # without considering the directory (i.e., check if the path should
+            # have been relevative to the root of the repository).
+            cleaned_filename = Pathname.new(filename).cleanpath.to_path
+
+            DependencyFile.new(
+              name: cleaned_filename,
+              content: fetch_file_content(cleaned_filename),
+              directory: directory,
+              type: type
+            )
+          end
+        rescue Octokit::NotFound, Gitlab::Error::NotFound
+          raise Dependabot::DependencyFileNotFound, path
+        end
+      end
+    end
+  end
+end