dependabot-npm_and_yarn 0.91.8 → 0.92.2

data/helpers/yarn/bin/run.js

@@ -1,14 +1,6 @@
-const lockfileParser = require("../lib/lockfile-parser");
-const updater = require("../lib/updater");
-const subdependencyUpdater = require("../lib/subdependency-updater");
-const peerDependencyChecker = require("../lib/peer-dependency-checker");
+#!/usr/bin/env node
 
-const functionMap = {
-  parseLockfile: lockfileParser.parse,
-  update: updater.updateDependencyFiles,
-  updateSubdependency: subdependencyUpdater.updateDependencyFile,
-  checkPeerDependencies: peerDependencyChecker.checkPeerDependencies
-};
+const functionMap = require("../lib");
 
 function output(obj) {
   process.stdout.write(JSON.stringify(obj));

data/helpers/yarn/lib/index.js

@@ -0,0 +1,11 @@
+const lockfileParser = require("../lib/lockfile-parser");
+const updater = require("../lib/updater");
+const subdependencyUpdater = require("../lib/subdependency-updater");
+const peerDependencyChecker = require("../lib/peer-dependency-checker");
+
+module.exports = {
+  parseLockfile: lockfileParser.parse,
+  update: updater.updateDependencyFiles,
+  updateSubdependency: subdependencyUpdater.updateDependencyFile,
+  checkPeerDependencies: peerDependencyChecker.checkPeerDependencies
+};

data/helpers/yarn/package.json

@@ -1,17 +1,12 @@
 {
-  "name": "dependabot-core",
+  "name": "@dependabot/yarn",
   "version": "0.0.0",
   "private": true,
   "dependencies": {
-    "@dependabot/yarn-lib": "1.13.0",
-    "semver": "5.6.0"
+    "@dependabot/yarn-lib": "^1.13.0",
+    "semver": "^5.6.0"
   },
-  "devDependencies": {
-    "eslint": "5.12.1",
-    "eslint-plugin-prettier": "3.0.1",
-    "fs-extra": "7.0.1",
-    "jest": "23.6.0",
-    "nock": "10.0.6",
-    "prettier": "1.16.0"
+  "bin": {
+    "dependabot-yarn": "./bin/run.js"
   }
 }

data/helpers/yarn/test/helpers.js

@@ -3,5 +3,5 @@ const fs = require("fs");
 
 module.exports = {
   loadFixture: fixturePath =>
-    fs.readFileSync(path.join("test", "fixtures", fixturePath)).toString()
+    fs.readFileSync(path.join(__dirname, "fixtures", fixturePath)).toString()
 };

data/helpers/yarn/test/updater.test.js

@@ -23,10 +23,16 @@ describe("updater", () => {
   afterEach(() => fs.removeSync(tempDir));
 
   async function copyDependencies(sourceDir, destDir) {
-    const srcPackageJson = `test/fixtures/updater/${sourceDir}/package.json`;
+    const srcPackageJson = path.join(
+      __dirname,
+      `fixtures/updater/${sourceDir}/package.json`
+    );
     await fs.copy(srcPackageJson, `${destDir}/package.json`);
 
-    const srcYarnLock = `test/fixtures/updater/${sourceDir}/yarn.lock`;
+    const srcYarnLock = path.join(
+      __dirname,
+      `fixtures/updater/${sourceDir}/yarn.lock`
+    );
     await fs.copy(srcYarnLock, `${destDir}/yarn.lock`);
   }
 

data/lib/dependabot/npm_and_yarn/file_updater/npm_lockfile_updater.rb

@@ -48,8 +48,8 @@ module Dependabot
         UNREACHABLE_GIT =
           /ls-remote (?:(-h -t)|(--tags --heads)) (?<url>.*)/.freeze
         FORBIDDEN_PACKAGE =
-          /(403 Forbidden|401 Unauthorized): (?<package_req>.*)/.freeze
-        MISSING_PACKAGE = /404 Not Found: (?<package_req>.*)/.freeze
+          %r{(?<package_req>[^/]+) - (Forbidden|Unauthorized)}.freeze
+        MISSING_PACKAGE = %r{(?<package_req>[^/]+) - Not found}.freeze
         INVALID_PACKAGE = /Can't install (?<package_req>.*): Missing/.freeze
 
         def top_level_dependencies
@@ -69,12 +69,19 @@ module Dependabot
           end
         end
 
+        def lockfile_dependencies(lockfile)
+          @lockfile_dependencies ||= {}
+          @lockfile_dependencies[lockfile.name] ||=
+            NpmAndYarn::FileParser.new(
+              dependency_files: [lockfile, *package_files],
+              source: nil,
+              credentials: credentials
+            ).parse
+        end
+
         def dependency_up_to_date?(lockfile, dependency)
-          existing_dep = NpmAndYarn::FileParser.new(
-            dependency_files: [lockfile, *package_files],
-            source: nil,
-            credentials: credentials
-          ).parse.find { |dep| dep.name == dependency.name }
+          existing_dep = lockfile_dependencies(lockfile).
+                         find { |dep| dep.name == dependency.name }
 
           # If the dependency is missing but top level it should be treated as
           # not up to date
@@ -167,19 +174,9 @@ module Dependabot
             package_name =
               error.message.match(MISSING_PACKAGE).
               named_captures["package_req"].
-              split(/(?<=\w)\@/).first
+              gsub("%2f", "/")
             handle_missing_package(package_name, error, lockfile)
           end
-          names = dependencies.map(&:name)
-          if names.any? { |name| error.message.include?("#{name}@") } &&
-             error.message.start_with?("No matching vers") &&
-             resolvable_before_update?(lockfile)
-            # This happens if a new version has been published that relies on
-            # but npm is having consistency issues. We raise a bespoke error
-            # so we can capture and ignore it if we're trying to create a new
-            # PR (which will be created successfully at a later date).
-            raise Dependabot::InconsistentRegistryResponse, error.message
-          end
 
           # When the package.json doesn't include a name or version, or name
           # has non url-friendly characters
@@ -188,24 +185,43 @@ module Dependabot
             raise_resolvability_error(error, lockfile)
           end
 
-          if error.message.start_with?("No matching vers", "404 Not Found") ||
-             error.message.include?("not match any file(s) known to git") ||
-             error.message.include?("Non-registry package missing package") ||
-             error.message.include?("Cannot read property 'match' of ") ||
-             error.message.include?("Invalid tag name")
-            # This happens if a new version has been published that relies on
-            # subdependencies that have not yet been published.
-            raise if resolvable_before_update?(lockfile)
+          # TODO: Move this logic to the version resolver and check if a new
+          # version and all of its subdependencies are resolvable
+
+          # Make sure the error in question matches the current list of
+          # dependencies or matches an existing scoped package, this handles the
+          # case where a new version (e.g. @angular-devkit/build-angular) relies
+          # on a added dependency which hasn't been published yet under the same
+          # scope (e.g. @angular-devkit/build-optimizer)
+          #
+          # This seems to happen when big monorepo projects publish all of their
+          # packages sequentially, which might take enough time for Dependabot
+          # to hear about a new version before all of its dependencies have been
+          # published
+          #
+          # OR
+          #
+          # This happens if a new version has been published but npm is having
+          # consistency issues and the version isn't fully available on all
+          # queries
+          if error.message.start_with?("No matching vers") &&
+             dependencies_in_error_message?(error.message) &&
+             resolvable_before_update?(lockfile)
 
-            raise_resolvability_error(error, lockfile)
+            # Raise a bespoke error so we can capture and ignore it if
+            # we're trying to create a new PR (which will be created
+            # successfully at a later date)
+            raise Dependabot::InconsistentRegistryResponse, error.message
           end
+
           if error.message.match?(FORBIDDEN_PACKAGE)
             package_name =
               error.message.match(FORBIDDEN_PACKAGE).
               named_captures["package_req"].
-              split(/(?<=\w)\@/).first
+              gsub("%2f", "/")
             handle_missing_package(package_name, error, lockfile)
           end
+
           if error.message.match?(UNREACHABLE_GIT)
             dependency_url =
               error.message.match(UNREACHABLE_GIT).
@@ -213,7 +229,23 @@ module Dependabot
 
             raise Dependabot::GitDependenciesNotReachable, dependency_url
           end
-          raise
+
+          if error.message.start_with?("No matching vers", "404 Not Found") ||
+             error.message.include?("not match any file(s) known to git") ||
+             error.message.include?("Non-registry package missing package") ||
+             error.message.include?("Cannot read property 'match' of ") ||
+             error.message.include?("Invalid tag name")
+
+            unless resolvable_before_update?(lockfile)
+              raise_resolvability_error(error, lockfile)
+            end
+
+            # Dependabot has probably messed something up with the update and we
+            # want to hear about it
+            raise error
+          end
+
+          raise error
         end
         # rubocop:enable Metrics/AbcSize
         # rubocop:enable Metrics/CyclomaticComplexity
@@ -228,11 +260,8 @@ module Dependabot
         end
 
         def handle_missing_package(package_name, error, lockfile)
-          missing_dep = NpmAndYarn::FileParser.new(
-            dependency_files: dependency_files,
-            source: nil,
-            credentials: credentials
-          ).parse.find { |dep| dep.name == package_name }
+          missing_dep = lockfile_dependencies(lockfile).
+                        find { |dep| dep.name == package_name }
 
           raise_resolvability_error(error, lockfile) unless missing_dep
 
@@ -283,6 +312,15 @@ module Dependabot
             end
         end
 
+        def dependencies_in_error_message?(message)
+          names = dependencies.map { |dep| dep.name.split("/").first }
+          # Example foramt: No matching version found for
+          # @dependabot/dummy-pkg-b@^1.3.0
+          names.any? do |name|
+            message.match?(%r{#{Regexp.quote(name)}[\/@]})
+          end
+        end
+
         def write_temporary_dependency_files(lockfile_name,
                                              update_package_json: true)
           write_lockfiles(lockfile_name)

data/lib/dependabot/npm_and_yarn/file_updater/yarn_lockfile_updater.rb

@@ -177,34 +177,46 @@ module Dependabot
             package_name =
               error.message.match(/package "(?<package_req>.*)?"/).
               named_captures["package_req"].
-              split(/(?<=\w)\@/).first
+              split(/(?<=\w)\@/).first.
+              gsub("%2f", "/")
             handle_missing_package(package_name, error, yarn_lock)
           end
 
           if error.message.match?(%r{/[^/]+: Not found})
             package_name =
               error.message.match(%r{/(?<package_name>[^/]+): Not found}).
-              named_captures["package_name"]
+              named_captures["package_name"].
+              gsub("%2f", "/")
             handle_missing_package(package_name, error, yarn_lock)
           end
 
-          if error.message.start_with?("Couldn't find any versions") ||
-             error.message.include?(": Not found")
-
-            names = dependencies.map(&:name)
-            if names.any? { |name| error.message.include?(%("#{name}")) }
-              # This happens if a new version has been published but npm is
-              # having consistency issues. We raise a bespoke error so we can
-              # capture and ignore it if we're trying to create a new PR
-              # (which will be created successfully at a later date).
-              raise Dependabot::InconsistentRegistryResponse, error.message
-            end
-
-            # This happens if a new version has been published that relies on
-            # subdependencies that have not yet been published.
-            raise error if resolvable_before_update?(yarn_lock)
-
-            raise_resolvability_error(error, yarn_lock)
+          # TODO: Move this logic to the version resolver and check if a new
+          # version and all of its subdependencies are resolvable
+
+          # Make sure the error in question matches the current list of
+          # dependencies or matches an existing scoped package, this handles the
+          # case where a new version (e.g. @angular-devkit/build-angular) relies
+          # on a added dependency which hasn't been published yet under the same
+          # scope (e.g. @angular-devkit/build-optimizer)
+          #
+          # This seems to happen when big monorepo projects publish all of their
+          # packages sequentially, which might take enough time for Dependabot
+          # to hear about a new version before all of its dependencies have been
+          # published
+          #
+          # OR
+          #
+          # This happens if a new version has been published but npm is having
+          # consistency issues and the version isn't fully available on all
+          # queries
+          if error.message.start_with?("Couldn't find any versions") &&
+             dependencies_in_error_message?(error.message) &&
+             resolvable_before_update?(yarn_lock)
+
+            # Raise a bespoke error so we can capture and ignore it if
+            # we're trying to create a new PR (which will be created
+            # successfully at a later date)
+            raise Dependabot::InconsistentRegistryResponse, error.message
           end
 
           if error.message.include?("Workspaces can only be enabled in priva")
@@ -219,10 +231,22 @@ module Dependabot
           end
 
           if error.message.match?(TIMEOUT_FETCHING_PACKAGE)
-            handle_timeout(error.message)
+            handle_timeout(error.message, yarn_lock)
           end
 
-          raise
+          if error.message.start_with?("Couldn't find any versions") ||
+             error.message.include?(": Not found")
+
+            unless resolvable_before_update?(yarn_lock)
+              raise_resolvability_error(error, yarn_lock)
+            end
+
+            # Dependabot has probably messed something up with the update and we
+            # want to hear about it
+            raise error
+          end
+
+          raise error
         end
         # rubocop:enable Metrics/AbcSize
         # rubocop:enable Metrics/CyclomaticComplexity
@@ -230,16 +254,34 @@ module Dependabot
         # rubocop:enable Metrics/MethodLength
 
         def resolvable_before_update?(yarn_lock)
-          SharedHelpers.in_a_temporary_directory do
-            write_temporary_dependency_files(update_package_json: false)
-            lockfile_name = Pathname.new(yarn_lock.name).basename.to_s
-            path = Pathname.new(yarn_lock.name).dirname.to_s
-            run_previous_yarn_update(path: path, lockfile_name: lockfile_name)
+          @resolvable_before_update ||= {}
+          if @resolvable_before_update.key?(yarn_lock.name)
+            return @resolvable_before_update[yarn_lock.name]
           end
 
-          true
-        rescue SharedHelpers::HelperSubprocessFailed
-          false
+          @resolvable_before_update[yarn_lock.name] =
+            begin
+              SharedHelpers.in_a_temporary_directory do
+                write_temporary_dependency_files(update_package_json: false)
+                lockfile_name = Pathname.new(yarn_lock.name).basename.to_s
+                path = Pathname.new(yarn_lock.name).dirname.to_s
+                run_previous_yarn_update(path: path,
+                                         lockfile_name: lockfile_name)
+              end
+
+              true
+            rescue SharedHelpers::HelperSubprocessFailed
+              false
+            end
+        end
+
+        def dependencies_in_error_message?(message)
+          names = dependencies.map { |dep| dep.name.split("/").first }
+          # Example format: Couldn't find any versions for
+          # "@dependabot/dummy-pkg-b" that matches "^1.3.0"
+          names.any? do |name|
+            message.match?(%r{"#{Regexp.quote(name)}["\/]})
+          end
         end
 
         def write_temporary_dependency_files(update_package_json: true)
@@ -368,12 +410,19 @@ module Dependabot
           content.lines.reject { |l| l.match?(/\s*integrity sha/) }.join
         end
 
+        def lockfile_dependencies(lockfile)
+          @lockfile_dependencies ||= {}
+          @lockfile_dependencies[lockfile.name] ||=
+            NpmAndYarn::FileParser.new(
+              dependency_files: [lockfile, *package_files],
+              source: nil,
+              credentials: credentials
+            ).parse
+        end
+
         def handle_missing_package(package_name, error, yarn_lock)
-          missing_dep = NpmAndYarn::FileParser.new(
-            dependency_files: dependency_files,
-            source: nil,
-            credentials: credentials
-          ).parse.find { |dep| dep.name == package_name }
+          missing_dep = lockfile_dependencies(yarn_lock).
+                        find { |dep| dep.name == package_name }
 
           raise_resolvability_error(error, yarn_lock) unless missing_dep
 
@@ -406,7 +455,7 @@ module Dependabot
           raise Dependabot::DependencyFileNotResolvable, msg
         end
 
-        def handle_timeout(message)
+        def handle_timeout(message, yarn_lock)
           url = message.match(TIMEOUT_FETCHING_PACKAGE).named_captures["url"]
           return if url.start_with?("https://registry.npmjs.org")
 
@@ -414,11 +463,8 @@ module Dependabot
             message.match(TIMEOUT_FETCHING_PACKAGE).
             named_captures["package"].gsub("%2f", "/").gsub("%2F", "/")
 
-          dep = NpmAndYarn::FileParser.new(
-            dependency_files: dependency_files,
-            source: nil,
-            credentials: credentials
-          ).parse.find { |d| d.name == package_name }
+          dep = lockfile_dependencies(yarn_lock).
+                find { |d| d.name == package_name }
           return unless dep
 
           raise PrivateSourceTimedOut, url.gsub(%r{https?://}, "")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dependabot-npm_and_yarn
 version: !ruby/object:Gem::Version
-  version: 0.91.8
+  version: 0.92.2
 platform: ruby
 authors:
 - Dependabot
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-01-20 00:00:00.000000000 Z
+date: 2019-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dependabot-core
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.91.8
+        version: 0.92.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.91.8
+        version: 0.92.2
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -143,14 +143,14 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- helpers/.eslintrc
 - helpers/build
-- helpers/npm/.eslintrc
 - helpers/npm/bin/run.js
 - helpers/npm/lib/helpers.js
+- helpers/npm/lib/index.js
 - helpers/npm/lib/peer-dependency-checker.js
 - helpers/npm/lib/subdependency-updater.js
 - helpers/npm/lib/updater.js
-- helpers/npm/package-lock.json
 - helpers/npm/package.json
 - helpers/npm/test/fixtures/npm-left-pad.json
 - helpers/npm/test/fixtures/updater/original/package-lock.json
@@ -158,11 +158,12 @@ files:
 - helpers/npm/test/fixtures/updater/updated/package-lock.json
 - helpers/npm/test/helpers.js
 - helpers/npm/test/updater.test.js
-- helpers/npm/yarn.lock
-- helpers/yarn/.eslintrc
+- helpers/package.json
+- helpers/yarn.lock
 - helpers/yarn/bin/run.js
 - helpers/yarn/lib/fix-duplicates.js
 - helpers/yarn/lib/helpers.js
+- helpers/yarn/lib/index.js
 - helpers/yarn/lib/lockfile-parser.js
 - helpers/yarn/lib/peer-dependency-checker.js
 - helpers/yarn/lib/replace-lockfile-declaration.js
@@ -178,7 +179,6 @@ files:
 - helpers/yarn/test/fixtures/yarnpkg-left-pad.json
 - helpers/yarn/test/helpers.js
 - helpers/yarn/test/updater.test.js
-- helpers/yarn/yarn.lock
 - lib/dependabot/npm_and_yarn.rb
 - lib/dependabot/npm_and_yarn/dependency_files_filterer.rb
 - lib/dependabot/npm_and_yarn/file_fetcher.rb