dependabot-npm_and_yarn 0.91.0

data/helpers/yarn/.eslintrc

@@ -0,0 +1,14 @@
+{
+  "plugins": [
+    "prettier"
+  ],
+  "rules": {
+    "prettier/prettier": "error"
+  },
+  "env": {
+    "node": true
+  },
+  "parserOptions": {
+    "ecmaVersion": 8
+  }
+}

data/helpers/yarn/bin/run.js

@@ -0,0 +1,36 @@
+const lockfileParser = require("../lib/lockfile-parser");
+const updater = require("../lib/updater");
+const subdependencyUpdater = require("../lib/subdependency-updater");
+const peerDependencyChecker = require("../lib/peer-dependency-checker");
+
+const functionMap = {
+  parseLockfile: lockfileParser.parse,
+  update: updater.updateDependencyFiles,
+  updateSubdependency: subdependencyUpdater.updateDependencyFile,
+  checkPeerDependencies: peerDependencyChecker.checkPeerDependencies
+};
+
+function output(obj) {
+  process.stdout.write(JSON.stringify(obj));
+}
+
+const input = [];
+process.stdin.on("data", data => input.push(data));
+process.stdin.on("end", () => {
+  const request = JSON.parse(input.join(""));
+  const func = functionMap[request.function];
+  if (!func) {
+    output({ error: `Invalid function ${request.function}` });
+    process.exit(1);
+  }
+
+  func
+    .apply(null, request.args)
+    .then(result => {
+      output({ result: result });
+    })
+    .catch(error => {
+      output({ error: error.message });
+      process.exit(1);
+    });
+});

data/helpers/yarn/lib/fix-duplicates.js

@@ -0,0 +1,78 @@
+const parse = require("@dependabot/yarn-lib/lib/lockfile/parse").default;
+const stringify = require("@dependabot/yarn-lib/lib/lockfile/stringify")
+  .default;
+const semver = require("semver");
+
+function flattenIndirectDependencies(packages) {
+  return (packages || []).reduce((acc, { pkg }) => {
+    if ("dependencies" in pkg) {
+      return acc.concat(Object.keys(pkg.dependencies));
+    }
+    return acc;
+  }, []);
+}
+
+// Inspired by yarn-deduplicate. Altered to ensure the latest version is always used
+// for version ranges which allow it.
+module.exports = (data, updatedDependencyName) => {
+  if (!updatedDependencyName) {
+    throw new Error("Yarn fix duplicates: must provide dependency name");
+  }
+
+  const json = parse(data).object;
+  const enableLockfileVersions = Boolean(data.match(/^# yarn v/m));
+  const noHeader = !Boolean(data.match(/^# THIS IS AN AU/m));
+
+  const packages = {};
+  const re = /^(.*)@([^@]*?)$/;
+
+  Object.entries(json).forEach(([name, pkg]) => {
+    if (name.match(re)) {
+      const [_, packageName, requestedVersion] = name.match(re);
+      packages[packageName] = packages[packageName] || [];
+      packages[packageName].push(
+        Object.assign({}, { name, pkg, packageName, requestedVersion })
+      );
+    }
+  });
+
+  const packageEntries = Object.entries(packages);
+
+  const updatedPackageEntry = packageEntries.filter(([name]) => {
+    return updatedDependencyName === name;
+  });
+
+  const updatedDependencyPackage =
+    updatedPackageEntry[0] && updatedPackageEntry[0][1];
+
+  const indirectDependencies = flattenIndirectDependencies(
+    updatedDependencyPackage
+  );
+
+  const packagesToDedupe = [updatedDependencyName, ...indirectDependencies];
+
+  packageEntries
+    .filter(([name]) => packagesToDedupe.includes(name))
+    .forEach(([name, packages]) => {
+      // Reverse sort, so we'll find the maximum satisfying version first
+      const versions = packages.map(p => p.pkg.version).sort(semver.rcompare);
+      const ranges = packages.map(p => p.requestedVersion);
+
+      // Dedup each package to its maxSatisfying version
+      packages.forEach(p => {
+        const targetVersion = semver.maxSatisfying(
+          versions,
+          p.requestedVersion
+        );
+        if (targetVersion === null) return;
+        if (targetVersion !== p.pkg.version) {
+          const dedupedPackage = packages.find(
+            p => p.pkg.version === targetVersion
+          );
+          json[`${name}@${p.requestedVersion}`] = dedupedPackage.pkg;
+        }
+      });
+    });
+
+  return stringify(json, noHeader, enableLockfileVersions);
+};

data/helpers/yarn/lib/helpers.js

@@ -0,0 +1,5 @@
+function isString(value) {
+  return Object.prototype.toString.call(value) === "[object String]";
+}
+
+module.exports = { isString };

data/helpers/yarn/lib/lockfile-parser.js

@@ -0,0 +1,21 @@
+/* YARN.LOCK PARSER
+ *
+ * Inputs:
+ *  - directory containing a yarn.lock
+ *
+ * Outputs:
+ *  - JSON formatted yarn.lock
+ */
+const fs = require("fs");
+const path = require("path");
+const parseLockfile = require("@dependabot/yarn-lib/lib/lockfile/parse")
+  .default;
+
+async function parse(directory) {
+  const readFile = fileName =>
+    fs.readFileSync(path.join(directory, fileName)).toString();
+  const data = readFile("yarn.lock");
+  return parseLockfile(data).object;
+}
+
+module.exports = { parse };

data/helpers/yarn/lib/peer-dependency-checker.js

@@ -0,0 +1,130 @@
+/* PEER DEPENDENCY CHECKER
+ *
+ * Inputs:
+ *  - directory containing a package.json and a yarn.lock
+ *  - dependency name
+ *  - new dependency version
+ *  - requirements for this dependency
+ *
+ * Outputs:
+ *  - successful completion, or an error if there are peer dependency warnings
+ */
+const path = require("path");
+const { Add } = require("@dependabot/yarn-lib/lib/cli/commands/add");
+const Config = require("@dependabot/yarn-lib/lib/config").default;
+const { BufferReporter } = require("@dependabot/yarn-lib/lib/reporters");
+const Lockfile = require("@dependabot/yarn-lib/lib/lockfile").default;
+const { isString } = require("./helpers");
+const fetcher = require("@dependabot/yarn-lib/lib/package-fetcher.js");
+
+// Check peer dependencies without downloading node_modules or updating
+// package/lockfiles
+//
+// Logic copied from the import command
+class LightweightAdd extends Add {
+  async bailout() {
+    const manifests = await fetcher.fetch(
+      this.resolver.getManifests(),
+      this.config
+    );
+    this.resolver.updateManifests(manifests);
+    await this.linker.resolvePeerModules();
+    return true;
+  }
+}
+
+function devRequirement(requirements) {
+  const groups = requirements.groups;
+  return (
+    groups.indexOf("devDependencies") > -1 &&
+    groups.indexOf("dependencies") == -1
+  );
+}
+
+function optionalRequirement(requirements) {
+  const groups = requirements.groups;
+  return (
+    groups.indexOf("optionalDependencies") > -1 &&
+    groups.indexOf("dependencies") == -1
+  );
+}
+
+function installArgsWithVersion(depName, desiredVersion, requirements) {
+  const source =
+    "source" in requirements
+      ? requirements.source
+      : (requirements.find(req => req.source) || {}).source;
+  const req =
+    "requirement" in requirements
+      ? requirements.requirement
+      : (requirements.find(req => req.requirement) || {}).requirement;
+
+  if (source && source.type === "git") {
+    if (desiredVersion) {
+      return [`${depName}@${source.url}#${desiredVersion}`];
+    } else {
+      return [`${depName}@${source.url}`];
+    }
+  } else {
+    return [`${depName}@${desiredVersion || req}`];
+  }
+}
+
+async function checkPeerDependencies(
+  directory,
+  depName,
+  desiredVersion,
+  requirements
+) {
+  for (let req of requirements) {
+    await checkPeerDepsForReq(directory, depName, desiredVersion, req);
+  }
+}
+
+async function checkPeerDepsForReq(
+  directory,
+  depName,
+  desiredVersion,
+  requirement
+) {
+  const flags = {
+    ignoreScripts: true,
+    ignoreWorkspaceRootCheck: true,
+    ignoreEngines: true,
+    dev: devRequirement(requirement),
+    optional: optionalRequirement(requirement)
+  };
+  const reporter = new BufferReporter();
+  const config = new Config(reporter);
+
+  await config.init({
+    cwd: path.join(directory, path.dirname(requirement.file)),
+    nonInteractive: true,
+    enableDefaultRc: true
+  });
+
+  const lockfile = await Lockfile.fromDirectory(directory, reporter);
+
+  // Returns dep name and version for yarn add, example: ["react@16.6.0"]
+  let args = installArgsWithVersion(depName, desiredVersion, requirement);
+
+  // Just as if we'd run `yarn add package@version`, but using our lightweight
+  // implementation of Add that doesn't actually download and install packages
+  const add = new LightweightAdd(args, flags, config, reporter, lockfile);
+
+  await add.init();
+
+  const eventBuffer = reporter.getBuffer();
+  const peerDependencyWarnings = eventBuffer
+    .map(({ data }) => data)
+    .filter(data => {
+      // Guard against event.data sometimes being an object
+      return isString(data) && data.match(/(unmet|incorrect) peer dependency/);
+    });
+
+  if (peerDependencyWarnings.length) {
+    throw new Error(peerDependencyWarnings.join("\n"));
+  }
+}
+
+module.exports = { checkPeerDependencies };

data/helpers/yarn/lib/replace-lockfile-declaration.js

@@ -0,0 +1,57 @@
+const parse = require("@dependabot/yarn-lib/lib/lockfile/parse").default;
+const stringify = require("@dependabot/yarn-lib/lib/lockfile/stringify")
+  .default;
+
+// Get an array of a dependency's requested version ranges from a lockfile
+function getRequestedVersions(depName, lockfileJson) {
+  const requestedVersions = [];
+  // TODO: Rethink this regex matching, for example, we don't currently match:
+  // @dependabot/pack-core@^git+ssh://git@github.com:dependabot/pack-core.git
+  const re = /^(.*)@([^@]*?)$/;
+
+  Object.entries(lockfileJson).forEach(([name, _]) => {
+    if (name.match(re)) {
+      const [_, packageName, requestedVersion] = name.match(re);
+      if (packageName === depName) {
+        requestedVersions.push(requestedVersion);
+      }
+    }
+  });
+
+  return requestedVersions;
+}
+
+module.exports = (
+  oldLockfileContent,
+  newLockfileContent,
+  depName,
+  newVersionRequirement,
+  existingVersionRequirement
+) => {
+  const oldJson = parse(oldLockfileContent).object;
+  const newJson = parse(newLockfileContent).object;
+
+  const enableLockfileVersions = Boolean(
+    oldLockfileContent.match(/^# yarn v/m)
+  );
+  const noHeader = !Boolean(oldLockfileContent.match(/^# THIS IS AN AU/m));
+
+  const oldPackageReqs = getRequestedVersions(depName, oldJson);
+  const newPackageReqs = getRequestedVersions(depName, newJson);
+
+  const reqToReplace = newPackageReqs.find(pattern => {
+    return !oldPackageReqs.includes(pattern);
+  });
+
+  // If the new lockfile has entries that don't exist in the old lockfile,
+  // replace these version requirements with a range (will currently be an
+  // exact version because we tell yarn to install a specific version)
+  if (reqToReplace) {
+    newJson[
+      `${depName}@${newVersionRequirement || existingVersionRequirement}`
+    ] = newJson[`${depName}@${reqToReplace}`];
+    delete newJson[`${depName}@${reqToReplace}`];
+  }
+
+  return stringify(newJson, noHeader, enableLockfileVersions);
+};

data/helpers/yarn/lib/subdependency-updater.js

@@ -0,0 +1,69 @@
+/* DEPENDENCY FILE UPDATER
+ *
+ * Inputs:
+ *  - directory containing a package.json and a yarn.lock
+ *  - dependency name
+ *
+ * Outputs:
+ *  - yarn.lock file
+ *
+ * Update the sub-dependency versions for this dependency to that latest
+ * possible versions, without unlocking any other dependencies
+ */
+const fs = require("fs");
+const path = require("path");
+const { Install } = require("@dependabot/yarn-lib/lib/cli/commands/install");
+const Config = require("@dependabot/yarn-lib/lib/config").default;
+const { EventReporter } = require("@dependabot/yarn-lib/lib/reporters");
+const Lockfile = require("@dependabot/yarn-lib/lib/lockfile").default;
+
+class LightweightInstall extends Install {
+  async bailout(patterns, workspaceLayout) {
+    await this.saveLockfileAndIntegrity(patterns, workspaceLayout);
+    return true;
+  }
+}
+
+// Replace the version comments in the new lockfile with the ones from the old
+// lockfile. If they weren't present in the old lockfile, delete them.
+function recoverVersionComments(oldLockfile, newLockfile) {
+  const yarnRegex = /^# yarn v(\S+)\n/gm;
+  const nodeRegex = /^# node v(\S+)\n/gm;
+  const oldMatch = regex => [].concat(oldLockfile.match(regex))[0];
+  return newLockfile
+    .replace(yarnRegex, () => oldMatch(yarnRegex) || "")
+    .replace(nodeRegex, () => oldMatch(nodeRegex) || "");
+}
+
+async function updateDependencyFile(directory, lockfileName) {
+  const readFile = fileName =>
+    fs.readFileSync(path.join(directory, fileName)).toString();
+  const originalYarnLock = readFile(lockfileName);
+
+  const flags = {
+    ignoreScripts: true,
+    ignoreWorkspaceRootCheck: true,
+    ignoreEngines: true
+  };
+  const reporter = new EventReporter();
+  const config = new Config(reporter);
+  await config.init({
+    cwd: directory,
+    nonInteractive: true,
+    enableDefaultRc: true
+  });
+  config.enableLockfileVersions = Boolean(originalYarnLock.match(/^# yarn v/m));
+
+  const lockfile = await Lockfile.fromDirectory(directory, reporter);
+  const install = new LightweightInstall(flags, config, reporter, lockfile);
+  await install.init();
+  var updatedYarnLock = readFile(lockfileName);
+
+  updatedYarnLock = recoverVersionComments(originalYarnLock, updatedYarnLock);
+
+  return {
+    [lockfileName]: updatedYarnLock
+  };
+}
+
+module.exports = { updateDependencyFile };

data/helpers/yarn/lib/updater.js

@@ -0,0 +1,266 @@
+/* DEPENDENCY FILE UPDATER
+ *
+ * Inputs:
+ *  - directory containing a package.json and a yarn.lock
+ *  - dependency name
+ *  - new dependency version
+ *  - new requirements for this dependency
+ *
+ * Outputs:
+ *  - updated package.json and yarn.lock files
+ *
+ * Update the dependency to the version specified and rewrite the package.json
+ * and yarn.lock files.
+ */
+const fs = require("fs");
+const path = require("path");
+const { Add } = require("@dependabot/yarn-lib/lib/cli/commands/add");
+const { Install } = require("@dependabot/yarn-lib/lib/cli/commands/install");
+const {
+  cleanLockfile
+} = require("@dependabot/yarn-lib/lib/cli/commands/upgrade");
+const Config = require("@dependabot/yarn-lib/lib/config").default;
+const { EventReporter } = require("@dependabot/yarn-lib/lib/reporters");
+const Lockfile = require("@dependabot/yarn-lib/lib/lockfile").default;
+const parse = require("@dependabot/yarn-lib/lib/lockfile/parse").default;
+const fixDuplicates = require("./fix-duplicates");
+const replaceDeclaration = require("./replace-lockfile-declaration");
+const urlParse = require("url").parse;
+
+// Add is a subclass of the Install CLI command, which is responsible for
+// adding packages to a package.json and yarn.lock. Upgrading a package is
+// exactly the same as adding, except the package already exists in the
+// manifests.
+//
+// Usually, calling Add.init() would execute a series of steps: resolve, fetch,
+// link, run lifecycle scripts, cleanup, then save new manifest (package.json).
+// We only care about the first and last steps: resolve, then save the new
+// manifest. Fotunately, overriding bailout() gives us an opportunity to skip
+// over the intermediate steps in a relatively painless fashion.
+class LightweightAdd extends Add {
+  // This method is called by init() at the end of the resolve step, and is
+  // responsible for checking if any dependnecies need to be updated locally.
+  // If everything is up to date, it'll save a new lockfile and return true,
+  // which causes init() to skip over the next few steps (fetching and
+  // installing packages). If there are packages that need updating, it'll
+  // return false, and init() will continue on to the fetching and installing
+  // steps.
+  //
+  // Add overrides Install's implementation to always return false - meaning
+  // that it will always continue to the fetch and install steps. We want to
+  // do the opposite - just save the new lockfile and stop there.
+  async bailout(patterns, workspaceLayout) {
+    // This is the only part of the original bailout implementation that
+    // matters: saving the new lockfile
+    await this.saveLockfileAndIntegrity(patterns, workspaceLayout);
+
+    // Skip over the unnecessary steps - fetching and linking packages, etc.
+    return true;
+  }
+}
+
+class LightweightInstall extends Install {
+  async bailout(patterns, workspaceLayout) {
+    await this.saveLockfileAndIntegrity(patterns, workspaceLayout);
+    return true;
+  }
+}
+
+async function flattenAllDependencies(config) {
+  const manifest = await config.readRootManifest();
+  return Object.assign(
+    {},
+    manifest.optionalDependencies,
+    manifest.peerDependencies,
+    manifest.devDependencies,
+    manifest.dependencies
+  );
+}
+
+// Replace the version comments in the new lockfile with the ones from the old
+// lockfile. If they weren't present in the old lockfile, delete them.
+function recoverVersionComments(oldLockfile, newLockfile) {
+  const yarnRegex = /^# yarn v(\S+)\n/gm;
+  const nodeRegex = /^# node v(\S+)\n/gm;
+  const oldMatch = regex => [].concat(oldLockfile.match(regex))[0];
+  return newLockfile
+    .replace(yarnRegex, match => oldMatch(yarnRegex) || "")
+    .replace(nodeRegex, match => oldMatch(nodeRegex) || "");
+}
+
+function devRequirement(requirements) {
+  const groups = requirements.groups;
+  return (
+    groups.indexOf("devDependencies") > -1 &&
+    groups.indexOf("dependencies") == -1
+  );
+}
+
+function optionalRequirement(requirements) {
+  const groups = requirements.groups;
+  return (
+    groups.indexOf("optionalDependencies") > -1 &&
+    groups.indexOf("dependencies") == -1
+  );
+}
+
+const GIT_HOSTS = [
+  "github.com",
+  "gitlab.com",
+  "bitbucket.com",
+  "bitbucket.org"
+];
+
+function getGitShorthand(depName, url, lockfileJson) {
+  const { hostname, path } = urlParse(url);
+  const isSupportedHost = hostname && path && GIT_HOSTS.indexOf(hostname) >= 0;
+  if (!isSupportedHost) return;
+
+  const hostShorthand = hostname.replace(/\.[a-z]{3}$/, "");
+  const repoShorthand = path.replace(/^\//, "");
+
+  return [repoShorthand, `${hostShorthand}:${repoShorthand}`].find(
+    shorthand => {
+      return Object.keys(lockfileJson).some(name => {
+        return name.startsWith(`${depName}@${shorthand}`);
+      });
+    }
+  );
+}
+
+function installArgsWithVersion(
+  depName,
+  desiredVersion,
+  requirements,
+  lockfileJson
+) {
+  const source = requirements.source;
+
+  // TODO: Use logic from npm updater to find original version instead of doing
+  // all this mad git shorthand logic
+  // e.g. const originalVersion = flattenAllDependencies(oldPackage)[depName];
+  if (source && source.type === "git") {
+    // Handle packages added using the github shorthand, e.g.
+    // - yarn add discord.js@discordjs/discord.js
+    //
+    // To keep the correct resolved url in the lockfile we need to explicitly
+    // tell yarn to install using the shorthand and not using the git url
+    //
+    // The resolved url from the shorthand case comes from
+    // https://codeload.github.com/org/repo.. whereas it comes from
+    // https://github.com/org/repo.. in the usual git install case:
+    // yarn add https://github.com/org/repo..
+    const repoShortHand = getGitShorthand(depName, source.url, lockfileJson);
+    if (repoShortHand) {
+      return [`${depName}@${repoShortHand}#${desiredVersion}`];
+    } else {
+      return [`${depName}@${source.url}#${desiredVersion}`];
+    }
+  } else {
+    return [`${depName}@${desiredVersion}`];
+  }
+}
+
+async function updateDependencyFiles(directory, dependencies) {
+  const readFile = fileName =>
+    fs.readFileSync(path.join(directory, fileName)).toString();
+  let updateRunResults = { "yarn.lock": readFile("yarn.lock") };
+
+  for (let dep of dependencies) {
+    for (let reqs of dep.requirements) {
+      updateRunResults = Object.assign(
+        updateRunResults,
+        await updateDependencyFile(directory, dep.name, dep.version, reqs)
+      );
+    }
+  }
+
+  return updateRunResults;
+}
+
+async function updateDependencyFile(
+  directory,
+  depName,
+  desiredVersion,
+  requirements
+) {
+  const readFile = fileName =>
+    fs.readFileSync(path.join(directory, fileName)).toString();
+  const originalYarnLock = readFile("yarn.lock");
+  const originalPackageJson = readFile(requirements.file);
+
+  const flags = {
+    ignoreScripts: true,
+    ignoreWorkspaceRootCheck: true,
+    ignoreEngines: true,
+    dev: devRequirement(requirements),
+    optional: optionalRequirement(requirements)
+  };
+  const reporter = new EventReporter();
+  const config = new Config(reporter);
+  await config.init({
+    cwd: path.join(directory, path.dirname(requirements.file)),
+    nonInteractive: true,
+    enableDefaultRc: true
+  });
+  config.enableLockfileVersions = Boolean(originalYarnLock.match(/^# yarn v/m));
+
+  const lockfile = await Lockfile.fromDirectory(directory, reporter);
+
+  // Just as if we'd run `yarn add package@version`, but using our lightweight
+  // implementation of Add that doesn't actually download and install packages
+  const lockfileJson = parse(originalYarnLock).object;
+  const args = installArgsWithVersion(
+    depName,
+    desiredVersion,
+    requirements,
+    lockfileJson
+  );
+
+  const add = new LightweightAdd(args, flags, config, reporter, lockfile);
+
+  // Despite the innocent-sounding name, this actually does all the hard work
+  await add.init();
+
+  const dedupedYarnLock = fixDuplicates(readFile("yarn.lock"), depName);
+
+  const newVersionRequirement = requirements.requirement;
+
+  const flattenedDependencies = await flattenAllDependencies(config);
+  const existingVersionRequirement = flattenedDependencies[depName];
+
+  // Replace the version requirement in the lockfile (which will currently be an
+  // exact version, not a requirement range)
+  // If we don't have new requirement (e.g. git source) use the existing version
+  // requirement from the package manifest
+  const replacedDeclarationYarnLock = replaceDeclaration(
+    originalYarnLock,
+    dedupedYarnLock,
+    depName,
+    newVersionRequirement,
+    existingVersionRequirement
+  );
+
+  // Do a normal install to ensure the lockfile doesn't change when we do
+  fs.writeFileSync(
+    path.join(directory, "yarn.lock"),
+    replacedDeclarationYarnLock
+  );
+  fs.writeFileSync(
+    path.join(directory, requirements.file),
+    originalPackageJson
+  );
+
+  const lockfile2 = await Lockfile.fromDirectory(directory, reporter);
+  const install2 = new LightweightInstall(flags, config, reporter, lockfile2);
+  await install2.init();
+
+  let updatedYarnLock = readFile("yarn.lock");
+  updatedYarnLock = recoverVersionComments(originalYarnLock, updatedYarnLock);
+
+  return {
+    "yarn.lock": updatedYarnLock
+  };
+}
+
+module.exports = { updateDependencyFiles };