shipit-engine 0.30.0 → 0.34.0

data/lib/shipit/simple_message_verifier.rb

@@ -1,8 +1,9 @@
+# frozen_string_literal: true
 module Shipit
   class SimpleMessageVerifier < ActiveSupport::MessageVerifier
-    def initialize(secret, options = {})
+    def initialize(secret, **options)
       options[:serializer] ||= ToS
-      super(secret, options)
+      super(secret, **options)
     end
 
     private

data/lib/shipit/stack_commands.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+# rubocop:disable Lint/MissingSuper
 require 'pathname'
 require 'fileutils'
 
@@ -7,26 +9,36 @@ module Shipit
       @stack = stack
     end
 
+    def env
+      super.merge(@stack.env)
+    end
+
     def fetch
       create_directories
-      if Dir.exist?(@stack.git_path)
+      if valid_git_repository?(@stack.git_path)
         git('fetch', 'origin', '--tags', @stack.branch, env: env, chdir: @stack.git_path)
       else
+        @stack.clear_git_cache!
         git_clone(@stack.repo_git_url, @stack.git_path, branch: @stack.branch, env: env, chdir: @stack.deploys_path)
       end
     end
 
     def fetched?(commit)
-      git_dir = File.join(@stack.git_path, '.git')
-      if Dir.exist?(git_dir)
+      if valid_git_repository?(@stack.git_path)
         git('rev-parse', '--quiet', '--verify', "#{commit.sha}^{commit}", env: env, chdir: @stack.git_path)
       else
-        Command.new('test', '-d', git_dir, env: env, chdir: @stack.deploys_path)
+        # When the stack's git cache is not valid, the commit is
+        # NOT fetched. To keep the interface of this method
+        # consistent, we must return a Shipit::Command whose #success?
+        # method returns false - has a non-zero exit status. We utilize
+        # the POSIX 'test' command with no arguments which should
+        # always have an exit status of 1.
+        Command.new('test', env: env, chdir: @stack.deploys_path)
       end
     end
 
     def fetch_deployed_revision
-      with_temporary_working_directory(commit: @stack.commits.last) do |dir|
+      with_temporary_working_directory(commit: @stack.commits.reachable.last) do |dir|
         spec = DeploySpec::FileSystem.new(dir, @stack.environment)
         outputs = spec.fetch_deployed_revision_steps!.map do |command_line|
           Command.new(command_line, env: env, chdir: dir).run
@@ -42,7 +54,7 @@ module Shipit
     end
 
     def with_temporary_working_directory(commit: nil)
-      commit ||= @stack.last_deployed_commit.presence || @stack.commits.last
+      commit ||= @stack.last_deployed_commit.presence || @stack.commits.reachable.last
 
       if !commit || !fetched?(commit).tap(&:run).success?
         @stack.acquire_git_cache_lock do
@@ -55,7 +67,7 @@ module Shipit
       Dir.mktmpdir do |dir|
         git(
           'clone', @stack.git_path, @stack.repo_name,
-          '--origin', 'cache',
+          '--recursive', '--origin', 'cache',
           chdir: dir
         ).run!
 
@@ -65,6 +77,18 @@ module Shipit
       end
     end
 
+    def valid_git_repository?(path)
+      path.exist? &&
+        !path.empty? &&
+        git_cmd_succeeds?(path)
+    end
+
+    def git_cmd_succeeds?(path)
+      git("rev-parse", "--git-dir", chdir: path)
+        .tap(&:run)
+        .success?
+    end
+
     def git_clone(url, path, branch: 'master', **kwargs)
       git('clone', *modern_git_args, '--recursive', '--branch', branch, url, path, **kwargs)
     end
@@ -77,5 +101,11 @@ module Shipit
     def create_directories
       FileUtils.mkdir_p(@stack.deploys_path)
     end
+
+    private
+
+    def github
+      Shipit.github(organization: @stack.repository.owner)
+    end
   end
 end

data/lib/shipit/stat.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
   module Stat
     extend self

data/lib/shipit/task_commands.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+# rubocop:disable Lint/MissingSuper
 module Shipit
   class TaskCommands < Commands
     delegate :fetch, :fetched?, to: :stack_commands
@@ -28,22 +30,20 @@ module Shipit
     end
 
     def env
-      super.merge(
-        'ENVIRONMENT' => @stack.environment,
-        'BRANCH' => @stack.branch,
-        'SHIPIT_USER' => "#{@task.author.login} (#{normalized_author_name}) via Shipit",
-        'EMAIL' => @task.author.email,
-        'BUNDLE_PATH' => Rails.root.join('data', 'bundler').to_s,
-        'SHIPIT_LINK' => @task.permalink,
-        'LAST_DEPLOYED_SHA' => @stack.last_deployed_commit.sha,
-        'TASK_ID' => @task.id.to_s,
-        'DEPLOY_URL' => @stack.deploy_url,
-        'IGNORED_SAFETIES' => @task.ignored_safeties? ? '1' : '0',
-        'GIT_COMMITTER_NAME' => @task.user&.name || Shipit.committer_name,
-        'GIT_COMMITTER_EMAIL' => @task.user&.email || Shipit.committer_email,
-        'GITHUB_REPO_OWNER' => @stack.repository.owner,
-        'GITHUB_REPO_NAME' => @stack.repository.name,
-      ).merge(deploy_spec.machine_env).merge(@task.env)
+      super
+        .merge(@stack.env)
+        .merge(
+          'SHIPIT_USER' => "#{@task.author.login} (#{normalized_author_name}) via Shipit",
+          'EMAIL' => @task.author.email,
+          'BUNDLE_PATH' => Rails.root.join('data', 'bundler').to_s,
+          'SHIPIT_LINK' => @task.permalink,
+          'TASK_ID' => @task.id.to_s,
+          'IGNORED_SAFETIES' => @task.ignored_safeties? ? '1' : '0',
+          'GIT_COMMITTER_NAME' => @task.user&.name || Shipit.committer_name,
+          'GIT_COMMITTER_EMAIL' => @task.user&.email || Shipit.committer_email,
+        )
+        .merge(deploy_spec.machine_env)
+        .merge(@task.env)
     end
 
     def checkout(commit)
@@ -54,6 +54,7 @@ module Shipit
       [
         git(
           'clone',
+          '--quiet',
           '--local',
           '--origin', 'cache',
           @stack.git_path,
@@ -87,5 +88,11 @@ module Shipit
         @task.working_directory
       end
     end
+
+    private
+
+    def github
+      Shipit.github(organization: @stack.repository.owner)
+    end
   end
 end

data/lib/shipit/version.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Shipit
-  VERSION = '0.30.0'.freeze
+  VERSION = '0.34.0'
 end

data/lib/snippets/publish-lerna-independent-packages

@@ -1,39 +1,40 @@
 #!/usr/bin/env node
-const {resolve} = require('path');
-const {execSync} = require('child_process')
-const Repository = require(resolve('.', 'node_modules', 'lerna', 'lib', 'Repository'));
-const PackageUtilities = require(resolve('.', 'node_modules', 'lerna', 'lib', 'PackageUtilities'));
-
-function npmTag(version) {
-  const isNext = ['-beta', '-alpha', '-rc', '-next'].some(distributionType =>
-    version.includes(distributionType),
-  );
-  return isNext ? 'next' : 'latest';
-}
+const { execSync } = require("child_process");
 
-const taggedPackages = execSync('git tag --points-at HEAD')
+const taggedPackages = execSync("git tag --points-at HEAD")
   .toString()
   .trim()
-  .split('\n')
-  .map(tag => {
-    if (tag.startsWith('@')) {
-      return `@${tag.split('@')[1]}`;
-    }
-    return tag.split('@')[0];
-  });
-
-const repository = new Repository('.');
-const unsortedPackages = PackageUtilities.getPackages({rootPath: '.', packageConfigs: repository.packageConfigs});
-const packages = PackageUtilities.topologicallyBatchPackages(unsortedPackages, true)
-  .reduce((acc, packageGroup) => [...acc, ...packageGroup], [])
-  .filter(({name}) => taggedPackages.includes(name));
-
-packages.forEach(({name, version}) => {
-  const command = `node_modules/.bin/lerna publish --yes --npm-client=npm --skip-npm=false --skip-git --force-publish=${name} --repo-version=${version} --scope=${name} --npm-tag=${npmTag(
-    version,
-  )}`;
-
-  // eslint-disable-next-line no-console
-  console.log(command);
-  require('child_process').execSync(command);
+  .split("\n");
+
+// anything that matches `-` after MAJOR.MINOR.PATCH
+const validPattern = new RegExp(/\d+\.\d+\.\d+-(\w+)/);
+
+// Ensure that all releases use the same prerelease suffix.
+// We want to only release all final versions, or all betas etc
+const allPrereleaseSuffixes =  taggedPackages.map((version) => {
+  const result = validPattern.exec(version);
+  return result ? result[1] : '';
 });
+if ((new Set(allPrereleaseSuffixes)).size > 1) {
+  throw Error("All packages must be of the same type of release. Versions cannot be mixed.");
+}
+
+// set this using machine.environment.SHIPIT_LERNA_PUBLISH_MECHANISM in your shipit.yml
+const publishMechanism = process.env.SHIPIT_LERNA_PUBLISH_MECHANISM || 'from-git';
+
+if (!['from-package', 'from-git'].includes(publishMechanism)) {
+  throw Error("SHIPIT_LERNA_PUBLISH_MECHANISM must be 'from-package' or 'from-git'.");
+}
+
+const command = [
+  "node_modules/.bin/lerna publish",
+  publishMechanism,
+  "--yes",
+  "--dist-tag latest",
+  "--pre-dist-tag next",
+];
+
+const commandString = command.join(" ");
+
+console.log(`${commandString}`);
+execSync(commandString);

data/lib/snippets/publish-lerna-independent-packages-legacy

@@ -0,0 +1,39 @@
+#!/usr/bin/env node
+const {resolve} = require('path');
+const {execSync} = require('child_process')
+const Repository = require(resolve('.', 'node_modules', 'lerna', 'lib', 'Repository'));
+const PackageUtilities = require(resolve('.', 'node_modules', 'lerna', 'lib', 'PackageUtilities'));
+
+function npmTag(version) {
+  const isNext = ['-beta', '-alpha', '-rc', '-next'].some(distributionType =>
+    version.includes(distributionType),
+  );
+  return isNext ? 'next' : 'latest';
+}
+
+const taggedPackages = execSync('git tag --points-at HEAD')
+  .toString()
+  .trim()
+  .split('\n')
+  .map(tag => {
+    if (tag.startsWith('@')) {
+      return `@${tag.split('@')[1]}`;
+    }
+    return tag.split('@')[0];
+  });
+
+const repository = new Repository('.');
+const unsortedPackages = PackageUtilities.getPackages({rootPath: '.', packageConfigs: repository.packageConfigs});
+const packages = PackageUtilities.topologicallyBatchPackages(unsortedPackages, true)
+  .reduce((acc, packageGroup) => [...acc, ...packageGroup], [])
+  .filter(({name}) => taggedPackages.includes(name));
+
+packages.forEach(({name, version}) => {
+  const command = `node_modules/.bin/lerna publish --yes --npm-client=npm --skip-npm=false --skip-git --force-publish=${name} --repo-version=${version} --scope=${name} --npm-tag=${npmTag(
+    version,
+  )}`;
+
+  // eslint-disable-next-line no-console
+  console.log(command);
+  require('child_process').execSync(command);
+});

data/lib/snippets/release-gem

@@ -50,8 +50,12 @@ spec = Gem::Specification.load(spec_path)
 if RubygemsAPI.published?(spec.name, spec.version)
   puts "#{spec.name} version #{spec.version} is already published."
   exit 0
+elsif !spec.metadata['allowed_push_host']
+  puts "Can't release the gem: spec.metadata['allowed_push_host'] must be defined."
+  exit 1
 else
-  Git.tag_and_push(spec.version) do
+  is_successful = Git.tag_and_push(spec.version) do
     system(*release_command)
   end
+  is_successful ? exit(0) : exit(1)
 end

data/lib/tasks/cron.rake

@@ -1,13 +1,16 @@
+# frozen_string_literal: true
 namespace :cron do
   desc "Updates deployed revisions"
   task minutely: :environment do
     Shipit::Stack.refresh_deployed_revisions
     Shipit::Stack.schedule_continuous_delivery
     Shipit::GithubStatus.refresh_status
-    Shipit::PullRequest.schedule_merges
+    Shipit::MergeRequest.schedule_merges
+    Shipit::ReapDeadTasksJob.perform_later
+    Shipit::ReviewStackProvisioningQueue.work
   end
 
-  task hourly: %i(rollup refresh_users)
+  task hourly: %i(rollup refresh_users clear_stale_caches delete_old_deployment_directories)
 
   desc "Rolls-up output chunks for completed deploys older than an hour"
   task rollup: :environment do
@@ -17,4 +20,12 @@ namespace :cron do
   task refresh_users: :environment do
     Shipit::User.refresh_shard(Time.now.hour % 24, 24)
   end
+
+  task clear_stale_caches: :environment do
+    Shipit::ReviewStack.clear_stale_caches
+  end
+
+  task delete_old_deployment_directories: :environment do
+    Shipit::ReviewStack.delete_old_deployment_directories
+  end
 end

data/lib/tasks/dev.rake

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 namespace :dev do
   desc "Appends chunks to the last deploy, or specify with DEPLOY=id"
   task stream: :environment do
@@ -11,7 +12,7 @@ namespace :dev do
 
     deploy.update_attribute(:output, nil)
 
-    logger.info "Starting to stream data to deploy ##{deploy.id}"
+    logger.info("Starting to stream data to deploy ##{deploy.id}")
 
     loop do
       sentence = Faker::Lorem.sentence.split.map do |word|
@@ -22,7 +23,7 @@ namespace :dev do
         end
       end.join(' ')
 
-      logger.error sentence
+      logger.error(sentence)
 
       deploy.chunks.create(text: sentence + "\n")
       sleep 1

data/lib/tasks/shipit.rake

@@ -1,25 +1,26 @@
+# frozen_string_literal: true
 namespace :shipit do
   desc "Deploy from a running instance. "
   task deploy: :environment do
-      stack = ENV['stack']
-      revision = ENV['revision']
+    stack = ENV['stack']
+    revision = ENV['revision']
 
-      raise ArgumentError.new('The first argument has to be a stack, e.g. shopify/shipit/production') if stack.nil?
-      raise ArgumentError.new('The second argument has to be a revision') if revision.nil?
+    raise ArgumentError, 'The first argument has to be a stack, e.g. shopify/shipit/production' if stack.nil?
+    raise ArgumentError, 'The second argument has to be a revision' if revision.nil?
 
-      module Shipit
-        class Task
-          def write(text)
-            p text
-            chunks.create!(text: text)
-          end
+    module Shipit
+      class Task
+        def write(text)
+          p(text)
+          chunks.create!(text: text)
         end
       end
+    end
 
-      Shipit::Stack.run_deploy_in_foreground(stack: stack, revision: revision)
+    Shipit::Stack.run_deploy_in_foreground(stack: stack, revision: revision)
   rescue ArgumentError
-      p "Use this command as follows:"
-      p "bundle exec rake shipit:deploy stack='shopify/shipit/production' revision='$SHA'"
-      raise
+    p("Use this command as follows:")
+    p("bundle exec rake shipit:deploy stack='shopify/shipit/production' revision='$SHA'")
+    raise
   end
 end

data/lib/tasks/teams.rake

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 namespace :teams do
   desc "Import the members of each team configured through the github.oauth.teams config"
   task fetch: :environment do

data/test/controllers/api/base_controller_test.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'test_helper'
 
 module Shipit
@@ -6,7 +7,7 @@ module Shipit
       test "authentication is required" do
         get :index
         assert_response :unauthorized
-        assert_equal({message: 'Bad credentials'}.to_json, response.body)
+        assert_equal({ message: 'Bad credentials' }.to_json, response.body)
       end
 
       test "with proper credentials the request is processed" do
@@ -17,7 +18,7 @@ module Shipit
       test "#index respond with a list of endpoints" do
         authenticate!
         get :index, format: :json
-        assert_equal({stacks_url: api_stacks_url}.to_json, response.body)
+        assert_equal({ stacks_url: api_stacks_url }.to_json, response.body)
       end
     end
   end

data/test/controllers/api/ccmenu_controller_test.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'test_helper'
 
 module Shipit
@@ -10,26 +11,26 @@ module Shipit
 
       test "a request with insufficient permissions will render a 403" do
         @client.update!(permissions: [])
-        get :show, params: {stack_id: @stack.to_param}
+        get :show, params: { stack_id: @stack.to_param }
         assert_response :forbidden
         assert_json 'message', 'This operation requires the `read:stack` permission'
       end
 
       test "#show renders the xml" do
-        get :show, params: {stack_id: @stack.to_param}
+        get :show, params: { stack_id: @stack.to_param }
         assert_response :ok
         assert_payload 'name', @stack.to_param
       end
 
       test "can authenticate with query string token" do
         request.headers['Authorization'] = 'bleh'
-        get :show, params: {stack_id: @stack.to_param, token: @client.authentication_token}
+        get :show, params: { stack_id: @stack.to_param, token: @client.authentication_token }
         assert_response :ok
         assert_payload 'name', @stack.to_param
       end
 
       test "xml contains required attributes" do
-        get :show, params: {stack_id: @stack.to_param}
+        get :show, params: { stack_id: @stack.to_param }
         project = get_project_from_xml(response.body)
         %w(name activity lastBuildStatus lastBuildLabel lastBuildTime webUrl).each do |attribute|
           assert_includes project, attribute, "Response missing required attribute: #{attribute}"
@@ -38,13 +39,13 @@ module Shipit
 
       test "locked stacks show as failed" do
         @stack.lock('test', @user)
-        get :show, params: {stack_id: @stack.to_param}
+        get :show, params: { stack_id: @stack.to_param }
         assert_payload 'lastBuildStatus', 'Failure'
       end
 
       test "stacks with no deploys render correctly" do
-        stack = Stack.create!(repository: Repository.new(owner: "foo", name: "bar"))
-        get :show, params: {stack_id: stack.to_param}
+        stack = Stack.create!(repository: Repository.new(owner: "foo", name: "bar"), branch: 'main')
+        get :show, params: { stack_id: stack.to_param }
         assert_payload 'lastBuildStatus', 'Success'
       end
 
@@ -56,7 +57,7 @@ module Shipit
 
       def assert_payload(k, v)
         @project ||= get_project_from_xml(response.body)
-        assert_equal v, @project[k]
+        assert_equal(v, @project[k])
       end
     end
   end