shipit-engine 0.32.0 → 0.35.1

data/lib/shipit/task_commands.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+# rubocop:disable Lint/MissingSuper
 module Shipit
   class TaskCommands < Commands
     delegate :fetch, :fetched?, to: :stack_commands
@@ -46,7 +47,7 @@ module Shipit
     end
 
     def checkout(commit)
-      git('checkout', commit.sha, chdir: @task.working_directory)
+      git('-c', 'advice.detachedHead=false', 'checkout', commit.sha, chdir: @task.working_directory)
     end
 
     def clone
@@ -87,5 +88,11 @@ module Shipit
         @task.working_directory
       end
     end
+
+    private
+
+    def github
+      Shipit.github(organization: @stack.repository.owner)
+    end
   end
 end

data/lib/shipit/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Shipit
-  VERSION = '0.32.0'
+  VERSION = '0.35.1'
 end

data/lib/shipit.rb

@@ -24,7 +24,6 @@ require 'safe_yaml/load'
 require 'securecompare'
 
 require 'redis-objects'
-require 'redis-namespace'
 
 require 'octokit'
 require 'faraday-http-cache'
@@ -43,6 +42,7 @@ require 'shipit/simple_message_verifier'
 require 'shipit/command'
 require 'shipit/commands'
 require 'shipit/stack_commands'
+require 'shipit/review_stack_commands'
 require 'shipit/task_commands'
 require 'shipit/deploy_commands'
 require 'shipit/rollback_commands'
@@ -59,10 +59,22 @@ SafeYAML::OPTIONS[:deserialize_symbols] = false
 module Shipit
   extend self
 
+  GithubOrganizationUnknown = Class.new(StandardError)
+  TOP_LEVEL_GH_KEYS = [:app_id, :installation_id, :webhook_secret, :private_key, :oauth, :domain]
+
   delegate :table_name_prefix, to: :secrets
 
-  attr_accessor :disable_api_authentication, :timeout_exit_codes
-  attr_writer :internal_hook_receivers, :task_logger, :preferred_org_emails
+  attr_accessor :disable_api_authentication, :timeout_exit_codes, :deployment_checks
+  attr_writer(
+    :internal_hook_receivers,
+    :preferred_org_emails,
+    :task_execution_strategy,
+    :task_logger,
+  )
+
+  def task_execution_strategy
+    @task_execution_strategy ||= Shipit::TaskExecutionStrategy::Default
+  end
 
   self.timeout_exit_codes = [].freeze
 
@@ -82,7 +94,7 @@ module Shipit
     secrets.redis_url.present? ? URI(secrets.redis_url) : nil
   end
 
-  def redis(namespace = nil)
+  def redis
     @redis ||= Redis.new(
       url: redis_url.to_s.presence,
       logger: Rails.logger,
@@ -90,17 +102,41 @@ module Shipit
       reconnect_delay: 0.5,
       reconnect_delay_max: 1,
     )
-    return @redis unless namespace
-    Redis::Namespace.new(namespace, redis: @redis)
   end
 
-  def github
-    @github ||= GitHubApp.new(secrets.github)
+  def github(organization: github_default_organization)
+    # Backward compatibility
+    # nil signifies the single github app config schema is being used
+    if github_default_organization.nil?
+      config = secrets.github
+    else
+      config = github_app_config(organization)
+      raise GithubOrganizationUnknown, organization if config.nil?
+    end
+    @github ||= {}
+    @github[organization] ||= GitHubApp.new(organization, config)
+  end
+
+  def github_default_organization
+    return nil unless secrets&.github
+    org = secrets.github.keys.first
+    TOP_LEVEL_GH_KEYS.include?(org) ? nil : org
+  end
+
+  def github_organizations
+    return [nil] unless github_default_organization
+    secrets.github.keys
+  end
+
+  def github_app_config(organization)
+    github_config = secrets.github.deep_transform_keys(&:downcase)
+    github_organization = organization.downcase.to_sym
+    github_config[github_organization]
   end
 
   def legacy_github_api
     if secrets&.github_api.present?
-      @legacy_github_api ||= github.new_client(access_token: secrets.github_api['access_token'])
+      @legacy_github_api ||= github.new_client(access_token: secrets.github_api[:access_token])
     end
   end
 
@@ -165,13 +201,11 @@ module Shipit
   end
 
   def revision
-    @revision ||= begin
-      if revision_file.exist?
-        revision_file.read
-      else
-        %x(git rev-parse HEAD)
-      end.strip
-    end
+    @revision ||= if revision_file.exist?
+      revision_file.read
+    else
+      %x(git rev-parse HEAD)
+    end.strip
   end
 
   def default_inactivity_timeout

data/lib/snippets/publish-lerna-independent-packages

@@ -1,39 +1,40 @@
 #!/usr/bin/env node
-const {resolve} = require('path');
-const {execSync} = require('child_process')
-const Repository = require(resolve('.', 'node_modules', 'lerna', 'lib', 'Repository'));
-const PackageUtilities = require(resolve('.', 'node_modules', 'lerna', 'lib', 'PackageUtilities'));
-
-function npmTag(version) {
-  const isNext = ['-beta', '-alpha', '-rc', '-next'].some(distributionType =>
-    version.includes(distributionType),
-  );
-  return isNext ? 'next' : 'latest';
-}
+const { execSync } = require("child_process");
 
-const taggedPackages = execSync('git tag --points-at HEAD')
+const taggedPackages = execSync("git tag --points-at HEAD")
   .toString()
   .trim()
-  .split('\n')
-  .map(tag => {
-    if (tag.startsWith('@')) {
-      return `@${tag.split('@')[1]}`;
-    }
-    return tag.split('@')[0];
-  });
-
-const repository = new Repository('.');
-const unsortedPackages = PackageUtilities.getPackages({rootPath: '.', packageConfigs: repository.packageConfigs});
-const packages = PackageUtilities.topologicallyBatchPackages(unsortedPackages, true)
-  .reduce((acc, packageGroup) => [...acc, ...packageGroup], [])
-  .filter(({name}) => taggedPackages.includes(name));
-
-packages.forEach(({name, version}) => {
-  const command = `node_modules/.bin/lerna publish --yes --npm-client=npm --skip-npm=false --skip-git --force-publish=${name} --repo-version=${version} --scope=${name} --npm-tag=${npmTag(
-    version,
-  )}`;
-
-  // eslint-disable-next-line no-console
-  console.log(command);
-  require('child_process').execSync(command);
+  .split("\n");
+
+// anything that matches `-` after MAJOR.MINOR.PATCH
+const validPattern = new RegExp(/\d+\.\d+\.\d+-(\w+)/);
+
+// Ensure that all releases use the same prerelease suffix.
+// We want to only release all final versions, or all betas etc
+const allPrereleaseSuffixes =  taggedPackages.map((version) => {
+  const result = validPattern.exec(version);
+  return result ? result[1] : '';
 });
+if ((new Set(allPrereleaseSuffixes)).size > 1) {
+  throw Error("All packages must be of the same type of release. Versions cannot be mixed.");
+}
+
+// set this using machine.environment.SHIPIT_LERNA_PUBLISH_MECHANISM in your shipit.yml
+const publishMechanism = process.env.SHIPIT_LERNA_PUBLISH_MECHANISM || 'from-git';
+
+if (!['from-package', 'from-git'].includes(publishMechanism)) {
+  throw Error("SHIPIT_LERNA_PUBLISH_MECHANISM must be 'from-package' or 'from-git'.");
+}
+
+const command = [
+  "node_modules/.bin/lerna publish",
+  publishMechanism,
+  "--yes",
+  "--dist-tag latest",
+  "--pre-dist-tag next",
+];
+
+const commandString = command.join(" ");
+
+console.log(`${commandString}`);
+execSync(commandString);

data/lib/snippets/publish-lerna-independent-packages-legacy

@@ -0,0 +1,39 @@
+#!/usr/bin/env node
+const {resolve} = require('path');
+const {execSync} = require('child_process')
+const Repository = require(resolve('.', 'node_modules', 'lerna', 'lib', 'Repository'));
+const PackageUtilities = require(resolve('.', 'node_modules', 'lerna', 'lib', 'PackageUtilities'));
+
+function npmTag(version) {
+  const isNext = ['-beta', '-alpha', '-rc', '-next'].some(distributionType =>
+    version.includes(distributionType),
+  );
+  return isNext ? 'next' : 'latest';
+}
+
+const taggedPackages = execSync('git tag --points-at HEAD')
+  .toString()
+  .trim()
+  .split('\n')
+  .map(tag => {
+    if (tag.startsWith('@')) {
+      return `@${tag.split('@')[1]}`;
+    }
+    return tag.split('@')[0];
+  });
+
+const repository = new Repository('.');
+const unsortedPackages = PackageUtilities.getPackages({rootPath: '.', packageConfigs: repository.packageConfigs});
+const packages = PackageUtilities.topologicallyBatchPackages(unsortedPackages, true)
+  .reduce((acc, packageGroup) => [...acc, ...packageGroup], [])
+  .filter(({name}) => taggedPackages.includes(name));
+
+packages.forEach(({name, version}) => {
+  const command = `node_modules/.bin/lerna publish --yes --npm-client=npm --skip-npm=false --skip-git --force-publish=${name} --repo-version=${version} --scope=${name} --npm-tag=${npmTag(
+    version,
+  )}`;
+
+  // eslint-disable-next-line no-console
+  console.log(command);
+  require('child_process').execSync(command);
+});

data/lib/tasks/cron.rake

@@ -5,11 +5,12 @@ namespace :cron do
     Shipit::Stack.refresh_deployed_revisions
     Shipit::Stack.schedule_continuous_delivery
     Shipit::GithubStatus.refresh_status
-    Shipit::PullRequest.schedule_merges
+    Shipit::MergeRequest.schedule_merges
     Shipit::ReapDeadTasksJob.perform_later
+    Shipit::ReviewStackProvisioningQueue.work
   end
 
-  task hourly: %i(rollup refresh_users)
+  task hourly: %i(rollup refresh_users clear_stale_caches delete_old_deployment_directories)
 
   desc "Rolls-up output chunks for completed deploys older than an hour"
   task rollup: :environment do
@@ -19,4 +20,12 @@ namespace :cron do
   task refresh_users: :environment do
     Shipit::User.refresh_shard(Time.now.hour % 24, 24)
   end
+
+  task clear_stale_caches: :environment do
+    Shipit::ReviewStack.clear_stale_caches
+  end
+
+  task delete_old_deployment_directories: :environment do
+    Shipit::ReviewStack.delete_old_deployment_directories
+  end
 end

data/test/controllers/api/ccmenu_controller_test.rb

@@ -44,7 +44,7 @@ module Shipit
       end
 
       test "stacks with no deploys render correctly" do
-        stack = Stack.create!(repository: Repository.new(owner: "foo", name: "bar"))
+        stack = Stack.create!(repository: Repository.new(owner: "foo", name: "bar"), branch: 'main')
         get :show, params: { stack_id: stack.to_param }
         assert_payload 'lastBuildStatus', 'Success'
       end

data/test/controllers/api/deploys_controller_test.rb

@@ -83,6 +83,23 @@ module Shipit
         assert_response :conflict
       end
 
+      test "#create refuses to deploy unsuccessful commits if the require_ci flag is passed" do
+        Commit.any_instance.expects(:deployable?).returns(false)
+
+        assert_no_difference -> { @stack.deploys.count } do
+          post :create, params: { stack_id: @stack.to_param, sha: @commit.sha, require_ci: true }
+        end
+        assert_response :unprocessable_entity
+        assert_json 'errors.require_ci', ["Commit is not deployable"]
+      end
+
+      test "#create deploys failing commits if the require_ci flag is not passed" do
+        Commit.any_instance.expects(:deployable?).returns(false)
+
+        post :create, params: { stack_id: @stack.to_param, sha: @commit.sha }
+        assert_response :accepted
+      end
+
       test "#create refuses to deploy locked stacks" do
         @stack.update!(lock_reason: 'Something broken')
 

data/test/controllers/api/{pull_requests_controller_test.rb → merge_requests_controller_test.rb}

@@ -3,52 +3,52 @@ require 'test_helper'
 
 module Shipit
   module Api
-    class PullRequestsControllerTest < ActionController::TestCase
+    class MergeRequestsControllerTest < ActionController::TestCase
       setup do
         @stack = shipit_stacks(:shipit)
-        @pull_request = shipit_pull_requests(:shipit_pending)
+        @merge_request = shipit_merge_requests(:shipit_pending)
         authenticate!
       end
 
       test "#index returns a list of pull requests" do
-        pull_request = @stack.pull_requests.last
+        merge_request = @stack.merge_requests.last
 
         get :index, params: { stack_id: @stack.to_param }
         assert_response :ok
-        assert_json '0.id', pull_request.id
+        assert_json '0.id', merge_request.id
       end
 
       test "#show returns a single pull requests" do
-        get :show, params: { stack_id: @stack.to_param, id: @pull_request.number.to_s }
+        get :show, params: { stack_id: @stack.to_param, id: @merge_request.number.to_s }
         assert_response :ok
-        assert_json 'id', @pull_request.id
+        assert_json 'id', @merge_request.id
       end
 
       test "#update responds with Accepted if the pull request was queued" do
-        assert_enqueued_with(job: RefreshPullRequestJob) do
+        assert_enqueued_with(job: RefreshMergeRequestJob) do
           put :update, params: { stack_id: @stack.to_param, id: '64' }
         end
         assert_response :accepted
       end
 
       test "#update responds with Accepted if the pull request was already queued" do
-        assert_enqueued_with(job: RefreshPullRequestJob) do
+        assert_enqueued_with(job: RefreshMergeRequestJob) do
           put :update, params: { stack_id: @stack.to_param, id: '65' }
         end
         assert_response :accepted
       end
 
       test "#update responds with method not allowed if the pull request was already merged" do
-        @pull_request.complete!
-        put :update, params: { stack_id: @stack.to_param, id: @pull_request.number.to_s }
+        @merge_request.complete!
+        put :update, params: { stack_id: @stack.to_param, id: @merge_request.number.to_s }
         assert_response :method_not_allowed
         assert_json 'message', 'This pull request was already merged.'
       end
 
       test "#destroy cancels the merge if the pull request was waiting" do
-        delete :destroy, params: { stack_id: @stack.to_param, id: @pull_request.number.to_s }
+        delete :destroy, params: { stack_id: @stack.to_param, id: @merge_request.number.to_s }
         assert_response :no_content
-        assert_predicate @pull_request.reload, :canceled?
+        assert_predicate @merge_request.reload, :canceled?
       end
 
       test "#destroy silently fail if the pull request was unknown" do

data/test/controllers/api/outputs_controller_test.rb

@@ -11,6 +11,7 @@ module Shipit
 
       test "#show returns the task output as plain text" do
         task = @stack.tasks.last
+        task.write("dummy output")
 
         get :show, params: { stack_id: @stack.to_param, task_id: task.id }
         assert_response :ok

data/test/controllers/api/rollback_controller_test.rb

@@ -106,7 +106,7 @@ module Shipit
         last_deploy.reload
         assert_response :accepted
         refute_predicate last_deploy, :active?
-        assert_json 'rollback_once_aborted_to.until_commit_id', @commit.id
+        assert_json 'rollback_once_aborted_to.revision.sha', @commit.sha
       end
     end
   end

data/test/controllers/api/stacks_controller_test.rb

@@ -23,7 +23,7 @@ module Shipit
 
       test "#create fails with invalid stack" do
         assert_no_difference "Stack.count" do
-          post :create, params: { repo_owner: 'some', repo_name: 'owner/path' }
+          post :create, params: { repo_owner: 'some', repo_name: 'owner/path', branch: 'main' }
         end
         assert_response :unprocessable_entity
         assert_json 'errors', 'repository' => ['is invalid']
@@ -48,12 +48,12 @@ module Shipit
 
         assert_no_difference -> { Stack.count } do
           post :create,
-               params: {
-                 repo_name: existing_stack.repo_name,
-                 repo_owner: existing_stack.repo_owner,
-                 environment: existing_stack.environment,
-                 branch: existing_stack.branch,
-               }
+            params: {
+              repo_name: existing_stack.repo_name,
+              repo_owner: existing_stack.repo_owner,
+              environment: existing_stack.environment,
+              branch: existing_stack.branch,
+            }
         end
 
         assert_response :unprocessable_entity
@@ -82,9 +82,22 @@ module Shipit
         assert_equal true, @stack.continuous_deployment
       end
 
+      test "#update allows changing the branch name" do
+        assert_equal 'master', @stack.branch
+
+        patch :update, params: {
+          id: @stack.to_param,
+          branch: 'test',
+        }
+
+        assert_response :ok
+        @stack.reload
+
+        assert_equal 'test', @stack.branch
+      end
+
       test "#index returns a list of stacks" do
         stack = Stack.last
-
         get :index
         assert_response :ok
         assert_json '0.id', stack.id
@@ -93,6 +106,27 @@ module Shipit
         end
       end
 
+      test "#index returns a list of stacks filtered by repo if name and owner given" do
+        repo = shipit_repositories(:shipit)
+        get :index, params: { repo_owner: repo.owner, repo_name: repo.name }
+        assert_response :ok
+        assert_json do |stacks|
+          assert_equal stacks.size, repo.stacks.size
+        end
+      end
+
+      test "#index returns a list of stacks filtered by repo and api client" do
+        authenticate!(:here_come_the_walrus)
+
+        repo = shipit_repositories(:soc)
+
+        get :index, params: { repo_owner: repo.owner, repo_name: repo.name }
+        assert_response :ok
+        assert_json do |stacks|
+          assert_equal 0, stacks.size
+        end
+      end
+
       test "#index is paginable" do
         get :index, params: { page_size: 1 }
         assert_json do |list|