shipit-engine 0.14.0 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 80b89004571af2792321c0d57a3659ebb7cb21aa
-  data.tar.gz: 06363fdf4d7f9e3ba64bf992eaf11855ecfb1061
+  metadata.gz: 1ed7609bec0e259e4962046f8d6107263c7429db
+  data.tar.gz: 835c09964fcd482736103a6d8e40f59bd2aa1f0e
 SHA512:
-  metadata.gz: ea768d3e022264786c9ddf0b9bbc1d36fb5f8379fc362c9a285d4eab28014d17c248ea9471ff55b0ef45dafa7ac529858e632ee2c2b99aa96cb1f0f1e1d5a00a
-  data.tar.gz: 26c9579f88a71172c84c510172d882c31a2edac8b468ef1c983ab056b5a37ff367331a79524172321abd2cecfa19b89a88c40e1784a25600de713f156d3e4009
+  metadata.gz: cacebc0ec7fcce7a0af8feefa79d7d48878548bd07b8a75b02878383ed89d7db4364e9b623f910bf9090b5bbbd7e4d201926f86309f265ef279ac40a1b554ee9
+  data.tar.gz: 6ef2234ef1e1be3d462572a817edeab7ad042a0ca47639834ad349b88308a7d9541c0d3527f63697752735b9e4c3528b597197de2e882c569db9b9ef9fbc6521

data/README.md

@@ -33,6 +33,7 @@ This guide aims to help you [set up](#installation-and-setup), [use](#using-ship
 * [Format and content of shipit.yml](#configuring-shipit)
 * [Format and content of secrets.yml](#configuring-secrets)
 * [Script parameters](#script-parameters)
+* [Configuring providers](#configuring-providers)
 * [Free samples](/examples/shipit.yml)
 
 * * *
@@ -320,6 +321,19 @@ For example:
 fetch:
   curl --silent https://app.example.com/services/ping/version
 ```
+<h3 id="kubernetes">Kubernetes</h3>
+
+**<code>kubernetes</code>** allows to specify a Kubernetes namespace and context to deploy to.
+
+For example:
+```yml
+kubernetes:
+  namespace: my-app-production
+  context: tier4
+```
+
+**<code>kubernetes.template_dir</code>** allows to specify a Kubernetes template directory. It defaults to `./config/deploy/$ENVIRONMENT`
+
 <h3 id="environment">Environment</h3>
 
 **<code>machine.environment</code>** contains the extra environment variables that you want to provide during task execution.
@@ -463,6 +477,24 @@ review:
     - bundle exec rake db:migrate:status
 ```
 
+<h3 id="shell-commands-timeout">Shell commands timeout</h3>
+
+All the shell commands can take an optional `timeout` parameter to limit their duration:
+
+```yml
+deploy:
+  override:
+    - ./script/deploy:
+        timeout: 30
+  post:
+    - ./script/notify_deploy_end: { timeout: 15 }
+review:
+  checks:
+    - bundle exec rake db:migrate:status:
+        timeout: 60
+```
+
+See also `commands_inactivity_timeout` in `secrets.yml` for a global timeout setting.
 
 
 ***
@@ -576,3 +608,16 @@ These variables are accessible only during deploys and rollback:
 
 * `REVISION`: the git SHA of the revision that must be deployed in production
 * `SHA`: alias for REVISION
+
+<h2 id="configuring-providers">Configuring providers</h2>
+
+### Heroku
+
+To use Heroku integration (`lib/snippets/push-to-heroku`), make sure that the environment has [Heroku toolbelt](https://devcenter.heroku.com/articles/heroku-cli) available.
+
+### Kubernetes
+
+For Kubernetes, you have to provision Shipit environment with the following tools:
+
+* `kubectl`
+* `kubernetes-deploy` [gem](https://github.com/Shopify/kubernetes-deploy)

data/app/jobs/shipit/deferred_touch_job.rb

@@ -0,0 +1,9 @@
+module Shipit
+  class DeferredTouchJob < BackgroundJob
+    include BackgroundJob::Unique
+
+    def perform
+      DeferredTouch.touch_now!
+    end
+  end
+end

data/app/jobs/shipit/perform_task_job.rb

@@ -52,7 +52,7 @@ module Shipit
     end
 
     def perform_task
-      Bundler.with_clean_env do
+      Bundler.with_original_env do
         capture_all! @commands.install_dependencies
         capture_all! @commands.perform
       end

data/app/models/concerns/shipit/deferred_touch.rb

@@ -0,0 +1,92 @@
+module Shipit
+  module DeferredTouch
+    extend ActiveSupport::Concern
+
+    SET_KEY = 'shipit:deferred_touches'.freeze
+    TMP_KEY = "#{SET_KEY}:updating".freeze
+    CACHE_KEY = "#{SET_KEY}:scheduled".freeze
+    THROTTLE_TTL = 1.second
+
+    included do
+      class_attribute :deferred_touches, instance_accessor: false
+      after_commit :schedule_touches
+    end
+
+    class << self
+      attr_accessor :enabled
+
+      def touch_now!
+        now = Time.now.utc
+        fetch do |touches|
+          records = []
+          touches.each do |model, changes|
+            changes.each do |attribute, ids|
+              records << [model.where(id: ids).to_a, attribute]
+            end
+          end
+
+          ActiveRecord::Base.transaction do
+            records.each do |instances, attribute|
+              instances.each { |i| i.touch(attribute, time: now) }
+            end
+          end
+        end
+      end
+
+      private
+
+      def fetch
+        fetch_members do |records|
+          return if records.empty?
+          records = records.each_with_object({}) do |(model, id, attribute), hash|
+            attributes = (hash[model] ||= {})
+            ids = (attributes[attribute] ||= [])
+            ids << id
+          end
+          yield records.transform_keys(&:constantize)
+        end
+      end
+
+      def fetch_members
+        Shipit.redis.multi do
+          Shipit.redis.sunionstore(TMP_KEY, SET_KEY)
+          Shipit.redis.del(SET_KEY)
+        end
+
+        yield Shipit.redis.smembers(TMP_KEY).map { |r| r.split('|') }
+
+        Shipit.redis.del(TMP_KEY)
+      end
+    end
+
+    self.enabled = true
+
+    module ClassMethods
+      def deferred_touch(touches)
+        touches = touches.transform_values(&Array.method(:wrap))
+        self.deferred_touches = touches.flat_map do |association_name, attributes|
+          association = reflect_on_association(association_name)
+          Array.wrap(attributes).map do |attribute|
+            [association.klass.name, association.foreign_key, attribute]
+          end
+        end
+      end
+    end
+
+    private
+
+    def schedule_touches
+      return unless self.class.deferred_touches
+      touches = self.class.deferred_touches.map { |m, fk, a| [m, self[fk], a].join('|') }
+      Shipit.redis.sadd(SET_KEY, touches)
+      if DeferredTouch.enabled
+        Rails.cache.fetch(CACHE_KEY, expires_in: THROTTLE_TTL) do
+          DeferredTouchJob.perform_later
+          true
+        end
+      else
+        DeferredTouch.touch_now!
+      end
+    end
+  end
+end

data/app/models/shipit/commit.rb

@@ -1,19 +1,21 @@
 module Shipit
   class Commit < ActiveRecord::Base
+    include DeferredTouch
+
     AmbiguousRevision = Class.new(StandardError)
 
-    belongs_to :stack, touch: true
+    belongs_to :stack
     has_many :deploys
-    has_many :statuses, -> { order(created_at: :desc) }
+    has_many :statuses, -> { order(created_at: :desc) }, dependent: :destroy
     has_many :commit_deployments, dependent: :destroy
 
+    deferred_touch stack: :updated_at
+
     after_commit { broadcast_update }
     after_create { stack.update_undeployed_commits_count }
 
     after_commit :schedule_refresh_statuses!, :schedule_fetch_stats!, :schedule_continuous_delivery, on: :create
 
-    after_touch :touch_stack
-
     belongs_to :author, class_name: 'User', inverse_of: :authored_commits
     belongs_to :committer, class_name: 'User', inverse_of: :commits
 
@@ -88,7 +90,7 @@ module Shipit
 
     def create_status_from_github!(github_status)
       add_status do
-        statuses.replicate_from_github!(github_status)
+        statuses.replicate_from_github!(stack_id, github_status)
       end
     end
 
@@ -207,9 +209,5 @@ module Shipit
     def missing_statuses
       stack.required_statuses - last_statuses.map(&:context)
     end
-
-    def touch_stack
-      stack.touch
-    end
   end
 end

data/app/models/shipit/delivery.rb

@@ -5,6 +5,8 @@ module Shipit
 
     belongs_to :hook
 
+    scope :due_for_deletion, -> { where('created_at < ?', 1.month.ago) }
+
     validates :url, presence: true, url: {no_local: true, allow_blank: true}
     validates :content_type, presence: true
 

data/app/models/shipit/deploy_spec.rb

@@ -85,6 +85,10 @@ module Shipit
       Array.wrap(config('deploy', 'variables')).map(&VariableDefinition.method(:new))
     end
 
+    def default_deploy_env
+      deploy_variables.map { |v| [v.name, v.default] }.to_h
+    end
+
     def rollback_steps
       around_steps('rollback') do
         config('rollback', 'override') { discover_rollback_steps }
@@ -182,6 +186,10 @@ module Shipit
     def discover_fetch_deployed_revision_steps
     end
 
+    def discover_machine_env
+      {}
+    end
+
     def task_not_found!(id)
       raise TaskDefinition::NotFound.new("No definition for task #{id.inspect}")
     end

data/app/models/shipit/deploy_spec/file_system.rb

@@ -1,6 +1,8 @@
 module Shipit
   class DeploySpec
     class FileSystem < DeploySpec
+      include NpmDiscovery
+      include KubernetesDiscovery
       include PypiDiscovery
       include RubygemsDiscovery
       include CapistranoDiscovery
@@ -33,7 +35,7 @@ module Shipit
             'require' => required_statuses,
           },
           'machine' => {
-            'environment' => machine_env,
+            'environment' => discover_machine_env.merge(machine_env),
             'directory' => directory,
             'cleanup' => true,
           },

data/app/models/shipit/deploy_spec/kubernetes_discovery.rb

@@ -0,0 +1,37 @@
+module Shipit
+  class DeploySpec
+    module KubernetesDiscovery
+      def discover_deploy_steps
+        discover_kubernetes || super
+      end
+
+      def discover_rollback_steps
+        discover_kubernetes || super
+      end
+
+      def discover_machine_env
+        env = super
+        env = env.merge('K8S_TEMPLATE_FOLDER' => kube_config['template_dir']) if kube_config['template_dir']
+        env
+      end
+
+      private
+
+      def discover_kubernetes
+        return unless kube_config.present?
+
+        [
+          Shellwords.join([
+            "kubernetes-deploy",
+            kube_config['namespace'],
+            kube_config['context'],
+          ]),
+        ]
+      end
+
+      def kube_config
+        @kube_config ||= config('kubernetes') || {}
+      end
+    end
+  end
+end

data/app/models/shipit/deploy_spec/npm_discovery.rb

@@ -0,0 +1,81 @@
+require 'json'
+
+module Shipit
+  class DeploySpec
+    module NpmDiscovery
+      def discover_dependencies_steps
+        discover_package_json || super
+      end
+
+      def discover_package_json
+        npm_install if yarn? || npm?
+      end
+
+      def npm_install
+        [js_command('install --no-progress')]
+      end
+
+      def discover_review_checklist
+        discover_yarn_checklist || discover_npm_checklist || super
+      end
+
+      def discover_yarn_checklist
+        [%(<strong>Don't forget version and tag before publishing!</strong> You can do this with:<br/>
+          yarn version --new-version <strong>&lt;major|minor|patch&gt;</strong> && git push --tags</pre>)] if yarn?
+      end
+
+      def discover_npm_checklist
+        [%(<strong>Don't forget version and tag before publishing!</strong> You can do this with:<br/>
+          npm version <strong>&lt;major|minor|patch&gt;</strong> && git push --tags</pre>)] if npm?
+      end
+
+      def npm?
+        public?
+      end
+
+      def public?
+        file = package_json
+        return false unless file.exist?
+
+        JSON.parse(file.read)['private'].blank?
+      end
+
+      def package_json
+        file('package.json')
+      end
+
+      def yarn?
+        yarn_lock.exist? && public?
+      end
+
+      def yarn_lock
+        file('./yarn.lock')
+      end
+
+      def discover_npm_package
+        publish_npm_package if yarn? || npm?
+      end
+
+      def discover_deploy_steps
+        discover_npm_package || super
+      end
+
+      def publish_npm_package
+        check_tags = 'assert-npm-version-tag'
+        publish = js_command('publish')
+
+        [check_tags, publish]
+      end
+
+      def js_command(command_args)
+        runner = if yarn?
+          'yarn'
+        else
+          'npm'
+        end
+
+        "#{runner} #{command_args}"
+      end
+    end
+  end
+end

data/app/models/shipit/stack.rb

@@ -50,7 +50,6 @@ module Shipit
     after_commit :broadcast_update, on: :update
     after_commit :emit_merge_status_hooks, on: :update
     after_commit :setup_hooks, :sync_github, on: :create
-    after_touch :clear_cache
 
     validates :repo_name, uniqueness: {scope: %i(repo_owner environment),
                                        message: 'cannot be used more than once with this environment'}
@@ -136,7 +135,7 @@ module Shipit
         return
       end
 
-      trigger_deploy(commit, Shipit.user)
+      trigger_deploy(commit, Shipit.user, env: cached_deploy_spec.default_deploy_env)
     end
 
     def next_commit_to_deploy

data/app/models/shipit/status.rb

@@ -1,21 +1,26 @@
 module Shipit
   class Status < ActiveRecord::Base
+    include DeferredTouch
+
     STATES = %w(pending success failure error).freeze
     enum state: STATES.zip(STATES).to_h
 
-    belongs_to :commit, touch: true
+    belongs_to :stack, required: true
+    belongs_to :commit, required: true
+
+    deferred_touch stack: :updated_at, commit: :updated_at
 
     validates :state, inclusion: {in: STATES, allow_blank: true}, presence: true
 
     after_create :enable_ci_on_stack
     after_commit :schedule_continuous_delivery, :broadcast_update, on: :create
-    after_commit :touch_commit
 
     delegate :broadcast_update, to: :commit
 
     class << self
-      def replicate_from_github!(github_status)
+      def replicate_from_github!(stack_id, github_status)
         find_or_create_by!(
+          stack_id: stack_id,
           state: github_status.state,
           description: github_status.description,
           target_url: github_status.target_url,
@@ -25,8 +30,6 @@ module Shipit
       end
     end
 
-    delegate :stack, to: :commit
-
     def unknown?
       false
     end
@@ -49,10 +52,6 @@ module Shipit
       commit.stack.enable_ci!
     end
 
-    def touch_commit
-      commit.touch
-    end
-
     def schedule_continuous_delivery
       commit.schedule_continuous_delivery
     end