shipit-engine 0.14.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +45 -0
  3. data/app/jobs/shipit/deferred_touch_job.rb +9 -0
  4. data/app/jobs/shipit/perform_task_job.rb +1 -1
  5. data/app/jobs/shipit/{update_estimated_deploy_duration.rb → update_estimated_deploy_duration_job.rb} +0 -0
  6. data/app/models/concerns/shipit/deferred_touch.rb +92 -0
  7. data/app/models/shipit/commit.rb +7 -9
  8. data/app/models/shipit/delivery.rb +2 -0
  9. data/app/models/shipit/deploy_spec.rb +8 -0
  10. data/app/models/shipit/deploy_spec/file_system.rb +3 -1
  11. data/app/models/shipit/deploy_spec/kubernetes_discovery.rb +37 -0
  12. data/app/models/shipit/deploy_spec/npm_discovery.rb +81 -0
  13. data/app/models/shipit/stack.rb +1 -2
  14. data/app/models/shipit/status.rb +8 -9
  15. data/app/models/shipit/task.rb +5 -1
  16. data/config/secrets.development.example.yml +4 -0
  17. data/config/secrets.development.shopify.yml +4 -0
  18. data/config/secrets.development.yml +1 -1
  19. data/db/migrate/20161205144522_add_indexes_on_deliveries.rb +17 -0
  20. data/db/migrate/20161206104100_delete_orphan_statuses.rb +10 -0
  21. data/db/migrate/20161206104224_denormalize_stack_id_on_statuses.rb +5 -0
  22. data/db/migrate/20161206104817_backfill_stack_id_on_statuses.rb +13 -0
  23. data/db/migrate/20161206105318_makes_stack_id_not_null_on_statuses.rb +5 -0
  24. data/lib/shipit.rb +3 -0
  25. data/lib/shipit/strip_cache_control.rb +40 -0
  26. data/lib/shipit/version.rb +1 -1
  27. data/lib/snippets/assert-npm-version-tag +22 -0
  28. data/lib/snippets/deploy-to-gke +3 -4
  29. data/lib/tasks/cron.rake +7 -0
  30. data/test/dummy/config/environments/development.rb +6 -2
  31. data/test/dummy/config/environments/test.rb +4 -0
  32. data/test/dummy/db/development.sqlite3 +0 -0
  33. data/test/dummy/db/schema.rb +32 -42
  34. data/test/dummy/db/seeds.rb +2 -0
  35. data/test/dummy/db/test.sqlite3 +0 -0
  36. data/test/fixtures/shipit/statuses.yml +10 -0
  37. data/test/models/commits_test.rb +26 -20
  38. data/test/models/deploys_test.rb +2 -2
  39. data/test/models/stacks_test.rb +9 -12
  40. data/test/models/status_test.rb +4 -4
  41. data/test/unit/deploy_spec_test.rb +146 -1
  42. metadata +14 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 80b89004571af2792321c0d57a3659ebb7cb21aa
4
- data.tar.gz: 06363fdf4d7f9e3ba64bf992eaf11855ecfb1061
3
+ metadata.gz: 1ed7609bec0e259e4962046f8d6107263c7429db
4
+ data.tar.gz: 835c09964fcd482736103a6d8e40f59bd2aa1f0e
5
5
  SHA512:
6
- metadata.gz: ea768d3e022264786c9ddf0b9bbc1d36fb5f8379fc362c9a285d4eab28014d17c248ea9471ff55b0ef45dafa7ac529858e632ee2c2b99aa96cb1f0f1e1d5a00a
7
- data.tar.gz: 26c9579f88a71172c84c510172d882c31a2edac8b468ef1c983ab056b5a37ff367331a79524172321abd2cecfa19b89a88c40e1784a25600de713f156d3e4009
6
+ metadata.gz: cacebc0ec7fcce7a0af8feefa79d7d48878548bd07b8a75b02878383ed89d7db4364e9b623f910bf9090b5bbbd7e4d201926f86309f265ef279ac40a1b554ee9
7
+ data.tar.gz: 6ef2234ef1e1be3d462572a817edeab7ad042a0ca47639834ad349b88308a7d9541c0d3527f63697752735b9e4c3528b597197de2e882c569db9b9ef9fbc6521
data/README.md CHANGED
@@ -33,6 +33,7 @@ This guide aims to help you [set up](#installation-and-setup), [use](#using-ship
33
33
  * [Format and content of shipit.yml](#configuring-shipit)
34
34
  * [Format and content of secrets.yml](#configuring-secrets)
35
35
  * [Script parameters](#script-parameters)
36
+ * [Configuring providers](#configuring-providers)
36
37
  * [Free samples](/examples/shipit.yml)
37
38
 
38
39
  * * *
@@ -320,6 +321,19 @@ For example:
320
321
  fetch:
321
322
  curl --silent https://app.example.com/services/ping/version
322
323
  ```
324
+ <h3 id="kubernetes">Kubernetes</h3>
325
+
326
+ **<code>kubernetes</code>** allows to specify a Kubernetes namespace and context to deploy to.
327
+
328
+ For example:
329
+ ```yml
330
+ kubernetes:
331
+ namespace: my-app-production
332
+ context: tier4
333
+ ```
334
+
335
+ **<code>kubernetes.template_dir</code>** allows to specify a Kubernetes template directory. It defaults to `./config/deploy/$ENVIRONMENT`
336
+
323
337
  <h3 id="environment">Environment</h3>
324
338
 
325
339
  **<code>machine.environment</code>** contains the extra environment variables that you want to provide during task execution.
@@ -463,6 +477,24 @@ review:
463
477
  - bundle exec rake db:migrate:status
464
478
  ```
465
479
 
480
+ <h3 id="shell-commands-timeout">Shell commands timeout</h3>
481
+
482
+ All the shell commands can take an optional `timeout` parameter to limit their duration:
483
+
484
+ ```yml
485
+ deploy:
486
+ override:
487
+ - ./script/deploy:
488
+ timeout: 30
489
+ post:
490
+ - ./script/notify_deploy_end: { timeout: 15 }
491
+ review:
492
+ checks:
493
+ - bundle exec rake db:migrate:status:
494
+ timeout: 60
495
+ ```
496
+
497
+ See also `commands_inactivity_timeout` in `secrets.yml` for a global timeout setting.
466
498
 
467
499
 
468
500
  ***
@@ -576,3 +608,16 @@ These variables are accessible only during deploys and rollback:
576
608
 
577
609
  * `REVISION`: the git SHA of the revision that must be deployed in production
578
610
  * `SHA`: alias for REVISION
611
+
612
+ <h2 id="configuring-providers">Configuring providers</h2>
613
+
614
+ ### Heroku
615
+
616
+ To use Heroku integration (`lib/snippets/push-to-heroku`), make sure that the environment has [Heroku toolbelt](https://devcenter.heroku.com/articles/heroku-cli) available.
617
+
618
+ ### Kubernetes
619
+
620
+ For Kubernetes, you have to provision Shipit environment with the following tools:
621
+
622
+ * `kubectl`
623
+ * `kubernetes-deploy` [gem](https://github.com/Shopify/kubernetes-deploy)
@@ -0,0 +1,9 @@
1
+ module Shipit
2
+ class DeferredTouchJob < BackgroundJob
3
+ include BackgroundJob::Unique
4
+
5
+ def perform
6
+ DeferredTouch.touch_now!
7
+ end
8
+ end
9
+ end
@@ -52,7 +52,7 @@ module Shipit
52
52
  end
53
53
 
54
54
  def perform_task
55
- Bundler.with_clean_env do
55
+ Bundler.with_original_env do
56
56
  capture_all! @commands.install_dependencies
57
57
  capture_all! @commands.perform
58
58
  end
@@ -0,0 +1,92 @@
1
+ module Shipit
2
+ module DeferredTouch
3
+ extend ActiveSupport::Concern
4
+
5
+ SET_KEY = 'shipit:deferred_touches'.freeze
6
+ TMP_KEY = "#{SET_KEY}:updating".freeze
7
+ CACHE_KEY = "#{SET_KEY}:scheduled".freeze
8
+ THROTTLE_TTL = 1.second
9
+
10
+ included do
11
+ class_attribute :deferred_touches, instance_accessor: false
12
+ after_commit :schedule_touches
13
+ end
14
+
15
+ class << self
16
+ attr_accessor :enabled
17
+
18
+ def touch_now!
19
+ now = Time.now.utc
20
+ fetch do |touches|
21
+ records = []
22
+ touches.each do |model, changes|
23
+ changes.each do |attribute, ids|
24
+ records << [model.where(id: ids).to_a, attribute]
25
+ end
26
+ end
27
+
28
+ ActiveRecord::Base.transaction do
29
+ records.each do |instances, attribute|
30
+ instances.each { |i| i.touch(attribute, time: now) }
31
+ end
32
+ end
33
+ end
34
+ end
35
+
36
+ private
37
+
38
+ def fetch
39
+ fetch_members do |records|
40
+ return if records.empty?
41
+ records = records.each_with_object({}) do |(model, id, attribute), hash|
42
+ attributes = (hash[model] ||= {})
43
+ ids = (attributes[attribute] ||= [])
44
+ ids << id
45
+ end
46
+ yield records.transform_keys(&:constantize)
47
+ end
48
+ end
49
+
50
+ def fetch_members
51
+ Shipit.redis.multi do
52
+ Shipit.redis.sunionstore(TMP_KEY, SET_KEY)
53
+ Shipit.redis.del(SET_KEY)
54
+ end
55
+
56
+ yield Shipit.redis.smembers(TMP_KEY).map { |r| r.split('|') }
57
+
58
+ Shipit.redis.del(TMP_KEY)
59
+ end
60
+ end
61
+
62
+ self.enabled = true
63
+
64
+ module ClassMethods
65
+ def deferred_touch(touches)
66
+ touches = touches.transform_values(&Array.method(:wrap))
67
+ self.deferred_touches = touches.flat_map do |association_name, attributes|
68
+ association = reflect_on_association(association_name)
69
+ Array.wrap(attributes).map do |attribute|
70
+ [association.klass.name, association.foreign_key, attribute]
71
+ end
72
+ end
73
+ end
74
+ end
75
+
76
+ private
77
+
78
+ def schedule_touches
79
+ return unless self.class.deferred_touches
80
+ touches = self.class.deferred_touches.map { |m, fk, a| [m, self[fk], a].join('|') }
81
+ Shipit.redis.sadd(SET_KEY, touches)
82
+ if DeferredTouch.enabled
83
+ Rails.cache.fetch(CACHE_KEY, expires_in: THROTTLE_TTL) do
84
+ DeferredTouchJob.perform_later
85
+ true
86
+ end
87
+ else
88
+ DeferredTouch.touch_now!
89
+ end
90
+ end
91
+ end
92
+ end
@@ -1,19 +1,21 @@
1
1
  module Shipit
2
2
  class Commit < ActiveRecord::Base
3
+ include DeferredTouch
4
+
3
5
  AmbiguousRevision = Class.new(StandardError)
4
6
 
5
- belongs_to :stack, touch: true
7
+ belongs_to :stack
6
8
  has_many :deploys
7
- has_many :statuses, -> { order(created_at: :desc) }
9
+ has_many :statuses, -> { order(created_at: :desc) }, dependent: :destroy
8
10
  has_many :commit_deployments, dependent: :destroy
9
11
 
12
+ deferred_touch stack: :updated_at
13
+
10
14
  after_commit { broadcast_update }
11
15
  after_create { stack.update_undeployed_commits_count }
12
16
 
13
17
  after_commit :schedule_refresh_statuses!, :schedule_fetch_stats!, :schedule_continuous_delivery, on: :create
14
18
 
15
- after_touch :touch_stack
16
-
17
19
  belongs_to :author, class_name: 'User', inverse_of: :authored_commits
18
20
  belongs_to :committer, class_name: 'User', inverse_of: :commits
19
21
 
@@ -88,7 +90,7 @@ module Shipit
88
90
 
89
91
  def create_status_from_github!(github_status)
90
92
  add_status do
91
- statuses.replicate_from_github!(github_status)
93
+ statuses.replicate_from_github!(stack_id, github_status)
92
94
  end
93
95
  end
94
96
 
@@ -207,9 +209,5 @@ module Shipit
207
209
  def missing_statuses
208
210
  stack.required_statuses - last_statuses.map(&:context)
209
211
  end
210
-
211
- def touch_stack
212
- stack.touch
213
- end
214
212
  end
215
213
  end
@@ -5,6 +5,8 @@ module Shipit
5
5
 
6
6
  belongs_to :hook
7
7
 
8
+ scope :due_for_deletion, -> { where('created_at < ?', 1.month.ago) }
9
+
8
10
  validates :url, presence: true, url: {no_local: true, allow_blank: true}
9
11
  validates :content_type, presence: true
10
12
 
@@ -85,6 +85,10 @@ module Shipit
85
85
  Array.wrap(config('deploy', 'variables')).map(&VariableDefinition.method(:new))
86
86
  end
87
87
 
88
+ def default_deploy_env
89
+ deploy_variables.map { |v| [v.name, v.default] }.to_h
90
+ end
91
+
88
92
  def rollback_steps
89
93
  around_steps('rollback') do
90
94
  config('rollback', 'override') { discover_rollback_steps }
@@ -182,6 +186,10 @@ module Shipit
182
186
  def discover_fetch_deployed_revision_steps
183
187
  end
184
188
 
189
+ def discover_machine_env
190
+ {}
191
+ end
192
+
185
193
  def task_not_found!(id)
186
194
  raise TaskDefinition::NotFound.new("No definition for task #{id.inspect}")
187
195
  end
@@ -1,6 +1,8 @@
1
1
  module Shipit
2
2
  class DeploySpec
3
3
  class FileSystem < DeploySpec
4
+ include NpmDiscovery
5
+ include KubernetesDiscovery
4
6
  include PypiDiscovery
5
7
  include RubygemsDiscovery
6
8
  include CapistranoDiscovery
@@ -33,7 +35,7 @@ module Shipit
33
35
  'require' => required_statuses,
34
36
  },
35
37
  'machine' => {
36
- 'environment' => machine_env,
38
+ 'environment' => discover_machine_env.merge(machine_env),
37
39
  'directory' => directory,
38
40
  'cleanup' => true,
39
41
  },
@@ -0,0 +1,37 @@
1
+ module Shipit
2
+ class DeploySpec
3
+ module KubernetesDiscovery
4
+ def discover_deploy_steps
5
+ discover_kubernetes || super
6
+ end
7
+
8
+ def discover_rollback_steps
9
+ discover_kubernetes || super
10
+ end
11
+
12
+ def discover_machine_env
13
+ env = super
14
+ env = env.merge('K8S_TEMPLATE_FOLDER' => kube_config['template_dir']) if kube_config['template_dir']
15
+ env
16
+ end
17
+
18
+ private
19
+
20
+ def discover_kubernetes
21
+ return unless kube_config.present?
22
+
23
+ [
24
+ Shellwords.join([
25
+ "kubernetes-deploy",
26
+ kube_config['namespace'],
27
+ kube_config['context'],
28
+ ]),
29
+ ]
30
+ end
31
+
32
+ def kube_config
33
+ @kube_config ||= config('kubernetes') || {}
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,81 @@
1
+ require 'json'
2
+
3
+ module Shipit
4
+ class DeploySpec
5
+ module NpmDiscovery
6
+ def discover_dependencies_steps
7
+ discover_package_json || super
8
+ end
9
+
10
+ def discover_package_json
11
+ npm_install if yarn? || npm?
12
+ end
13
+
14
+ def npm_install
15
+ [js_command('install --no-progress')]
16
+ end
17
+
18
+ def discover_review_checklist
19
+ discover_yarn_checklist || discover_npm_checklist || super
20
+ end
21
+
22
+ def discover_yarn_checklist
23
+ [%(<strong>Don't forget version and tag before publishing!</strong> You can do this with:<br/>
24
+ yarn version --new-version <strong>&lt;major|minor|patch&gt;</strong> && git push --tags</pre>)] if yarn?
25
+ end
26
+
27
+ def discover_npm_checklist
28
+ [%(<strong>Don't forget version and tag before publishing!</strong> You can do this with:<br/>
29
+ npm version <strong>&lt;major|minor|patch&gt;</strong> && git push --tags</pre>)] if npm?
30
+ end
31
+
32
+ def npm?
33
+ public?
34
+ end
35
+
36
+ def public?
37
+ file = package_json
38
+ return false unless file.exist?
39
+
40
+ JSON.parse(file.read)['private'].blank?
41
+ end
42
+
43
+ def package_json
44
+ file('package.json')
45
+ end
46
+
47
+ def yarn?
48
+ yarn_lock.exist? && public?
49
+ end
50
+
51
+ def yarn_lock
52
+ file('./yarn.lock')
53
+ end
54
+
55
+ def discover_npm_package
56
+ publish_npm_package if yarn? || npm?
57
+ end
58
+
59
+ def discover_deploy_steps
60
+ discover_npm_package || super
61
+ end
62
+
63
+ def publish_npm_package
64
+ check_tags = 'assert-npm-version-tag'
65
+ publish = js_command('publish')
66
+
67
+ [check_tags, publish]
68
+ end
69
+
70
+ def js_command(command_args)
71
+ runner = if yarn?
72
+ 'yarn'
73
+ else
74
+ 'npm'
75
+ end
76
+
77
+ "#{runner} #{command_args}"
78
+ end
79
+ end
80
+ end
81
+ end
@@ -50,7 +50,6 @@ module Shipit
50
50
  after_commit :broadcast_update, on: :update
51
51
  after_commit :emit_merge_status_hooks, on: :update
52
52
  after_commit :setup_hooks, :sync_github, on: :create
53
- after_touch :clear_cache
54
53
 
55
54
  validates :repo_name, uniqueness: {scope: %i(repo_owner environment),
56
55
  message: 'cannot be used more than once with this environment'}
@@ -136,7 +135,7 @@ module Shipit
136
135
  return
137
136
  end
138
137
 
139
- trigger_deploy(commit, Shipit.user)
138
+ trigger_deploy(commit, Shipit.user, env: cached_deploy_spec.default_deploy_env)
140
139
  end
141
140
 
142
141
  def next_commit_to_deploy
@@ -1,21 +1,26 @@
1
1
  module Shipit
2
2
  class Status < ActiveRecord::Base
3
+ include DeferredTouch
4
+
3
5
  STATES = %w(pending success failure error).freeze
4
6
  enum state: STATES.zip(STATES).to_h
5
7
 
6
- belongs_to :commit, touch: true
8
+ belongs_to :stack, required: true
9
+ belongs_to :commit, required: true
10
+
11
+ deferred_touch stack: :updated_at, commit: :updated_at
7
12
 
8
13
  validates :state, inclusion: {in: STATES, allow_blank: true}, presence: true
9
14
 
10
15
  after_create :enable_ci_on_stack
11
16
  after_commit :schedule_continuous_delivery, :broadcast_update, on: :create
12
- after_commit :touch_commit
13
17
 
14
18
  delegate :broadcast_update, to: :commit
15
19
 
16
20
  class << self
17
- def replicate_from_github!(github_status)
21
+ def replicate_from_github!(stack_id, github_status)
18
22
  find_or_create_by!(
23
+ stack_id: stack_id,
19
24
  state: github_status.state,
20
25
  description: github_status.description,
21
26
  target_url: github_status.target_url,
@@ -25,8 +30,6 @@ module Shipit
25
30
  end
26
31
  end
27
32
 
28
- delegate :stack, to: :commit
29
-
30
33
  def unknown?
31
34
  false
32
35
  end
@@ -49,10 +52,6 @@ module Shipit
49
52
  commit.stack.enable_ci!
50
53
  end
51
54
 
52
- def touch_commit
53
- commit.touch
54
- end
55
-
56
55
  def schedule_continuous_delivery
57
56
  commit.schedule_continuous_delivery
58
57
  end