shipit-engine 0.14.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +45 -0
- data/app/jobs/shipit/deferred_touch_job.rb +9 -0
- data/app/jobs/shipit/perform_task_job.rb +1 -1
- data/app/jobs/shipit/{update_estimated_deploy_duration.rb → update_estimated_deploy_duration_job.rb} +0 -0
- data/app/models/concerns/shipit/deferred_touch.rb +92 -0
- data/app/models/shipit/commit.rb +7 -9
- data/app/models/shipit/delivery.rb +2 -0
- data/app/models/shipit/deploy_spec.rb +8 -0
- data/app/models/shipit/deploy_spec/file_system.rb +3 -1
- data/app/models/shipit/deploy_spec/kubernetes_discovery.rb +37 -0
- data/app/models/shipit/deploy_spec/npm_discovery.rb +81 -0
- data/app/models/shipit/stack.rb +1 -2
- data/app/models/shipit/status.rb +8 -9
- data/app/models/shipit/task.rb +5 -1
- data/config/secrets.development.example.yml +4 -0
- data/config/secrets.development.shopify.yml +4 -0
- data/config/secrets.development.yml +1 -1
- data/db/migrate/20161205144522_add_indexes_on_deliveries.rb +17 -0
- data/db/migrate/20161206104100_delete_orphan_statuses.rb +10 -0
- data/db/migrate/20161206104224_denormalize_stack_id_on_statuses.rb +5 -0
- data/db/migrate/20161206104817_backfill_stack_id_on_statuses.rb +13 -0
- data/db/migrate/20161206105318_makes_stack_id_not_null_on_statuses.rb +5 -0
- data/lib/shipit.rb +3 -0
- data/lib/shipit/strip_cache_control.rb +40 -0
- data/lib/shipit/version.rb +1 -1
- data/lib/snippets/assert-npm-version-tag +22 -0
- data/lib/snippets/deploy-to-gke +3 -4
- data/lib/tasks/cron.rake +7 -0
- data/test/dummy/config/environments/development.rb +6 -2
- data/test/dummy/config/environments/test.rb +4 -0
- data/test/dummy/db/development.sqlite3 +0 -0
- data/test/dummy/db/schema.rb +32 -42
- data/test/dummy/db/seeds.rb +2 -0
- data/test/dummy/db/test.sqlite3 +0 -0
- data/test/fixtures/shipit/statuses.yml +10 -0
- data/test/models/commits_test.rb +26 -20
- data/test/models/deploys_test.rb +2 -2
- data/test/models/stacks_test.rb +9 -12
- data/test/models/status_test.rb +4 -4
- data/test/unit/deploy_spec_test.rb +146 -1
- metadata +14 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1ed7609bec0e259e4962046f8d6107263c7429db
|
4
|
+
data.tar.gz: 835c09964fcd482736103a6d8e40f59bd2aa1f0e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cacebc0ec7fcce7a0af8feefa79d7d48878548bd07b8a75b02878383ed89d7db4364e9b623f910bf9090b5bbbd7e4d201926f86309f265ef279ac40a1b554ee9
|
7
|
+
data.tar.gz: 6ef2234ef1e1be3d462572a817edeab7ad042a0ca47639834ad349b88308a7d9541c0d3527f63697752735b9e4c3528b597197de2e882c569db9b9ef9fbc6521
|
data/README.md
CHANGED
@@ -33,6 +33,7 @@ This guide aims to help you [set up](#installation-and-setup), [use](#using-ship
|
|
33
33
|
* [Format and content of shipit.yml](#configuring-shipit)
|
34
34
|
* [Format and content of secrets.yml](#configuring-secrets)
|
35
35
|
* [Script parameters](#script-parameters)
|
36
|
+
* [Configuring providers](#configuring-providers)
|
36
37
|
* [Free samples](/examples/shipit.yml)
|
37
38
|
|
38
39
|
* * *
|
@@ -320,6 +321,19 @@ For example:
|
|
320
321
|
fetch:
|
321
322
|
curl --silent https://app.example.com/services/ping/version
|
322
323
|
```
|
324
|
+
<h3 id="kubernetes">Kubernetes</h3>
|
325
|
+
|
326
|
+
**<code>kubernetes</code>** allows to specify a Kubernetes namespace and context to deploy to.
|
327
|
+
|
328
|
+
For example:
|
329
|
+
```yml
|
330
|
+
kubernetes:
|
331
|
+
namespace: my-app-production
|
332
|
+
context: tier4
|
333
|
+
```
|
334
|
+
|
335
|
+
**<code>kubernetes.template_dir</code>** allows to specify a Kubernetes template directory. It defaults to `./config/deploy/$ENVIRONMENT`
|
336
|
+
|
323
337
|
<h3 id="environment">Environment</h3>
|
324
338
|
|
325
339
|
**<code>machine.environment</code>** contains the extra environment variables that you want to provide during task execution.
|
@@ -463,6 +477,24 @@ review:
|
|
463
477
|
- bundle exec rake db:migrate:status
|
464
478
|
```
|
465
479
|
|
480
|
+
<h3 id="shell-commands-timeout">Shell commands timeout</h3>
|
481
|
+
|
482
|
+
All the shell commands can take an optional `timeout` parameter to limit their duration:
|
483
|
+
|
484
|
+
```yml
|
485
|
+
deploy:
|
486
|
+
override:
|
487
|
+
- ./script/deploy:
|
488
|
+
timeout: 30
|
489
|
+
post:
|
490
|
+
- ./script/notify_deploy_end: { timeout: 15 }
|
491
|
+
review:
|
492
|
+
checks:
|
493
|
+
- bundle exec rake db:migrate:status:
|
494
|
+
timeout: 60
|
495
|
+
```
|
496
|
+
|
497
|
+
See also `commands_inactivity_timeout` in `secrets.yml` for a global timeout setting.
|
466
498
|
|
467
499
|
|
468
500
|
***
|
@@ -576,3 +608,16 @@ These variables are accessible only during deploys and rollback:
|
|
576
608
|
|
577
609
|
* `REVISION`: the git SHA of the revision that must be deployed in production
|
578
610
|
* `SHA`: alias for REVISION
|
611
|
+
|
612
|
+
<h2 id="configuring-providers">Configuring providers</h2>
|
613
|
+
|
614
|
+
### Heroku
|
615
|
+
|
616
|
+
To use Heroku integration (`lib/snippets/push-to-heroku`), make sure that the environment has [Heroku toolbelt](https://devcenter.heroku.com/articles/heroku-cli) available.
|
617
|
+
|
618
|
+
### Kubernetes
|
619
|
+
|
620
|
+
For Kubernetes, you have to provision Shipit environment with the following tools:
|
621
|
+
|
622
|
+
* `kubectl`
|
623
|
+
* `kubernetes-deploy` [gem](https://github.com/Shopify/kubernetes-deploy)
|
data/app/jobs/shipit/{update_estimated_deploy_duration.rb → update_estimated_deploy_duration_job.rb}
RENAMED
File without changes
|
@@ -0,0 +1,92 @@
|
|
1
|
+
module Shipit
|
2
|
+
module DeferredTouch
|
3
|
+
extend ActiveSupport::Concern
|
4
|
+
|
5
|
+
SET_KEY = 'shipit:deferred_touches'.freeze
|
6
|
+
TMP_KEY = "#{SET_KEY}:updating".freeze
|
7
|
+
CACHE_KEY = "#{SET_KEY}:scheduled".freeze
|
8
|
+
THROTTLE_TTL = 1.second
|
9
|
+
|
10
|
+
included do
|
11
|
+
class_attribute :deferred_touches, instance_accessor: false
|
12
|
+
after_commit :schedule_touches
|
13
|
+
end
|
14
|
+
|
15
|
+
class << self
|
16
|
+
attr_accessor :enabled
|
17
|
+
|
18
|
+
def touch_now!
|
19
|
+
now = Time.now.utc
|
20
|
+
fetch do |touches|
|
21
|
+
records = []
|
22
|
+
touches.each do |model, changes|
|
23
|
+
changes.each do |attribute, ids|
|
24
|
+
records << [model.where(id: ids).to_a, attribute]
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
ActiveRecord::Base.transaction do
|
29
|
+
records.each do |instances, attribute|
|
30
|
+
instances.each { |i| i.touch(attribute, time: now) }
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def fetch
|
39
|
+
fetch_members do |records|
|
40
|
+
return if records.empty?
|
41
|
+
records = records.each_with_object({}) do |(model, id, attribute), hash|
|
42
|
+
attributes = (hash[model] ||= {})
|
43
|
+
ids = (attributes[attribute] ||= [])
|
44
|
+
ids << id
|
45
|
+
end
|
46
|
+
yield records.transform_keys(&:constantize)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def fetch_members
|
51
|
+
Shipit.redis.multi do
|
52
|
+
Shipit.redis.sunionstore(TMP_KEY, SET_KEY)
|
53
|
+
Shipit.redis.del(SET_KEY)
|
54
|
+
end
|
55
|
+
|
56
|
+
yield Shipit.redis.smembers(TMP_KEY).map { |r| r.split('|') }
|
57
|
+
|
58
|
+
Shipit.redis.del(TMP_KEY)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
self.enabled = true
|
63
|
+
|
64
|
+
module ClassMethods
|
65
|
+
def deferred_touch(touches)
|
66
|
+
touches = touches.transform_values(&Array.method(:wrap))
|
67
|
+
self.deferred_touches = touches.flat_map do |association_name, attributes|
|
68
|
+
association = reflect_on_association(association_name)
|
69
|
+
Array.wrap(attributes).map do |attribute|
|
70
|
+
[association.klass.name, association.foreign_key, attribute]
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
private
|
77
|
+
|
78
|
+
def schedule_touches
|
79
|
+
return unless self.class.deferred_touches
|
80
|
+
touches = self.class.deferred_touches.map { |m, fk, a| [m, self[fk], a].join('|') }
|
81
|
+
Shipit.redis.sadd(SET_KEY, touches)
|
82
|
+
if DeferredTouch.enabled
|
83
|
+
Rails.cache.fetch(CACHE_KEY, expires_in: THROTTLE_TTL) do
|
84
|
+
DeferredTouchJob.perform_later
|
85
|
+
true
|
86
|
+
end
|
87
|
+
else
|
88
|
+
DeferredTouch.touch_now!
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
data/app/models/shipit/commit.rb
CHANGED
@@ -1,19 +1,21 @@
|
|
1
1
|
module Shipit
|
2
2
|
class Commit < ActiveRecord::Base
|
3
|
+
include DeferredTouch
|
4
|
+
|
3
5
|
AmbiguousRevision = Class.new(StandardError)
|
4
6
|
|
5
|
-
belongs_to :stack
|
7
|
+
belongs_to :stack
|
6
8
|
has_many :deploys
|
7
|
-
has_many :statuses, -> { order(created_at: :desc) }
|
9
|
+
has_many :statuses, -> { order(created_at: :desc) }, dependent: :destroy
|
8
10
|
has_many :commit_deployments, dependent: :destroy
|
9
11
|
|
12
|
+
deferred_touch stack: :updated_at
|
13
|
+
|
10
14
|
after_commit { broadcast_update }
|
11
15
|
after_create { stack.update_undeployed_commits_count }
|
12
16
|
|
13
17
|
after_commit :schedule_refresh_statuses!, :schedule_fetch_stats!, :schedule_continuous_delivery, on: :create
|
14
18
|
|
15
|
-
after_touch :touch_stack
|
16
|
-
|
17
19
|
belongs_to :author, class_name: 'User', inverse_of: :authored_commits
|
18
20
|
belongs_to :committer, class_name: 'User', inverse_of: :commits
|
19
21
|
|
@@ -88,7 +90,7 @@ module Shipit
|
|
88
90
|
|
89
91
|
def create_status_from_github!(github_status)
|
90
92
|
add_status do
|
91
|
-
statuses.replicate_from_github!(github_status)
|
93
|
+
statuses.replicate_from_github!(stack_id, github_status)
|
92
94
|
end
|
93
95
|
end
|
94
96
|
|
@@ -207,9 +209,5 @@ module Shipit
|
|
207
209
|
def missing_statuses
|
208
210
|
stack.required_statuses - last_statuses.map(&:context)
|
209
211
|
end
|
210
|
-
|
211
|
-
def touch_stack
|
212
|
-
stack.touch
|
213
|
-
end
|
214
212
|
end
|
215
213
|
end
|
@@ -85,6 +85,10 @@ module Shipit
|
|
85
85
|
Array.wrap(config('deploy', 'variables')).map(&VariableDefinition.method(:new))
|
86
86
|
end
|
87
87
|
|
88
|
+
def default_deploy_env
|
89
|
+
deploy_variables.map { |v| [v.name, v.default] }.to_h
|
90
|
+
end
|
91
|
+
|
88
92
|
def rollback_steps
|
89
93
|
around_steps('rollback') do
|
90
94
|
config('rollback', 'override') { discover_rollback_steps }
|
@@ -182,6 +186,10 @@ module Shipit
|
|
182
186
|
def discover_fetch_deployed_revision_steps
|
183
187
|
end
|
184
188
|
|
189
|
+
def discover_machine_env
|
190
|
+
{}
|
191
|
+
end
|
192
|
+
|
185
193
|
def task_not_found!(id)
|
186
194
|
raise TaskDefinition::NotFound.new("No definition for task #{id.inspect}")
|
187
195
|
end
|
@@ -1,6 +1,8 @@
|
|
1
1
|
module Shipit
|
2
2
|
class DeploySpec
|
3
3
|
class FileSystem < DeploySpec
|
4
|
+
include NpmDiscovery
|
5
|
+
include KubernetesDiscovery
|
4
6
|
include PypiDiscovery
|
5
7
|
include RubygemsDiscovery
|
6
8
|
include CapistranoDiscovery
|
@@ -33,7 +35,7 @@ module Shipit
|
|
33
35
|
'require' => required_statuses,
|
34
36
|
},
|
35
37
|
'machine' => {
|
36
|
-
'environment' => machine_env,
|
38
|
+
'environment' => discover_machine_env.merge(machine_env),
|
37
39
|
'directory' => directory,
|
38
40
|
'cleanup' => true,
|
39
41
|
},
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Shipit
|
2
|
+
class DeploySpec
|
3
|
+
module KubernetesDiscovery
|
4
|
+
def discover_deploy_steps
|
5
|
+
discover_kubernetes || super
|
6
|
+
end
|
7
|
+
|
8
|
+
def discover_rollback_steps
|
9
|
+
discover_kubernetes || super
|
10
|
+
end
|
11
|
+
|
12
|
+
def discover_machine_env
|
13
|
+
env = super
|
14
|
+
env = env.merge('K8S_TEMPLATE_FOLDER' => kube_config['template_dir']) if kube_config['template_dir']
|
15
|
+
env
|
16
|
+
end
|
17
|
+
|
18
|
+
private
|
19
|
+
|
20
|
+
def discover_kubernetes
|
21
|
+
return unless kube_config.present?
|
22
|
+
|
23
|
+
[
|
24
|
+
Shellwords.join([
|
25
|
+
"kubernetes-deploy",
|
26
|
+
kube_config['namespace'],
|
27
|
+
kube_config['context'],
|
28
|
+
]),
|
29
|
+
]
|
30
|
+
end
|
31
|
+
|
32
|
+
def kube_config
|
33
|
+
@kube_config ||= config('kubernetes') || {}
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module Shipit
|
4
|
+
class DeploySpec
|
5
|
+
module NpmDiscovery
|
6
|
+
def discover_dependencies_steps
|
7
|
+
discover_package_json || super
|
8
|
+
end
|
9
|
+
|
10
|
+
def discover_package_json
|
11
|
+
npm_install if yarn? || npm?
|
12
|
+
end
|
13
|
+
|
14
|
+
def npm_install
|
15
|
+
[js_command('install --no-progress')]
|
16
|
+
end
|
17
|
+
|
18
|
+
def discover_review_checklist
|
19
|
+
discover_yarn_checklist || discover_npm_checklist || super
|
20
|
+
end
|
21
|
+
|
22
|
+
def discover_yarn_checklist
|
23
|
+
[%(<strong>Don't forget version and tag before publishing!</strong> You can do this with:<br/>
|
24
|
+
yarn version --new-version <strong><major|minor|patch></strong> && git push --tags</pre>)] if yarn?
|
25
|
+
end
|
26
|
+
|
27
|
+
def discover_npm_checklist
|
28
|
+
[%(<strong>Don't forget version and tag before publishing!</strong> You can do this with:<br/>
|
29
|
+
npm version <strong><major|minor|patch></strong> && git push --tags</pre>)] if npm?
|
30
|
+
end
|
31
|
+
|
32
|
+
def npm?
|
33
|
+
public?
|
34
|
+
end
|
35
|
+
|
36
|
+
def public?
|
37
|
+
file = package_json
|
38
|
+
return false unless file.exist?
|
39
|
+
|
40
|
+
JSON.parse(file.read)['private'].blank?
|
41
|
+
end
|
42
|
+
|
43
|
+
def package_json
|
44
|
+
file('package.json')
|
45
|
+
end
|
46
|
+
|
47
|
+
def yarn?
|
48
|
+
yarn_lock.exist? && public?
|
49
|
+
end
|
50
|
+
|
51
|
+
def yarn_lock
|
52
|
+
file('./yarn.lock')
|
53
|
+
end
|
54
|
+
|
55
|
+
def discover_npm_package
|
56
|
+
publish_npm_package if yarn? || npm?
|
57
|
+
end
|
58
|
+
|
59
|
+
def discover_deploy_steps
|
60
|
+
discover_npm_package || super
|
61
|
+
end
|
62
|
+
|
63
|
+
def publish_npm_package
|
64
|
+
check_tags = 'assert-npm-version-tag'
|
65
|
+
publish = js_command('publish')
|
66
|
+
|
67
|
+
[check_tags, publish]
|
68
|
+
end
|
69
|
+
|
70
|
+
def js_command(command_args)
|
71
|
+
runner = if yarn?
|
72
|
+
'yarn'
|
73
|
+
else
|
74
|
+
'npm'
|
75
|
+
end
|
76
|
+
|
77
|
+
"#{runner} #{command_args}"
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
data/app/models/shipit/stack.rb
CHANGED
@@ -50,7 +50,6 @@ module Shipit
|
|
50
50
|
after_commit :broadcast_update, on: :update
|
51
51
|
after_commit :emit_merge_status_hooks, on: :update
|
52
52
|
after_commit :setup_hooks, :sync_github, on: :create
|
53
|
-
after_touch :clear_cache
|
54
53
|
|
55
54
|
validates :repo_name, uniqueness: {scope: %i(repo_owner environment),
|
56
55
|
message: 'cannot be used more than once with this environment'}
|
@@ -136,7 +135,7 @@ module Shipit
|
|
136
135
|
return
|
137
136
|
end
|
138
137
|
|
139
|
-
trigger_deploy(commit, Shipit.user)
|
138
|
+
trigger_deploy(commit, Shipit.user, env: cached_deploy_spec.default_deploy_env)
|
140
139
|
end
|
141
140
|
|
142
141
|
def next_commit_to_deploy
|
data/app/models/shipit/status.rb
CHANGED
@@ -1,21 +1,26 @@
|
|
1
1
|
module Shipit
|
2
2
|
class Status < ActiveRecord::Base
|
3
|
+
include DeferredTouch
|
4
|
+
|
3
5
|
STATES = %w(pending success failure error).freeze
|
4
6
|
enum state: STATES.zip(STATES).to_h
|
5
7
|
|
6
|
-
belongs_to :
|
8
|
+
belongs_to :stack, required: true
|
9
|
+
belongs_to :commit, required: true
|
10
|
+
|
11
|
+
deferred_touch stack: :updated_at, commit: :updated_at
|
7
12
|
|
8
13
|
validates :state, inclusion: {in: STATES, allow_blank: true}, presence: true
|
9
14
|
|
10
15
|
after_create :enable_ci_on_stack
|
11
16
|
after_commit :schedule_continuous_delivery, :broadcast_update, on: :create
|
12
|
-
after_commit :touch_commit
|
13
17
|
|
14
18
|
delegate :broadcast_update, to: :commit
|
15
19
|
|
16
20
|
class << self
|
17
|
-
def replicate_from_github!(github_status)
|
21
|
+
def replicate_from_github!(stack_id, github_status)
|
18
22
|
find_or_create_by!(
|
23
|
+
stack_id: stack_id,
|
19
24
|
state: github_status.state,
|
20
25
|
description: github_status.description,
|
21
26
|
target_url: github_status.target_url,
|
@@ -25,8 +30,6 @@ module Shipit
|
|
25
30
|
end
|
26
31
|
end
|
27
32
|
|
28
|
-
delegate :stack, to: :commit
|
29
|
-
|
30
33
|
def unknown?
|
31
34
|
false
|
32
35
|
end
|
@@ -49,10 +52,6 @@ module Shipit
|
|
49
52
|
commit.stack.enable_ci!
|
50
53
|
end
|
51
54
|
|
52
|
-
def touch_commit
|
53
|
-
commit.touch
|
54
|
-
end
|
55
|
-
|
56
55
|
def schedule_continuous_delivery
|
57
56
|
commit.schedule_continuous_delivery
|
58
57
|
end
|