shipit-engine 0.14.0 → 0.15.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +45 -0
  3. data/app/jobs/shipit/deferred_touch_job.rb +9 -0
  4. data/app/jobs/shipit/perform_task_job.rb +1 -1
  5. data/app/jobs/shipit/{update_estimated_deploy_duration.rb → update_estimated_deploy_duration_job.rb} +0 -0
  6. data/app/models/concerns/shipit/deferred_touch.rb +92 -0
  7. data/app/models/shipit/commit.rb +7 -9
  8. data/app/models/shipit/delivery.rb +2 -0
  9. data/app/models/shipit/deploy_spec.rb +8 -0
  10. data/app/models/shipit/deploy_spec/file_system.rb +3 -1
  11. data/app/models/shipit/deploy_spec/kubernetes_discovery.rb +37 -0
  12. data/app/models/shipit/deploy_spec/npm_discovery.rb +81 -0
  13. data/app/models/shipit/stack.rb +1 -2
  14. data/app/models/shipit/status.rb +8 -9
  15. data/app/models/shipit/task.rb +5 -1
  16. data/config/secrets.development.example.yml +4 -0
  17. data/config/secrets.development.shopify.yml +4 -0
  18. data/config/secrets.development.yml +1 -1
  19. data/db/migrate/20161205144522_add_indexes_on_deliveries.rb +17 -0
  20. data/db/migrate/20161206104100_delete_orphan_statuses.rb +10 -0
  21. data/db/migrate/20161206104224_denormalize_stack_id_on_statuses.rb +5 -0
  22. data/db/migrate/20161206104817_backfill_stack_id_on_statuses.rb +13 -0
  23. data/db/migrate/20161206105318_makes_stack_id_not_null_on_statuses.rb +5 -0
  24. data/lib/shipit.rb +3 -0
  25. data/lib/shipit/strip_cache_control.rb +40 -0
  26. data/lib/shipit/version.rb +1 -1
  27. data/lib/snippets/assert-npm-version-tag +22 -0
  28. data/lib/snippets/deploy-to-gke +3 -4
  29. data/lib/tasks/cron.rake +7 -0
  30. data/test/dummy/config/environments/development.rb +6 -2
  31. data/test/dummy/config/environments/test.rb +4 -0
  32. data/test/dummy/db/development.sqlite3 +0 -0
  33. data/test/dummy/db/schema.rb +32 -42
  34. data/test/dummy/db/seeds.rb +2 -0
  35. data/test/dummy/db/test.sqlite3 +0 -0
  36. data/test/fixtures/shipit/statuses.yml +10 -0
  37. data/test/models/commits_test.rb +26 -20
  38. data/test/models/deploys_test.rb +2 -2
  39. data/test/models/stacks_test.rb +9 -12
  40. data/test/models/status_test.rb +4 -4
  41. data/test/unit/deploy_spec_test.rb +146 -1
  42. metadata +14 -3
@@ -109,6 +109,7 @@ module Shipit
109
109
  description: "Your tests ran on travis-ci",
110
110
  target_url: "https://example.com",
111
111
  commit_id: commit.id,
112
+ stack_id: commit.stack_id,
112
113
  created_at: Time.now,
113
114
  updated_at: Time.now,
114
115
  )
@@ -120,6 +121,7 @@ module Shipit
120
121
  description: "Your tests ran on circle-ci",
121
122
  target_url: "https://example.com",
122
123
  commit_id: commit.id,
124
+ stack_id: commit.stack_id,
123
125
  created_at: Time.now,
124
126
  updated_at: Time.now,
125
127
  )
Binary file
@@ -1,6 +1,7 @@
1
1
  # Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
2
2
 
3
3
  first_pending:
4
+ stack: shipit
4
5
  commit_id: 1 # first
5
6
  description: lets go
6
7
  context: ci/travis
@@ -9,6 +10,7 @@ first_pending:
9
10
  target_url: "http://www.example.com"
10
11
 
11
12
  second_pending_travis:
13
+ stack: shipit
12
14
  commit_id: 2 # second
13
15
  description: lets go
14
16
  context: ci/travis
@@ -17,6 +19,7 @@ second_pending_travis:
17
19
  target_url: "http://www.example.com"
18
20
 
19
21
  second_pending_coveralls:
22
+ stack: shipit
20
23
  commit_id: 2 # second
21
24
  description: lets go
22
25
  context: metrics/coveralls
@@ -25,6 +28,7 @@ second_pending_coveralls:
25
28
  target_url: "http://www.example.com"
26
29
 
27
30
  second_success_travis:
31
+ stack: shipit
28
32
  commit_id: 2 # second
29
33
  description: lets go
30
34
  context: ci/travis
@@ -33,6 +37,7 @@ second_success_travis:
33
37
  target_url: "http://www.example.com"
34
38
 
35
39
  second_failure_coveralls:
40
+ stack: shipit
36
41
  commit_id: 2 # second
37
42
  description: lets go
38
43
  context: metrics/coveralls
@@ -41,6 +46,7 @@ second_failure_coveralls:
41
46
  target_url: "http://www.example.com"
42
47
 
43
48
  third_success_travis:
49
+ stack: shipit
44
50
  commit_id: 3 # third
45
51
  description: lets go
46
52
  context: ci/travis
@@ -49,6 +55,7 @@ third_success_travis:
49
55
  target_url: "http://www.example.com"
50
56
 
51
57
  third_success_coveralls:
58
+ stack: shipit
52
59
  commit_id: 3 # third
53
60
  description: lets go
54
61
  context: metrics/coveralls
@@ -57,6 +64,7 @@ third_success_coveralls:
57
64
  target_url: "http://www.example.com"
58
65
 
59
66
  fourth_pending_travis:
67
+ stack: shipit
60
68
  commit_id: 4 # fourth
61
69
  description: lets go
62
70
  context: ci/travis
@@ -65,6 +73,7 @@ fourth_pending_travis:
65
73
  target_url: "http://www.example.com"
66
74
 
67
75
  fourth_success_coveralls:
76
+ stack: shipit
68
77
  commit_id: 4 # fourth
69
78
  description: lets go
70
79
  context: metrics/coveralls
@@ -73,6 +82,7 @@ fourth_success_coveralls:
73
82
  target_url: "http://www.example.com"
74
83
 
75
84
  cyclimse_success_travis:
85
+ stack: cyclimse
76
86
  commit_id: 6 # cyclimse_first
77
87
  description: lets go
78
88
  context: ci/travis
@@ -70,7 +70,7 @@ module Shipit
70
70
 
71
71
  assert_difference "Deploy.count" do
72
72
  assert_enqueued_with(job: ContinuousDeliveryJob, args: [@stack]) do
73
- @stack.commits.last.statuses.create!(state: 'success', context: 'ci/travis')
73
+ @stack.commits.last.statuses.create!(stack_id: @stack.id, state: 'success', context: 'ci/travis')
74
74
  end
75
75
  ContinuousDeliveryJob.new.perform(@stack)
76
76
  end
@@ -81,7 +81,7 @@ module Shipit
81
81
  @stack.trigger_deploy(@commit, @commit.committer)
82
82
 
83
83
  assert_no_difference "Deploy.count" do
84
- @commit.statuses.create!(state: 'success', context: 'ci/travis')
84
+ @commit.statuses.create!(stack_id: @stack.id, state: 'success', context: 'ci/travis')
85
85
  end
86
86
  end
87
87
 
@@ -90,7 +90,7 @@ module Shipit
90
90
  @stack.reload.update!(continuous_deployment: true, lock_reason: "Maintenance ongoing")
91
91
 
92
92
  assert_no_difference "Deploy.count" do
93
- @commit.statuses.create!(state: 'success', context: 'ci/travis')
93
+ @commit.statuses.create!(stack_id: @stack.id, state: 'success', context: 'ci/travis')
94
94
  end
95
95
  end
96
96
 
@@ -116,7 +116,7 @@ module Shipit
116
116
  )
117
117
 
118
118
  assert_no_difference "Deploy.count" do
119
- @commit.statuses.create!(state: 'success')
119
+ @commit.statuses.create!(stack_id: @stack.id, state: 'success')
120
120
  end
121
121
  end
122
122
 
@@ -124,7 +124,7 @@ module Shipit
124
124
  @stack.reload.update!(continuous_deployment: true)
125
125
 
126
126
  assert_no_difference "Deploy.count" do
127
- @stack.last_deployed_commit.statuses.create!(state: 'success')
127
+ @stack.last_deployed_commit.statuses.create!(stack_id: @stack.id, state: 'success')
128
128
  end
129
129
  end
130
130
 
@@ -132,7 +132,7 @@ module Shipit
132
132
  @stack.reload.deploys.destroy_all
133
133
 
134
134
  assert_no_difference "Deploy.count" do
135
- @commit.statuses.create!(state: 'success')
135
+ @commit.statuses.create!(stack_id: @stack.id, state: 'success')
136
136
  end
137
137
  end
138
138
 
@@ -141,19 +141,21 @@ module Shipit
141
141
  @stack.deploys.destroy_all
142
142
 
143
143
  assert_no_difference "Deploy.count" do
144
- @commit.statuses.create!(state: 'failure')
144
+ @commit.statuses.create!(stack_id: @stack.id, state: 'failure')
145
145
  end
146
146
  end
147
147
 
148
148
  test "creating broadcasts an update event" do
149
149
  expect_event(@stack)
150
150
  walrus = shipit_users(:walrus)
151
- @stack.commits.create(author: walrus,
152
- committer: walrus,
153
- sha: "ab12",
154
- authored_at: DateTime.now,
155
- committed_at: DateTime.now,
156
- message: "more fish!")
151
+ @stack.commits.create(
152
+ author: walrus,
153
+ committer: walrus,
154
+ sha: "ab12",
155
+ authored_at: DateTime.now,
156
+ committed_at: DateTime.now,
157
+ message: "more fish!",
158
+ )
157
159
  end
158
160
 
159
161
  test "refresh_statuses! pull state from github" do
@@ -241,8 +243,8 @@ module Shipit
241
243
  test "#state doesn't consider statuses that are hidden or allowed to fail" do
242
244
  assert_equal 'pending', @commit.state
243
245
 
244
- @commit.statuses.create!(context: 'metrics/coveralls', state: 'failure')
245
- @commit.statuses.create!(context: 'metrics/performance', state: 'failure')
246
+ @commit.statuses.create!(stack_id: @stack.id, context: 'metrics/coveralls', state: 'failure')
247
+ @commit.statuses.create!(stack_id: @stack.id, context: 'metrics/performance', state: 'failure')
246
248
  assert_equal 'failure', @commit.reload.state
247
249
 
248
250
  @commit.stack.update!(cached_deploy_spec: DeploySpec.new('ci' => {
@@ -316,7 +318,11 @@ module Shipit
316
318
  commit.statuses.destroy_all
317
319
  commit.reload
318
320
  unless initial_state == 'unknown'
319
- commit.statuses.create!(initial_status_attributes.merge(created_at: 10.days.ago.to_s(:db)))
321
+ attrs = initial_status_attributes.merge(
322
+ stack_id: commit.stack_id,
323
+ created_at: 10.days.ago.to_s(:db),
324
+ )
325
+ commit.statuses.create!(attrs)
320
326
  end
321
327
  assert_equal initial_state, commit.state
322
328
 
@@ -388,10 +394,10 @@ module Shipit
388
394
 
389
395
  test "#significant_status hierarchy uses failures and errors, then pending, then successes, then UnknownStatus" do
390
396
  commit = shipit_commits(:first)
391
- pending = commit.statuses.new(state: 'pending', context: 'ci/pending')
392
- failure = commit.statuses.new(state: 'failure', context: 'ci/failure')
393
- error = commit.statuses.new(state: 'error', context: 'ci/error')
394
- success = commit.statuses.new(state: 'success', context: 'ci/success')
397
+ pending = commit.statuses.new(stack_id: @stack.id, state: 'pending', context: 'ci/pending')
398
+ failure = commit.statuses.new(stack_id: @stack.id, state: 'failure', context: 'ci/failure')
399
+ error = commit.statuses.new(stack_id: @stack.id, state: 'error', context: 'ci/error')
400
+ success = commit.statuses.new(stack_id: @stack.id, state: 'success', context: 'ci/success')
395
401
 
396
402
  commit.reload.statuses = [pending, failure, success, error]
397
403
  assert_includes [error, failure], commit.significant_status
@@ -185,7 +185,7 @@ module Shipit
185
185
  end
186
186
 
187
187
  test "transitioning to success triggers next deploy when stack uses CD" do
188
- shipit_commits(:fifth).statuses.create!(state: 'success')
188
+ shipit_commits(:fifth).statuses.create!(stack_id: @stack.id, state: 'success')
189
189
 
190
190
  deploy = shipit_deploys(:shipit_running)
191
191
  deploy.stack.tasks.where.not(id: deploy.id).update_all(status: 'success')
@@ -200,7 +200,7 @@ module Shipit
200
200
  end
201
201
 
202
202
  test "transitioning to success skips CD deploy when stack doesn't use it" do
203
- shipit_commits(:fifth).statuses.create!(state: 'success')
203
+ shipit_commits(:fifth).statuses.create!(stack_id: @stack.id, state: 'success')
204
204
 
205
205
  deploy = shipit_deploys(:shipit_running)
206
206
 
@@ -258,16 +258,6 @@ module Shipit
258
258
  end
259
259
  end
260
260
 
261
- test "#active_task? cache is cleared if a deploy change state" do
262
- assert_queries(1) do
263
- 10.times { @stack.active_task? }
264
- end
265
- @stack.tasks.where(status: 'running').first.error!
266
- assert_queries(1) do
267
- 10.times { @stack.active_task? }
268
- end
269
- end
270
-
271
261
  test "#deployable? returns true if stack is not locked and is not deploying" do
272
262
  @stack.deploys.destroy_all
273
263
  assert @stack.deployable?
@@ -534,6 +524,13 @@ module Shipit
534
524
  end
535
525
  end
536
526
 
527
+ test "#trigger_continuous_delivery use default env vars" do
528
+ @stack.tasks.delete_all
529
+
530
+ deploy = @stack.trigger_continuous_delivery
531
+ assert_equal({'SAFETY_DISABLED' => '0'}, deploy.env)
532
+ end
533
+
537
534
  test "#next_commit_to_deploy returns the last deployable commit" do
538
535
  @stack.tasks.where.not(until_commit_id: shipit_commits(:second).id).destroy_all
539
536
  assert_equal shipit_commits(:second), @stack.last_deployed_commit
@@ -541,7 +538,7 @@ module Shipit
541
538
  assert_equal shipit_commits(:third), @stack.next_commit_to_deploy
542
539
 
543
540
  fifth_commit = shipit_commits(:fifth)
544
- fifth_commit.statuses.create!(state: 'success', context: 'ci/travis')
541
+ fifth_commit.statuses.create!(stack_id: @stack.id, state: 'success', context: 'ci/travis')
545
542
  assert_predicate fifth_commit, :deployable?
546
543
 
547
544
  assert_equal shipit_commits(:fifth), @stack.next_commit_to_deploy
@@ -551,7 +548,7 @@ module Shipit
551
548
  @stack.tasks.destroy_all
552
549
 
553
550
  fifth_commit = shipit_commits(:fifth)
554
- fifth_commit.statuses.create!(state: 'success', context: 'ci/travis')
551
+ fifth_commit.statuses.create!(stack_id: @stack.id, state: 'success', context: 'ci/travis')
555
552
  assert_predicate fifth_commit, :deployable?
556
553
 
557
554
  assert_equal shipit_commits(:fifth), @stack.next_commit_to_deploy
@@ -9,24 +9,24 @@ module Shipit
9
9
 
10
10
  test ".replicate_from_github! is idempotent" do
11
11
  assert_difference '@commit.statuses.count', 1 do
12
- @commit.statuses.replicate_from_github!(github_status)
12
+ @commit.statuses.replicate_from_github!(@stack.id, github_status)
13
13
  end
14
14
 
15
15
  assert_no_difference '@commit.statuses.count' do
16
- @commit.statuses.replicate_from_github!(github_status)
16
+ @commit.statuses.replicate_from_github!(@stack.id, github_status)
17
17
  end
18
18
  end
19
19
 
20
20
  test "once created a commit broadcasts an update event" do
21
21
  expect_event(@stack)
22
- @commit.statuses.create!(state: 'success')
22
+ @commit.statuses.create!(stack_id: @stack.id, state: 'success')
23
23
  end
24
24
 
25
25
  test ".replicate_from_github! touches the related stack" do
26
26
  stack_last_updated_at = @stack.updated_at
27
27
  commit_last_updated_at = @commit.updated_at
28
28
 
29
- @commit.statuses.replicate_from_github!(github_status)
29
+ @commit.statuses.replicate_from_github!(@stack.id, github_status)
30
30
 
31
31
  assert_not_equal commit_last_updated_at, @commit.reload.updated_at
32
32
  assert_not_equal stack_last_updated_at, @stack.reload.updated_at
@@ -116,6 +116,16 @@ module Shipit
116
116
  assert_equal ['bundle exec cap $ENVIRONMENT deploy'], @spec.deploy_steps
117
117
  end
118
118
 
119
+ test "#deploy_steps returns `kubernetes-deploy <namespace> <context>` if `kubernetes` is present" do
120
+ @spec.stubs(:load_config).returns(
121
+ 'kubernetes' => {
122
+ 'namespace' => 'foo',
123
+ 'context' => 'bar',
124
+ },
125
+ )
126
+ assert_equal ["kubernetes-deploy foo bar"], @spec.deploy_steps
127
+ end
128
+
119
129
  test "#deploy_steps prepend and append pre and post steps" do
120
130
  @spec.stubs(:load_config).returns('deploy' => {'pre' => ['before'], 'post' => ['after']})
121
131
  @spec.expects(:bundler?).returns(true).at_least_once
@@ -148,11 +158,32 @@ module Shipit
148
158
  assert_equal ['before', 'bundle exec cap $ENVIRONMENT deploy:rollback', 'after'], @spec.rollback_steps
149
159
  end
150
160
 
151
- test '#machine_env return an environment hash' do
161
+ test "#rollback_steps returns `kubernetes-deploy <namespace> <context>` if `kubernetes` is present" do
162
+ @spec.stubs(:load_config).returns(
163
+ 'kubernetes' => {
164
+ 'namespace' => 'foo',
165
+ 'context' => 'bar',
166
+ },
167
+ )
168
+ assert_equal ["kubernetes-deploy foo bar"], @spec.rollback_steps
169
+ end
170
+
171
+ test '#machine_env returns an environment hash' do
152
172
  @spec.stubs(:load_config).returns('machine' => {'environment' => {'GLOBAL' => '1'}})
153
173
  assert_equal({'GLOBAL' => '1'}, @spec.machine_env)
154
174
  end
155
175
 
176
+ test '#discover_machine_env contains K8S_TEMPLATE_FOLDER if `kubernetes.template_dir` is present' do
177
+ @spec.stubs(:load_config).returns(
178
+ 'kubernetes' => {
179
+ 'namespace' => 'foo',
180
+ 'context' => 'bar',
181
+ 'template_dir' => '/egg/spam',
182
+ },
183
+ )
184
+ assert_equal '/egg/spam', @spec.discover_machine_env['K8S_TEMPLATE_FOLDER']
185
+ end
186
+
156
187
  test '#load_config can grab the env-specific shipit.yml file' do
157
188
  config = {}
158
189
  config.expects(:exist?).returns(true)
@@ -359,5 +390,119 @@ module Shipit
359
390
  test "#clear_working_directory? returns true by default" do
360
391
  assert_predicate @spec, :clear_working_directory?
361
392
  end
393
+
394
+ test 'bundler installs take priority over npm installs' do
395
+ @spec.expects(:discover_package_json).never
396
+ @spec.stubs(:discover_bundler).returns(['fake bundler task']).once
397
+
398
+ assert_equal ['fake bundler task'], @spec.dependencies_steps
399
+ end
400
+
401
+ test 'Gems deploys take priority over npm deploys' do
402
+ @spec.expects(:discover_npm_package).never
403
+ @spec.stubs(:discover_gem).returns(['fake gem task']).once
404
+
405
+ assert_equal ['fake gem task'], @spec.deploy_steps
406
+ end
407
+
408
+ test '#npm? is false if there is no package.json' do
409
+ @spec.expects(:package_json).returns(Shipit::Engine.root.join("tmp-#{SecureRandom.hex}"))
410
+ refute @spec.npm?
411
+ end
412
+
413
+ test '#npm? is false if npm package is private' do
414
+ file = Pathname.new('/tmp/fake_package.json')
415
+ file.write('{"private": true}')
416
+
417
+ @spec.expects(:package_json).returns(file)
418
+ refute @spec.npm?
419
+ end
420
+
421
+ test '#npm? is true if npm package is public' do
422
+ file = Pathname.new('/tmp/fake_package.json')
423
+ file.write('{"private": false}')
424
+
425
+ @spec.expects(:package_json).returns(file)
426
+
427
+ assert @spec.npm?
428
+ end
429
+
430
+ test 'npm packages have a checklist' do
431
+ @spec.stubs(:npm?).returns(true).at_least_once
432
+ assert_match(/npm version/, @spec.review_checklist[0])
433
+ end
434
+
435
+ test '#dependencies_steps returns `npm install` if a `package.json` is present' do
436
+ @spec.expects(:npm?).returns(true).at_least_once
437
+ assert_equal ['npm install --no-progress'], @spec.dependencies_steps
438
+ end
439
+
440
+ test '#publish_npm_package checks if version tag exists, and then invokes npm deploy script' do
441
+ @spec.stubs(:npm?).returns(true)
442
+ assert_equal ['assert-npm-version-tag', 'npm publish'], @spec.deploy_steps
443
+ end
444
+
445
+ test 'bundler installs take priority over yarn installs' do
446
+ @spec.expects(:discover_yarn).never
447
+ @spec.stubs(:discover_bundler).returns(['fake bundler task']).once
448
+
449
+ assert_equal ['fake bundler task'], @spec.dependencies_steps
450
+ end
451
+
452
+ test 'Gems deploys take priority over yarn deploys' do
453
+ @spec.expects(:discover_yarn_package).never
454
+ @spec.stubs(:discover_gem).returns(['fake gem task']).once
455
+
456
+ assert_equal ['fake gem task'], @spec.deploy_steps
457
+ end
458
+
459
+ test '#yarn? is false if there is no package.json' do
460
+ @spec.expects(:package_json).returns(Shipit::Engine.root.join("tmp-#{SecureRandom.hex}"))
461
+ @spec.expects(:yarn_lock).returns(Shipit::Engine.root.join('Gemfile'))
462
+
463
+ refute @spec.yarn?
464
+ end
465
+
466
+ test '#yarn? is false if there is no yarn.lock' do
467
+ @spec.expects(:yarn_lock).returns(Shipit::Engine.root.join("tmp-#{SecureRandom.hex}"))
468
+
469
+ refute @spec.yarn?
470
+ end
471
+
472
+ test '#yarn? is false if a private package.json and yarn.lock are present' do
473
+ package_json = Pathname.new('/tmp/fake_package.json')
474
+ package_json.write('{"private": true}')
475
+
476
+ @spec.expects(:package_json).returns(package_json)
477
+ @spec.expects(:yarn_lock).returns(Shipit::Engine.root.join('Gemfile'))
478
+ refute @spec.yarn?
479
+ end
480
+
481
+ test '#yarn? is true if a public package.json and yarn.lock are present' do
482
+ package_json = Pathname.new('/tmp/fake_package.json')
483
+ package_json.write('{"private": false}')
484
+
485
+ yarn_lock = Pathname.new('/tmp/fake_yarn.lock')
486
+ yarn_lock.write('')
487
+
488
+ @spec.expects(:package_json).returns(package_json)
489
+ @spec.expects(:yarn_lock).returns(yarn_lock)
490
+ assert @spec.yarn?
491
+ end
492
+
493
+ test '#dependencies_steps returns `yarn install` if a `yarn.lock` is present' do
494
+ @spec.expects(:yarn?).returns(true).at_least_once
495
+ assert_equal ['yarn install --no-progress'], @spec.dependencies_steps
496
+ end
497
+
498
+ test '#publish_yarn_package checks if version tag exists, and then invokes yarn publish script' do
499
+ @spec.stubs(:yarn?).returns(true).at_least_once
500
+ assert_equal ['assert-npm-version-tag', 'yarn publish'], @spec.deploy_steps
501
+ end
502
+
503
+ test 'yarn checklist takes precedence over npm checklist' do
504
+ @spec.stubs(:yarn?).returns(true).at_least_once
505
+ assert_match(/yarn version/, @spec.review_checklist[0])
506
+ end
362
507
  end
363
508
  end