shipit-engine 0.15.0 → 0.16.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (111) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +34 -1
  3. data/app/assets/javascripts/shipit/page_updater.js.coffee +63 -0
  4. data/app/assets/javascripts/shipit/stacks.js.coffee +9 -21
  5. data/app/assets/stylesheets/_base/_base.scss +2 -2
  6. data/app/assets/stylesheets/_base/_colors.scss +0 -1
  7. data/app/assets/stylesheets/_base/_forms.scss +14 -0
  8. data/app/assets/stylesheets/_pages/_commits.scss +16 -6
  9. data/app/assets/stylesheets/_pages/_settings.scss +8 -0
  10. data/app/assets/stylesheets/_pages/_stacks.scss +1 -1
  11. data/app/controllers/shipit/api/base_controller.rb +7 -3
  12. data/app/controllers/shipit/api/ccmenu_controller.rb +33 -0
  13. data/app/controllers/shipit/api/pull_requests_controller.rb +36 -0
  14. data/app/controllers/shipit/api/stacks_controller.rb +1 -0
  15. data/app/controllers/shipit/ccmenu_url_controller.rb +22 -0
  16. data/app/controllers/shipit/pull_requests_controller.rb +30 -0
  17. data/app/controllers/shipit/stacks_controller.rb +7 -2
  18. data/app/controllers/shipit/webhooks_controller.rb +1 -2
  19. data/app/helpers/shipit/github_url_helper.rb +8 -2
  20. data/app/helpers/shipit/shipit_helper.rb +9 -0
  21. data/app/helpers/shipit/stacks_helper.rb +22 -7
  22. data/app/jobs/shipit/background_job/unique.rb +19 -1
  23. data/app/jobs/shipit/cache_deploy_spec_job.rb +1 -1
  24. data/app/jobs/shipit/merge_pull_requests_job.rb +26 -0
  25. data/app/jobs/shipit/perform_task_job.rb +1 -1
  26. data/app/jobs/shipit/refresh_pull_request_job.rb +8 -0
  27. data/app/models/concerns/shipit/deferred_touch.rb +6 -1
  28. data/app/models/shipit/anonymous_user.rb +4 -0
  29. data/app/models/shipit/application_record.rb +5 -0
  30. data/app/models/shipit/commit.rb +51 -49
  31. data/app/models/shipit/commit_message.rb +32 -0
  32. data/app/models/shipit/deploy.rb +5 -0
  33. data/app/models/shipit/deploy_spec.rb +26 -1
  34. data/app/models/shipit/deploy_spec/file_system.rb +6 -1
  35. data/app/models/shipit/deploy_spec/kubernetes_discovery.rb +10 -13
  36. data/app/models/shipit/deploy_spec/npm_discovery.rb +2 -1
  37. data/app/models/shipit/duration.rb +3 -1
  38. data/app/models/shipit/hook.rb +1 -0
  39. data/app/models/shipit/pull_request.rb +252 -0
  40. data/app/models/shipit/stack.rb +33 -17
  41. data/app/models/shipit/status.rb +1 -16
  42. data/app/models/shipit/status/common.rb +45 -0
  43. data/app/models/shipit/status/group.rb +82 -0
  44. data/app/models/shipit/status/missing.rb +30 -0
  45. data/app/models/shipit/status/unknown.rb +33 -0
  46. data/app/models/shipit/unlimited_api_client.rb +10 -0
  47. data/app/serializers/shipit/commit_serializer.rb +1 -1
  48. data/app/serializers/shipit/pull_request_serializer.rb +20 -0
  49. data/app/serializers/shipit/stack_serializer.rb +6 -2
  50. data/app/views/layouts/shipit.html.erb +41 -39
  51. data/app/views/shipit/ccmenu/project.xml.builder +13 -0
  52. data/app/views/shipit/commits/_commit.html.erb +1 -1
  53. data/app/views/shipit/deploys/_deploy.html.erb +1 -1
  54. data/app/views/shipit/pull_requests/_pull_request.html.erb +29 -0
  55. data/app/views/shipit/pull_requests/index.html.erb +20 -0
  56. data/app/views/shipit/shared/_author.html.erb +7 -0
  57. data/app/views/shipit/stacks/_header.html.erb +5 -0
  58. data/app/views/shipit/stacks/settings.html.erb +13 -0
  59. data/app/views/shipit/stacks/show.html.erb +3 -2
  60. data/app/views/shipit/statuses/_group.html.erb +1 -1
  61. data/app/views/shipit/tasks/_task.html.erb +1 -1
  62. data/config/initializers/inflections.rb +3 -0
  63. data/config/locales/en.yml +1 -3
  64. data/config/routes.rb +8 -0
  65. data/db/migrate/20170130113633_create_shipit_pull_requests.rb +25 -0
  66. data/db/migrate/20170208143657_add_pull_request_number_and_title_to_commits.rb +7 -0
  67. data/db/migrate/20170208154609_backfill_merge_commits.rb +13 -0
  68. data/db/migrate/20170209160355_add_branch_to_pull_requests.rb +5 -0
  69. data/db/migrate/20170215123538_add_merge_queue_enabled_to_stacks.rb +5 -0
  70. data/db/migrate/20170220152410_improve_users_indexing.rb +6 -0
  71. data/db/migrate/20170221102128_improve_tasks_indexing.rb +8 -0
  72. data/db/migrate/20170221130336_add_last_revalidated_at_on_pull_requests.rb +10 -0
  73. data/lib/shipit.rb +2 -0
  74. data/lib/shipit/version.rb +1 -1
  75. data/lib/tasks/cron.rake +1 -0
  76. data/test/controllers/api/ccmenu_controller_test.rb +57 -0
  77. data/test/controllers/api/commits_controller_test.rb +1 -1
  78. data/test/controllers/api/pull_requests_controller_test.rb +59 -0
  79. data/test/controllers/ccmenu_controller_test.rb +33 -0
  80. data/test/controllers/pull_requests_controller_test.rb +31 -0
  81. data/test/controllers/webhooks_controller_test.rb +3 -4
  82. data/test/dummy/config/environments/development.rb +3 -1
  83. data/test/dummy/data/stacks/shopify/junk/production/git/README.md +8 -0
  84. data/test/dummy/data/stacks/shopify/junk/production/git/circle.yml +4 -0
  85. data/test/dummy/data/stacks/shopify/junk/production/git/shipit.yml +4 -0
  86. data/test/dummy/db/development.sqlite3 +0 -0
  87. data/test/dummy/db/schema.rb +45 -11
  88. data/test/dummy/db/seeds.rb +33 -10
  89. data/test/dummy/db/test.sqlite3 +0 -0
  90. data/test/fixtures/shipit/commits.yml +14 -0
  91. data/test/fixtures/shipit/pull_requests.yml +56 -0
  92. data/test/fixtures/shipit/stacks.yml +5 -1
  93. data/test/fixtures/shipit/statuses.yml +8 -0
  94. data/test/helpers/json_helper.rb +16 -14
  95. data/test/jobs/merge_pull_requests_job_test.rb +59 -0
  96. data/test/models/commits_test.rb +104 -49
  97. data/test/{unit → models}/deploy_spec_test.rb +138 -12
  98. data/test/models/deploys_test.rb +10 -4
  99. data/test/models/pull_request_test.rb +197 -0
  100. data/test/models/stacks_test.rb +46 -53
  101. data/test/models/status/group_test.rb +44 -0
  102. data/test/models/status/missing_test.rb +23 -0
  103. data/test/models/status_test.rb +3 -6
  104. data/test/unit/csv_serializer_test.rb +10 -2
  105. metadata +57 -12
  106. data/app/models/shipit/missing_status.rb +0 -21
  107. data/app/models/shipit/status_group.rb +0 -35
  108. data/app/models/shipit/unknown_status.rb +0 -48
  109. data/app/views/shipit/commits/_commit_author.html.erb +0 -7
  110. data/test/models/missing_status_test.rb +0 -23
  111. data/test/models/status_group_test.rb +0 -26
@@ -2,11 +2,11 @@ module Shipit
2
2
  class DeploySpec
3
3
  class FileSystem < DeploySpec
4
4
  include NpmDiscovery
5
- include KubernetesDiscovery
6
5
  include PypiDiscovery
7
6
  include RubygemsDiscovery
8
7
  include CapistranoDiscovery
9
8
  include BundlerDiscovery
9
+ include KubernetesDiscovery
10
10
 
11
11
  def initialize(app_dir, env)
12
12
  @app_dir = Pathname(app_dir)
@@ -29,6 +29,11 @@ module Shipit
29
29
 
30
30
  def cacheable_config
31
31
  (config || {}).deep_merge(
32
+ 'merge' => {
33
+ 'require' => pull_request_required_statuses,
34
+ 'ignore' => pull_request_ignored_statuses,
35
+ 'revalidate_after' => revalidate_pull_requests_after.try!(:to_i),
36
+ },
32
37
  'ci' => {
33
38
  'hide' => hidden_statuses,
34
39
  'allow_failures' => soft_failing_statuses,
@@ -9,24 +9,21 @@ module Shipit
9
9
  discover_kubernetes || super
10
10
  end
11
11
 
12
- def discover_machine_env
13
- env = super
14
- env = env.merge('K8S_TEMPLATE_FOLDER' => kube_config['template_dir']) if kube_config['template_dir']
15
- env
16
- end
17
-
18
12
  private
19
13
 
20
14
  def discover_kubernetes
21
15
  return unless kube_config.present?
22
16
 
23
- [
24
- Shellwords.join([
25
- "kubernetes-deploy",
26
- kube_config['namespace'],
27
- kube_config['context'],
28
- ]),
29
- ]
17
+ cmd = ["kubernetes-deploy"]
18
+ if kube_config['template_dir']
19
+ cmd << '--template-dir'
20
+ cmd << kube_config['template_dir']
21
+ end
22
+
23
+ cmd << kube_config['namespace']
24
+ cmd << kube_config['context']
25
+
26
+ [Shellwords.join(cmd)]
30
27
  end
31
28
 
32
29
  def kube_config
@@ -62,7 +62,8 @@ module Shipit
62
62
 
63
63
  def publish_npm_package
64
64
  check_tags = 'assert-npm-version-tag'
65
- publish = js_command('publish')
65
+ # `yarn publish` requires user input, so always use npm.
66
+ publish = 'npm publish'
66
67
 
67
68
  [check_tags, publish]
68
69
  end
@@ -1,5 +1,7 @@
1
1
  module Shipit
2
2
  class Duration < ActiveSupport::Duration
3
+ ParseError = Class.new(ArgumentError)
4
+
3
5
  FORMAT = /
4
6
  \A
5
7
  (?<days>\d+d)?
@@ -18,7 +20,7 @@ module Shipit
18
20
  class << self
19
21
  def parse(value)
20
22
  unless match = FORMAT.match(value.to_s)
21
- raise ArgumentError, "not a duration: #{value.inspect}"
23
+ raise ParseError, "not a duration: #{value.inspect}"
22
24
  end
23
25
  parts = []
24
26
  UNITS.values.each do |unit|
@@ -16,6 +16,7 @@ module Shipit
16
16
  commit_status
17
17
  deployable_status
18
18
  merge_status
19
+ merge
19
20
  ).freeze
20
21
 
21
22
  belongs_to :stack, required: false
@@ -0,0 +1,252 @@
1
+ module Shipit
2
+ class PullRequest < ApplicationRecord
3
+ include DeferredTouch
4
+
5
+ WAITING_STATUSES = %w(fetching pending).freeze
6
+ QUEUED_STATUSES = %w(pending revalidating).freeze
7
+ REJECTION_REASONS = %w(ci_failing merge_conflict).freeze
8
+ InvalidTransition = Class.new(StandardError)
9
+ NotReady = Class.new(StandardError)
10
+
11
+ class StatusChecker < Status::Group
12
+ def initialize(commit, statuses, deploy_spec)
13
+ @deploy_spec = deploy_spec
14
+ super(commit, statuses)
15
+ end
16
+
17
+ private
18
+
19
+ attr_reader :deploy_spec
20
+
21
+ def reject_hidden(statuses)
22
+ statuses.reject { |s| ignored_statuses.include?(s.context) }
23
+ end
24
+
25
+ def reject_allowed_to_fail(statuses)
26
+ statuses.reject { |s| ignored_statuses.include?(s.context) }
27
+ end
28
+
29
+ def ignored_statuses
30
+ deploy_spec.try!(:pull_request_ignored_statuses) || []
31
+ end
32
+
33
+ def required_statuses
34
+ deploy_spec.try!(:pull_request_required_statuses) || []
35
+ end
36
+ end
37
+
38
+ belongs_to :stack
39
+ belongs_to :head, class_name: 'Shipit::Commit'
40
+ belongs_to :merge_requested_by, class_name: 'Shipit::User'
41
+ has_one :merge_commit, class_name: 'Shipit::Commit'
42
+
43
+ deferred_touch stack: :updated_at
44
+
45
+ validates :number, presence: true, uniqueness: {scope: :stack_id}
46
+
47
+ scope :waiting, -> { where(merge_status: WAITING_STATUSES) }
48
+ scope :pending, -> { where(merge_status: 'pending') }
49
+ scope :to_be_merged, -> { pending.order(merge_requested_at: :asc) }
50
+ scope :queued, -> { where(merge_status: QUEUED_STATUSES).order(merge_requested_at: :asc) }
51
+
52
+ after_save :record_merge_status_change
53
+ after_commit :emit_hooks
54
+
55
+ state_machine :merge_status, initial: :fetching do
56
+ state :fetching
57
+ state :pending
58
+ state :rejected
59
+ state :canceled
60
+ state :merged
61
+ state :revalidating
62
+
63
+ event :fetched do
64
+ transition fetching: :pending
65
+ end
66
+
67
+ event :reject do
68
+ transition pending: :rejected
69
+ end
70
+
71
+ event :revalidate do
72
+ transition pending: :revalidating
73
+ end
74
+
75
+ event :cancel do
76
+ transition any => :canceled
77
+ end
78
+
79
+ event :complete do
80
+ transition pending: :merged
81
+ end
82
+
83
+ event :retry do
84
+ transition %i(rejected canceled revalidating) => :pending
85
+ end
86
+
87
+ before_transition rejected: any do |pr|
88
+ pr.rejection_reason = nil
89
+ end
90
+
91
+ before_transition %i(fetching rejected canceled) => :pending do |pr|
92
+ pr.merge_requested_at = Time.now.utc
93
+ end
94
+
95
+ before_transition any => :pending do |pr|
96
+ pr.revalidated_at = Time.now.utc
97
+ end
98
+ end
99
+
100
+ def self.schedule_merges
101
+ Shipit::Stack.where(id: pending.uniq.pluck(:stack_id)).find_each(&:schedule_merges)
102
+ end
103
+
104
+ def self.extract_number(stack, number_or_url)
105
+ case number_or_url
106
+ when /\A#?(\d+)\z/
107
+ $1.to_i
108
+ when %r{\Ahttps://#{Regexp.escape(Shipit.github_domain)}/([^/]+)/([^/]+)/pull/(\d+)}
109
+ return unless $1.downcase == stack.repo_owner.downcase
110
+ return unless $2.downcase == stack.repo_name.downcase
111
+ return $3.to_i
112
+ end
113
+ end
114
+
115
+ def self.request_merge!(stack, number, user)
116
+ now = Time.now.utc
117
+ pull_request = begin
118
+ create_with(
119
+ merge_requested_at: now,
120
+ merge_requested_by: user.presence,
121
+ ).find_or_create_by!(
122
+ stack: stack,
123
+ number: number,
124
+ )
125
+ rescue ActiveRecord::RecordNotUnique
126
+ retry
127
+ end
128
+ pull_request.update!(merge_requested_by: user.presence)
129
+ pull_request.retry! if pull_request.rejected? || pull_request.canceled? || pull_request.revalidating?
130
+ pull_request.schedule_refresh!
131
+ pull_request
132
+ end
133
+
134
+ def reject!(reason)
135
+ unless REJECTION_REASONS.include?(reason)
136
+ raise ArgumentError, "invalid reason: #{reason.inspect}, must be one of: #{REJECTION_REASONS.inspect}"
137
+ end
138
+ self.rejection_reason = reason.presence
139
+ super()
140
+ true
141
+ end
142
+
143
+ def reject_unless_mergeable!
144
+ return reject!('merge_conflict') if merge_conflict?
145
+ return reject!('ci_failing') unless all_status_checks_passed?
146
+ false
147
+ end
148
+
149
+ def merge!
150
+ raise InvalidTransition unless pending?
151
+
152
+ raise NotReady if not_mergeable_yet?
153
+ if need_revalidation?
154
+ revalidate!
155
+ return false
156
+ end
157
+
158
+ Shipit.github_api.merge_pull_request(
159
+ stack.github_repo_name,
160
+ number,
161
+ merge_message,
162
+ sha: head.sha,
163
+ commit_message: 'Merged by Shipit',
164
+ merge_method: 'merge',
165
+ )
166
+ begin
167
+ Shipit.github_api.delete_branch(stack.github_repo_name, branch)
168
+ rescue Octokit::UnprocessableEntity
169
+ # branch was already deleted somehow
170
+ end
171
+ complete!
172
+ return true
173
+ rescue Octokit::MethodNotAllowed # merge conflict
174
+ reject!('merge_conflict')
175
+ return false
176
+ rescue Octokit::Conflict # shas didn't match, PR was updated.
177
+ raise NotReady
178
+ end
179
+
180
+ def all_status_checks_passed?
181
+ StatusChecker.new(head, head.statuses, stack.cached_deploy_spec).success?
182
+ end
183
+
184
+ def waiting?
185
+ WAITING_STATUSES.include?(merge_status)
186
+ end
187
+
188
+ def need_revalidation?
189
+ timeout = stack.cached_deploy_spec.try!(:revalidate_pull_requests_after)
190
+ return false unless timeout
191
+ (revalidated_at + timeout).past?
192
+ end
193
+
194
+ def merge_conflict?
195
+ mergeable == false
196
+ end
197
+
198
+ def not_mergeable_yet?
199
+ mergeable.nil?
200
+ end
201
+
202
+ def schedule_refresh!
203
+ RefreshPullRequestJob.perform_later(self)
204
+ end
205
+
206
+ def refresh!
207
+ update!(github_pull_request: Shipit.github_api.pull_request(stack.github_repo_name, number))
208
+ head.refresh_statuses!
209
+ fetched! if fetching?
210
+ end
211
+
212
+ def github_pull_request=(github_pull_request)
213
+ self.github_id = github_pull_request.id
214
+ self.api_url = github_pull_request.url
215
+ self.title = github_pull_request.title
216
+ self.state = github_pull_request.state
217
+ self.mergeable = github_pull_request.mergeable
218
+ self.additions = github_pull_request.additions
219
+ self.deletions = github_pull_request.deletions
220
+ self.branch = github_pull_request.head.ref
221
+ self.head = find_or_create_commit_from_github_by_sha!(github_pull_request.head.sha, detached: true)
222
+ end
223
+
224
+ def merge_message
225
+ return title unless merge_requested_by
226
+ "#{title}\n\nMerge-Requested-By: #{merge_requested_by.login}\n"
227
+ end
228
+
229
+ private
230
+
231
+ def record_merge_status_change
232
+ @merge_status_changed ||= merge_status_changed?
233
+ end
234
+
235
+ def emit_hooks
236
+ return unless @merge_status_changed
237
+ @merge_status_changed = nil
238
+ Hook.emit('merge', stack, pull_request: self, status: merge_status, stack: stack)
239
+ end
240
+
241
+ def find_or_create_commit_from_github_by_sha!(sha, attributes)
242
+ if commit = stack.commits.by_sha(sha)
243
+ return commit
244
+ else
245
+ github_commit = Shipit.github_api.commit(stack.github_repo_name, sha)
246
+ stack.commits.create_from_github!(github_commit, attributes)
247
+ end
248
+ rescue ActiveRecord::RecordNotUnique
249
+ retry
250
+ end
251
+ end
252
+ end
@@ -24,6 +24,7 @@ module Shipit
24
24
  REQUIRED_HOOKS = %i(push status).freeze
25
25
 
26
26
  has_many :commits, dependent: :destroy
27
+ has_many :pull_requests, dependent: :destroy
27
28
  has_many :tasks, dependent: :destroy
28
29
  has_many :deploys
29
30
  has_many :rollbacks
@@ -50,6 +51,7 @@ module Shipit
50
51
  after_commit :broadcast_update, on: :update
51
52
  after_commit :emit_merge_status_hooks, on: :update
52
53
  after_commit :setup_hooks, :sync_github, on: :create
54
+ after_commit :schedule_merges_if_necessary, on: :update
53
55
 
54
56
  validates :repo_name, uniqueness: {scope: %i(repo_owner environment),
55
57
  message: 'cannot be used more than once with this environment'}
@@ -138,6 +140,10 @@ module Shipit
138
140
  trigger_deploy(commit, Shipit.user, env: cached_deploy_spec.default_deploy_env)
139
141
  end
140
142
 
143
+ def schedule_merges
144
+ MergePullRequestsJob.perform_later(self)
145
+ end
146
+
141
147
  def next_commit_to_deploy
142
148
  commits_to_deploy = commits.order(id: :asc).newer_than(last_deployed_commit).reachable.preload(:statuses)
143
149
  commits_to_deploy = commits_to_deploy.limit(maximum_commits_per_deploy) if maximum_commits_per_deploy
@@ -179,15 +185,21 @@ module Shipit
179
185
  end
180
186
 
181
187
  def merge_status
182
- if locked?
183
- 'locked'
188
+ return 'locked' if locked?
189
+ return 'failure' if %w(failure error).freeze.include?(branch_status)
190
+ if maximum_commits_per_deploy && (undeployed_commits_count > maximum_commits_per_deploy * 1.5)
191
+ 'backlogged'
184
192
  else
185
- significant_statuses = undeployed_commits.map(&:significant_status)
186
- significant_statuses << last_deployed_commit.significant_status unless last_deployed_commit.blank?
193
+ 'success'
194
+ end
195
+ end
187
196
 
188
- last_finalized_status = significant_statuses.reject { |s| %w(pending unknown).include?(s.state) }.first
189
- last_finalized_status.try!(:simple_state) || 'pending'
197
+ def branch_status
198
+ undeployed_commits.each do |commit|
199
+ state = commit.status.simple_state
200
+ return state unless %w(pending unknown missing).freeze.include?(state)
190
201
  end
202
+ 'pending'
191
203
  end
192
204
 
193
205
  def status
@@ -217,18 +229,14 @@ module Shipit
217
229
  end
218
230
  end
219
231
 
220
- def filter_visible_statuses(statuses)
221
- statuses.reject { |s| hidden_statuses.include?(s.context) }
222
- end
223
-
224
- def filter_meaningful_statuses(statuses)
225
- filter_visible_statuses(statuses).reject { |s| soft_failing_statuses.include?(s.context) }
226
- end
227
-
228
232
  def deployable?
229
233
  !locked? && !active_task?
230
234
  end
231
235
 
236
+ def allows_merges?
237
+ merge_queue_enabled? && !locked? && merge_status == 'success'
238
+ end
239
+
232
240
  def repo_name=(name)
233
241
  super(name.try!(:downcase))
234
242
  end
@@ -369,9 +377,11 @@ module Shipit
369
377
  end
370
378
 
371
379
  def broadcast_update
372
- payload = {url: Shipit::Engine.routes.url_helpers.stack_path(self)}.to_json
373
- event = Pubsubstub::Event.new(payload, name: "stack.update")
374
- Pubsubstub::RedisPubSub.publish("stack.#{id}", event)
380
+ Pubsubstub.publish(
381
+ "stack.#{id}",
382
+ {id: id, updated_at: updated_at}.to_json,
383
+ name: 'update',
384
+ )
375
385
  end
376
386
 
377
387
  def setup_hooks
@@ -443,6 +453,12 @@ module Shipit
443
453
  self.branch = 'master' if branch.blank?
444
454
  end
445
455
 
456
+ def schedule_merges_if_necessary
457
+ if previous_changes.include?('lock_reason') && previous_changes['lock_reason'].last.blank?
458
+ schedule_merges
459
+ end
460
+ end
461
+
446
462
  def emit_lock_hooks
447
463
  return unless previous_changes.include?('lock_reason')
448
464
  Hook.emit(:lock, self, locked: locked?, stack: self)