shipit-engine 0.10.0 → 0.11.0

data/app/models/shipit/team.rb

@@ -1,6 +1,6 @@
 module Shipit
   class Team < ActiveRecord::Base
-    REQUIRED_HOOKS = %i(membership)
+    REQUIRED_HOOKS = %i(membership).freeze
 
     has_many :members, class_name: :User, through: :memberships, source: :user
     has_many :memberships

data/app/models/shipit/undeployed_commit.rb

@@ -0,0 +1,36 @@
+module Shipit
+  class UndeployedCommit < DelegateClass(Commit)
+    attr_reader :index
+
+    def initialize(commit, index)
+      super(commit)
+      @index = index
+    end
+
+    def deploy_state(bypass_safeties = false)
+      state = deployable? ? 'allowed' : status.state
+      unless bypass_safeties
+        state = 'deploying' if stack.active_task?
+        state = 'locked' if stack.locked?
+      end
+      state
+    end
+
+    def redeploy_state(bypass_safeties = false)
+      state = 'allowed'
+      unless bypass_safeties
+        state = 'locked' if stack.locked?
+        state = 'deploying' if stack.active_task?
+      end
+      state
+    end
+
+    def deploy_disallowed?
+      !deployable? || !stack.deployable?
+    end
+
+    def deploy_discouraged?
+      stack.maximum_commits_per_deploy && index >= stack.maximum_commits_per_deploy
+    end
+  end
+end

data/app/views/layouts/shipit.html.erb

@@ -13,14 +13,16 @@
     <%= content_tag :div, msg, class: ["flash-#{name}", :flash] %>
   <% end %>
 
-  <div class="banner notification banner--blue hidden">
+  <div class="banner enable-notifications banner--blue hidden">
     <div class="banner__inner wrapper">
       <div class="banner__content">
         <h2 class="banner__title">Do you want to enable desktop notifications?</h2>
+        <div class="banner__accessory">
+          <button class="banner__btn btn">Enable notifications</button>
+        </div>
       </div>
-      <div class="banner__accessory">
-        <button class="banner__btn btn">Enable notifications</button>
-      </div>
+
+      <a class="banner__dismiss">&times;</a>
     </div>
   </div>
 

data/app/views/shipit/deploys/show.html.erb

@@ -4,7 +4,7 @@
   <% else %>
     deploying
   <% end %>
-  <span class="short-sha"><%= render_commit_id_link(@deploy.until_commit) %></span>
+  <span class="short-sha"><%= link_to_github_deploy(@deploy) %></span>
   <%= timeago_tag(@deploy.created_at, force: true) %>
 <% end %>
 

data/app/views/shipit/stacks/_header.html.erb

@@ -24,6 +24,13 @@
     </li>
   </ul>
   <ul class="nav__list nav__list--secondary">
+    <% if stack.links.present? %>
+      <% stack.links.each do |name, url| %>
+        <li class="nav__list__item">
+          <%= link_to name.humanize, url, :target => '_blank', :class => "#{name.dasherize}-url" %>
+        </li>
+      <% end %>
+    <% end %>
     <li class="nav__list__item">
       <%= link_to 'View on GitHub', github_repo_url(stack.repo_owner, stack.repo_name) %>
     </li>

data/app/views/shipit/stacks/show.html.erb

@@ -64,7 +64,7 @@
       </div>
     </header>
     <ul class="commit-list">
-      <%= render partial: 'shipit/commits/commit', collection: @commits.to_a %>
+      <%= render partial: 'shipit/commits/commit', collection: @commits %>
     </ul>
   </section>
 

data/config/locales/en.yml

@@ -21,16 +21,22 @@
 
 en:
   deploy_button:
+    hint:
+      max_commits: It is recommended not to deploy more than %{maximum} commits at once.
     caption:
       pending: CI Pending...
       failure: CI Failure
       error: CI Error
       unknown: Not Run
       locked: Locked
-      deploying: Deploy in progress...
-      enabled: Deploy
+      deploying: A Deploy is in Progress...
+      allowed: Deploy
       missing: Missing CI
-
+  redeploy_button:
+    caption:
+      deploying: A Deploy is in Progress...
+      allowed: Redeploy
+      locked: Locked
   deploy_spec:
     hint:
       deploy: Impossible to detect how to deploy this application. Please define `deploy.override` in your shipit.yml

data/config/secrets.development.example.yml

@@ -0,0 +1,19 @@
+host: 'http://localhost:3000'
+redis_url: 'redis://127.0.0.1:6379/0'
+
+github_api:
+  # Can be obtained there: https://github.com/settings/tokens/new
+  # The required permissions are: `admin:org_hook`, `admin:repo_hook`, `read:org` and `repo`
+  access_token:
+
+# Can be obtained there: https://github.com/settings/applications/new
+# Set the "Authorization callback URL" as `<host>/github/auth/github/callback`
+github_oauth: 
+  id:
+  secret:
+  # teams: # Optional
+
+# To work on the beta bootstrap re-write
+# features:
+#   - bootstrap
+

data/config/secrets.development.shopify.yml

@@ -0,0 +1,19 @@
+host: 'http://shipit-engine.localhost'
+redis_url: 'redis://shipit-engine.railgun:6379'
+
+github_api:
+  # Can be obtained there: https://github.com/settings/tokens/new
+  # The required permissions are: `admin:org_hook`, `admin:repo_hook`, `read:org` and `repo`
+  access_token:
+
+# Can be obtained there: https://github.com/settings/applications/new
+# Set the "Authorization callback URL" as `<host>/github/auth/github/callback`
+github_oauth: 
+  id:
+  secret:
+  # teams: # Optional
+
+# To work on the beta bootstrap re-write
+# features:
+#   - bootstrap
+

data/config/secrets.development.yml

@@ -0,0 +1,16 @@
+host: 'http://shipit-engine.localhost'
+redis_url: 'redis://shipit-engine.railgun:6379'
+
+github_api:
+  access_token: caa4f8b2bb3d94269291faf9b54b85457db0173b
+
+# If you want to test GitHub Authentication
+github_oauth:
+  id: d2c8c96f8f226933186e
+  secret: e4ca5135bb7052318e8cc4e48f296ee58ad39e61
+  # teams:
+
+# To work on the beta bootstrap re-write
+# features:
+#   - bootstrap
+

data/db/migrate/20160502150713_add_estimated_deploy_duration_to_stacks.rb

@@ -0,0 +1,5 @@
+class AddEstimatedDeployDurationToStacks < ActiveRecord::Migration
+  def change
+    add_column :stacks, :estimated_deploy_duration, :integer, null: false, default: 1
+  end
+end

data/db/migrate/20160526192650_reorder_active_tasks_index.rb

@@ -0,0 +1,7 @@
+class ReorderActiveTasksIndex < ActiveRecord::Migration
+  def change
+    remove_index :tasks, name: :index_active_tasks
+    add_index :tasks, %i(stack_id allow_concurrency status), name: :index_active_tasks
+    remove_index :tasks, :stack_id # now useless since `index_active_tasks` starts by `stack_id`
+  end
+end

data/lib/shipit/command.rb

@@ -15,7 +15,7 @@ module Shipit
 
     attr_reader :out, :code, :chdir, :env, :args, :pid, :timeout
 
-    def initialize(*args, default_timeout: 5.minutes.to_i, env: {}, chdir:)
+    def initialize(*args, default_timeout: Shipit.default_inactivity_timeout, env: {}, chdir:)
       @args, options = parse_arguments(args)
       @timeout = options['timeout'.freeze] || options[:timeout] || default_timeout
       @env = env
@@ -170,12 +170,25 @@ module Shipit
     end
 
     def kill_and_wait(sig, wait, &block)
+      retry_count = 5
       kill(sig, &block)
       begin
         with_timeout(wait) do
           read_stream(@out, &block)
         end
-      rescue TimedOut, Errno::EIO # EIO is somewhat expected on Linux: http://stackoverflow.com/a/10306782
+      rescue TimedOut
+      rescue Errno::EIO # EIO is somewhat expected on Linux: http://stackoverflow.com/a/10306782
+        # If we try to read the stream right after sending a signal, we often get an Errno::EIO.
+        if status = Process.wait(@pid, Process::WNOHANG)
+          return status
+        else
+          # If we let the child a little bit of time, it solves it.
+          retry_count -= 1
+          if retry_count > 0
+            sleep 0.05
+            retry
+          end
+        end
       end
       Process.wait(@pid, Process::WNOHANG)
     end

data/lib/shipit/engine.rb

@@ -9,6 +9,7 @@ module Shipit
     initializer 'shipit.config' do |app|
       Rails.application.routes.default_url_options[:host] = Shipit.host
       Shipit::Engine.routes.default_url_options[:host] = Shipit.host
+      Pubsubstub.redis_url = Shipit.redis_url.to_s
 
       app.config.assets.paths << Emoji.images_path
       app.config.assets.precompile += %w(
@@ -20,7 +21,7 @@ module Shipit
         shipit_bs.css
       )
       app.config.assets.precompile << proc do |path|
-        path =~ /\Aplugins\/[\-\w]+\.(js|css)\Z/
+        path =~ %r{\Aplugins/[\-\w]+\.(js|css)\Z}
       end
       app.config.assets.precompile << proc do |path|
         path.start_with?('emoji/') && path.end_with?('.png')

data/lib/shipit/stat.rb

@@ -0,0 +1,13 @@
+module Shipit
+  module Stat
+    extend self
+
+    def p90(numbers)
+      percentile(90, numbers)
+    end
+
+    def percentile(percentile, numbers)
+      numbers.sort[((numbers.size - 1) * percentile / 100.0).floor]
+    end
+  end
+end

data/lib/shipit/version.rb

@@ -1,3 +1,3 @@
 module Shipit
-  VERSION = '0.10.0'
+  VERSION = '0.11.0'.freeze
 end

data/lib/shipit.rb

@@ -45,6 +45,7 @@ require 'shipit/task_commands'
 require 'shipit/deploy_commands'
 require 'shipit/rollback_commands'
 require 'shipit/environment_variables'
+require 'shipit/stat'
 
 SafeYAML::OPTIONS[:default_mode] = :safe
 SafeYAML::OPTIONS[:deserialize_symbols] = false
@@ -85,6 +86,14 @@ module Shipit
     github_url('/api/v3/') if github_enterprise?
   end
 
+  def user
+    if github_api.login
+      User.find_or_create_by_login!(github_api.login)
+    else
+      AnonymousUser.new
+    end
+  end
+
   def github_api
     @github_api ||= begin
       client = Octokit::Client.new(github_api_credentials)
@@ -184,6 +193,10 @@ module Shipit
     @bootstrap_view_path ||= Engine.root.join('app/views/bootstrap')
   end
 
+  def default_inactivity_timeout
+    secrets.commands_inactivity_timeout || 5.minutes.to_i
+  end
+
   protected
 
   def revision_file

data/lib/tasks/cron.rake

@@ -2,6 +2,7 @@ namespace :cron do
   desc "Updates deployed revisions"
   task minutely: :environment do
     Shipit::Stack.refresh_deployed_revisions
+    Shipit::Stack.schedule_continuous_delivery
   end
 
   desc "Rolls-up output chunks for completed deploys older than an hour"

data/test/controllers/github_authentication_controller_test.rb

@@ -10,11 +10,11 @@ module Shipit
         extra: OmniAuth::AuthHash.new(
           raw_info: OmniAuth::AuthHash.new(
             id: 44,
-            name: 'Shipit',
-            email: 'shipit@example.com',
-            login: 'shipit',
+            name: 'Shipit User',
+            email: 'shipit-user@example.com',
+            login: 'shipit-user',
             avatar_url: 'https://example.com',
-            api_url: 'https://github.com/api/v3/users/shipit',
+            api_url: 'https://github.com/api/v3/users/shipit-user',
           ),
         ),
       )
@@ -24,7 +24,7 @@ module Shipit
         get :callback
       end
 
-      user = User.find_by_login('shipit')
+      user = User.find_by_login('shipit-user')
       assert_equal 's3cr3t', user.github_access_token
       assert_equal 44, user.github_id
     end

data/test/dummy/config/initializers/0_load_development_secrets.rb

@@ -0,0 +1,9 @@
+local_secrets = Shipit::Engine.root.join('config/secrets.development.yml')
+if local_secrets.exist?
+  secrets = YAML.load(local_secrets.read)
+  if Rails.env.development?
+    Rails.application.secrets.deep_merge!(secrets)
+  elsif Rails.env.test?
+    Rails.application.secrets.merge!(redis_url: secrets['redis_url'])
+  end
+end

data/test/dummy/config/secrets.yml

@@ -1,20 +1,5 @@
 development:
-  features:
-    - bootstrap
   secret_key_base: s3cr3ts3cr3ts3cr3ts3cr3ts3cr3ts3cr3t
-  # github_domain: github.shopify.com
-  github_oauth:
-    id: 29f6c9084a8837f337ff
-    secret: 3f5d2c3bb35d4ff905189401c3a2c8f249800554
-    # team:
-  github_api:
-    access_token: d5a01c65f69d86ecc55dccdf347b19d6f9a16b3b
-    # login:
-    # password:
-    # api_endpoint:
-  host: 'http://localhost:3000'
-  redis_url: "redis://127.0.0.1:6379/7"
-
 test:
   secret_key_base: s3cr3ts3cr3ts3cr3ts3cr3ts3cr3ts3cr3t
   host: shipit.com
@@ -26,5 +11,9 @@ test:
   github_oauth:
     id: 1d
     secret: s3cr37
-    # team:
+    # teams:
+    #   -
+    #   -
   redis_url: "redis://127.0.0.1:6379/7"
+  # features:
+  #   - bootstrap

data/test/dummy/db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20160426155146) do
+ActiveRecord::Schema.define(version: 20160526192650) do
 
   create_table "api_clients", force: :cascade do |t|
     t.text     "permissions", limit: 65535
@@ -130,21 +130,22 @@ ActiveRecord::Schema.define(version: 20160426155146) do
   add_index "output_chunks", ["task_id"], name: "index_output_chunks_on_task_id"
 
   create_table "stacks", force: :cascade do |t|
-    t.string   "repo_name",                limit: 100,                          null: false
-    t.string   "repo_owner",               limit: 39,                           null: false
-    t.string   "environment",              limit: 50,    default: "production", null: false
+    t.string   "repo_name",                 limit: 100,                          null: false
+    t.string   "repo_owner",                limit: 39,                           null: false
+    t.string   "environment",               limit: 50,    default: "production", null: false
     t.datetime "created_at"
     t.datetime "updated_at"
-    t.string   "branch",                   limit: 255,   default: "master",     null: false
-    t.string   "deploy_url",               limit: 255
-    t.string   "lock_reason",              limit: 4096
-    t.integer  "tasks_count",              limit: 4,     default: 0,            null: false
-    t.boolean  "continuous_deployment",                  default: false,        null: false
-    t.integer  "undeployed_commits_count", limit: 4,     default: 0,            null: false
-    t.text     "cached_deploy_spec",       limit: 65535
-    t.integer  "lock_author_id",           limit: 4
+    t.string   "branch",                    limit: 255,   default: "master",     null: false
+    t.string   "deploy_url",                limit: 255
+    t.string   "lock_reason",               limit: 4096
+    t.integer  "tasks_count",               limit: 4,     default: 0,            null: false
+    t.boolean  "continuous_deployment",                   default: false,        null: false
+    t.integer  "undeployed_commits_count",  limit: 4,     default: 0,            null: false
+    t.text     "cached_deploy_spec",        limit: 65535
+    t.integer  "lock_author_id",            limit: 4
     t.boolean  "ignore_ci"
     t.datetime "inaccessible_since"
+    t.integer  "estimated_deploy_duration",               default: 1,            null: false
   end
 
   add_index "stacks", ["repo_owner", "repo_name", "environment"], name: "stack_unicity", unique: true
@@ -186,8 +187,7 @@ ActiveRecord::Schema.define(version: 20160426155146) do
 
   add_index "tasks", ["rolled_up", "created_at", "status"], name: "index_tasks_on_rolled_up_and_created_at_and_status"
   add_index "tasks", ["since_commit_id"], name: "index_tasks_on_since_commit_id"
-  add_index "tasks", ["stack_id"], name: "index_tasks_on_stack_id"
-  add_index "tasks", ["status", "stack_id", "allow_concurrency"], name: "index_active_tasks"
+  add_index "tasks", ["stack_id", "allow_concurrency", "status"], name: "index_active_tasks"
   add_index "tasks", ["type", "stack_id", "parent_id"], name: "index_tasks_by_stack_and_parent"
   add_index "tasks", ["type", "stack_id", "status"], name: "index_tasks_by_stack_and_status"
   add_index "tasks", ["until_commit_id"], name: "index_tasks_on_until_commit_id"

data/test/dummy/db/seeds.rb

@@ -72,6 +72,7 @@ module Shipit
             ]
           },
           "deploy": {
+            "max_commits": 3,
             "override": [
               "bundle exec cap $ENVIRONMENT deploy"
             ]

data/test/fixtures/shipit/stacks.yml

@@ -16,7 +16,7 @@ shipit:
         ]
       },
       "dependencies": {"override": []},
-      "deploy": {"override": null, "variables": [{"name": "SAFETY_DISABLED", "title": "Set to 1 to do dangerous things", "default": 0}]},
+      "deploy": {"override": null, "interval": 60, "variables": [{"name": "SAFETY_DISABLED", "title": "Set to 1 to do dangerous things", "default": 0}]},
       "rollback": {"override": ["echo 'Rollback!'"]},
       "fetch": ["echo '42'"],
       "tasks": {
@@ -65,7 +65,7 @@ cyclimse:
         ]
       },
       "dependencies": {"override": []},
-      "deploy": {"override": null},
+      "deploy": {"max_commits": 2, "interval": 60, "override": null},
       "rollback": {"override": ["echo 'Rollback!'"]},
       "fetch": ["echo '42'"],
       "tasks": {

data/test/fixtures/shipit/tasks.yml

@@ -8,6 +8,8 @@ shipit:
   additions: 1
   deletions: 1
   created_at: <%= (60 - 1).minutes.ago.to_s(:db) %>
+  started_at: <%= (60 - 1).minutes.ago.to_s(:db) %>
+  ended_at: <%= (60 - 3).minutes.ago.to_s(:db) %>
 
 shipit2:
   user: walrus
@@ -79,7 +81,7 @@ shipit_complete:
   deletions: 342
   created_at: <%= (60 - 6).minutes.ago.to_s(:db) %>
   started_at: <%= (60 - 6).minutes.ago.to_s(:db) %>
-  ended_at: <%= (60 - 2).minutes.ago.to_s(:db) %>
+  ended_at: <%= (60 - 8).minutes.ago.to_s(:db) %>
 
 shipit_aborted:
   user: bob

data/test/fixtures/shipit/users.yml

@@ -15,3 +15,8 @@ anonymous:
   name:  Anonymous Legion
   email: anonymous@example.com
   # no Github info
+
+shipit:
+  name: Shipit
+  email: shipit@example.com
+  login: shipit

data/test/helpers/queries_helper.rb

@@ -7,7 +7,7 @@ module QueriesHelper
     counter = SQLCounter.new(ignored_sql)
     subscriber = ActiveSupport::Notifications.subscribe('sql.active_record', counter)
     yield counter
-    queries = counter.log.size == 0 ? '' : "\nQueries:\n#{counter.log.join("\n")}"
+    queries = counter.log.empty? ? '' : "\nQueries:\n#{counter.log.join("\n")}"
     assert_equal num, counter.log.size, "#{counter.log.size} instead of #{num} queries were executed.#{queries}"
   ensure
     ActiveSupport::Notifications.unsubscribe(subscriber)

data/test/models/deploys_test.rb

@@ -218,6 +218,13 @@ module Shipit
       end
     end
 
+    test "transitioning to success schedule an update of the estimated deploy duration" do
+      @deploy = shipit_deploys(:shipit_running)
+      assert_enqueued_with(job: UpdateEstimatedDeployDurationJob, args: [@deploy.stack]) do
+        @deploy.complete!
+      end
+    end
+
     test "transitioning to success schedule a fetch of the deployed revision" do
       @deploy = shipit_deploys(:shipit_running)
       assert_enqueued_with(job: FetchDeployedRevisionJob, args: [@deploy.stack]) do

data/test/models/duration_test.rb

@@ -2,6 +2,17 @@ require 'test_helper'
 
 module Shipit
   class DurationTest < ActiveSupport::TestCase
+    test "#<=> allow comparisons" do
+      assert_equal Duration.new(1), Duration.new(1)
+      assert Duration.new(2) > Duration.new(1)
+      assert Duration.new(2) > 1
+      assert 1 < Duration.new(2)
+    end
+
+    test "can be added to a Time instance" do
+      assert_equal Time.at(42), Time.at(40) + Duration.new(2)
+    end
+
     test "#to_s is precise and readable for humans" do
       assert_equal '1m01s', Duration.new(61).to_s
       assert_equal '1m00s', Duration.new(60).to_s
@@ -9,5 +20,17 @@ module Shipit
       assert_equal '2d00h00m00s', Duration.new(2.days).to_s
       assert_equal '0s', Duration.new(0).to_s
     end
+
+    test ".parse can read human format" do
+      assert_equal Duration.new(61), Duration.parse('1m01s')
+      assert_equal Duration.new(60), Duration.parse('1m00s')
+      assert_equal Duration.new(59), Duration.parse('59s')
+      assert_equal Duration.new(2.days), Duration.parse('2d00h00m00s')
+      assert_equal Duration.new(0), Duration.parse('0s')
+    end
+
+    test ".parse accepts integers as seconds" do
+      assert_equal Duration.new(42), Duration.parse(42)
+    end
   end
 end