shipit-engine 0.14.0 → 0.15.0

data/app/models/shipit/task.rb

@@ -1,5 +1,7 @@
 module Shipit
   class Task < ActiveRecord::Base
+    include DeferredTouch
+
     PRESENCE_CHECK_TIMEOUT = 15
     ACTIVE_STATUSES = %w(pending running aborting).freeze
     COMPLETED_STATUSES = %w(success error failed flapping aborted).freeze
@@ -10,10 +12,12 @@ module Shipit
     belongs_to :deploy, foreign_key: :parent_id, required: false # required for fixtures
 
     belongs_to :user
-    belongs_to :stack, touch: true, counter_cache: true
+    belongs_to :stack, counter_cache: true
     belongs_to :until_commit, class_name: 'Commit'
     belongs_to :since_commit, class_name: 'Commit'
 
+    deferred_touch stack: :updated_at
+
     has_many :chunks, -> { order(:id) }, class_name: 'OutputChunk', dependent: :delete_all
 
     serialize :definition, TaskDefinition

data/config/secrets.development.example.yml

@@ -17,3 +17,7 @@ github_oauth:
 # features:
 #   - bootstrap
 
+# To work on the kubernetes deploy script
+# env:
+#   KUBECONFIG: # Path of the kubeconfig you want to use.
+#   GOOGLE_APPLICATION_CREDENTIALS: # Used if your kube auth provider is gcp, in which case it should be path of the service account credentials you want to use. Alternatively, you can run `gcloud auth application-default login` (and remove this variable) to use your own google account locally.

data/config/secrets.development.shopify.yml

@@ -17,3 +17,7 @@ github_oauth:
 # features:
 #   - bootstrap
 
+# To work on the kubernetes deploy script
+# env:
+#   KUBECONFIG: # Path of the kubeconfig you want to use.
+#   GOOGLE_APPLICATION_CREDENTIALS: # Used if your kube auth provider is gcp, in which case it should be path of the service account credentials you want to use. Alternatively, you can run `gcloud auth application-default login` (and remove this variable) to use your own google account locally.

data/config/secrets.development.yml

@@ -2,7 +2,7 @@ host: 'http://shipit-engine.localhost'
 redis_url: 'redis://shipit-engine.railgun:6379'
 
 github_api:
-  access_token: caa4f8b2bb3d94269291faf9b54b85457db0173b
+  access_token: 4cc410d7a8fb04f6095076f519f7e0981d915136
 
 # If you want to test GitHub Authentication
 github_oauth:

data/db/migrate/20161205144522_add_indexes_on_deliveries.rb

@@ -0,0 +1,17 @@
+class AddIndexesOnDeliveries < ActiveRecord::Migration[5.0]
+  def up
+    change_column :deliveries, :status, :string, limit: 50
+    change_column :deliveries, :event, :string, limit: 50
+    add_index :deliveries, [:hook_id, :event, :status]
+    add_index :deliveries, [:status, :event]
+    add_index :deliveries, :created_at
+  end
+
+  def down
+    change_column :deliveries, :status, :string, limit: 255
+    change_column :deliveries, :event, :string, limit: 255
+    remove_index :deliveries, [:hook_id, :event, :status]
+    remove_index :deliveries, [:status, :event]
+    remove_index :deliveries, :created_at
+  end
+end

data/db/migrate/20161206104100_delete_orphan_statuses.rb

@@ -0,0 +1,10 @@
+class DeleteOrphanStatuses < ActiveRecord::Migration[5.0]
+  def up
+    ids = Shipit::Status.left_joins(:commit).where(commits: {id: nil}).pluck(:id)
+    say "Found #{ids.size} orphan statuses"
+    Shipit::Status.where(id: ids).delete_all
+  end
+
+  def down
+  end
+end

data/db/migrate/20161206104224_denormalize_stack_id_on_statuses.rb

@@ -0,0 +1,5 @@
+class DenormalizeStackIdOnStatuses < ActiveRecord::Migration[5.0]
+  def change
+    add_column :statuses, :stack_id, :integer, null: true
+  end
+end

data/db/migrate/20161206104817_backfill_stack_id_on_statuses.rb

@@ -0,0 +1,13 @@
+class BackfillStackIdOnStatuses < ActiveRecord::Migration[5.0]
+  def up
+    Shipit::Commit.order(stack_id: :asc).find_in_batches do |commits|
+      commits.group_by(&:stack_id).each do |stack_id, stack_commits|
+        Shipit::Status.where(commit_id: stack_commits.map(&:id)).update_all(stack_id: stack_id)
+      end
+      print '.'
+    end
+  end
+
+  def down
+  end
+end

data/db/migrate/20161206105318_makes_stack_id_not_null_on_statuses.rb

@@ -0,0 +1,5 @@
+class MakesStackIdNotNullOnStatuses < ActiveRecord::Migration[5.0]
+  def change
+    change_column_null :statuses, :stack_id, false
+  end
+end

data/lib/shipit.rb

@@ -46,6 +46,7 @@ require 'shipit/deploy_commands'
 require 'shipit/rollback_commands'
 require 'shipit/environment_variables'
 require 'shipit/stat'
+require 'shipit/strip_cache_control'
 
 SafeYAML::OPTIONS[:default_mode] = :safe
 SafeYAML::OPTIONS[:deserialize_symbols] = false
@@ -111,8 +112,10 @@ module Shipit
         logger: Rails.logger,
         serializer: NullSerializer,
       )
+      builder.use StripCacheControl
       builder.use Octokit::Response::RaiseError
       builder.adapter Faraday.default_adapter
+      yield builder if block_given?
     end
   end
 

data/lib/shipit/strip_cache_control.rb

@@ -0,0 +1,40 @@
+module Shipit
+  class StripCacheControl < Faraday::Middleware
+    def call(request_env)
+      @app.call(request_env).on_complete do |response_env|
+        if headers = response_env[:response_headers]
+          headers.delete('last-modified')
+          directives = parse(headers['cache-control'].to_s)
+          directives.delete('max-age')
+          directives.delete('s-maxage')
+          headers['cache-control'] = dump(directives)
+        end
+      end
+    end
+
+    private
+
+    def dump(directives)
+      directives.map do |k, v|
+        if v == true
+          k
+        else
+          "#{k}=#{v}"
+        end
+      end.join(', ')
+    end
+
+    def parse(header)
+      directives = {}
+
+      header.delete(' ').split(',').each do |part|
+        next if part.empty?
+
+        name, value = part.split('=', 2)
+        directives[name.downcase] = (value || true) unless name.empty?
+      end
+
+      directives
+    end
+  end
+end

data/lib/shipit/version.rb

@@ -1,3 +1,3 @@
 module Shipit
-  VERSION = '0.14.0'.freeze
+  VERSION = '0.15.0'.freeze
 end

data/lib/snippets/assert-npm-version-tag

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+VERSION=`node --eval "var version = require('./package.json').version; console.log(version)"`
+
+echo -e "\033[0;32mTrying to publish version $VERSION\033[0m"
+
+git tag | grep "^v$VERSION$" > /dev/null
+if [ $? != '0' ]; then
+  echo -e "\033[1;31mYou need to create the \033[0;33mv$VERSION\033[1;31m tag first.\033[0m"
+  exit 1
+fi
+
+TAG_REV=`git rev-list -n1 v$VERSION`
+HEAD_REV=`git rev-parse HEAD`
+
+if [ $TAG_REV != $HEAD_REV ]; then
+  echo -e "\033[1;31mYou're attempting to publish \033[0;33m$HEAD_REV\033[1;31m as \033[0;33mv$VERSION\033[1;31m but it already points to \033[0;33m$TAG_REV\033[1;31m.\033[0m"
+  exit 1
+fi
+
+echo -e "\033[0;32mAll clear!\033[0m"
+exit 0

data/lib/snippets/deploy-to-gke

@@ -21,12 +21,12 @@ require 'tempfile'
 class GKEDeployment
   class FatalDeploymentError < StandardError; end
 
-  def initialize(namespace:, gcloud_deployment:, gcloud_project:, environment:, current_sha:, key_dir:, template_folder: nil)
+  def initialize(namespace:, gcloud_deployment:, gcloud_project:, environment:, current_sha:, key_file:, template_folder: nil)
     @namespace = namespace
     @gcloud_deployment = gcloud_deployment
     @gcloud_project = gcloud_project
     @current_sha = current_sha
-    @key_file = "#{key_dir}/#{@gcloud_project}.json"
+    @key_file = key_file
     @template_path = './' + (template_folder || "config/deploy/#{environment}")
 
     # Validate params + check existance of auth key and template(s)
@@ -106,7 +106,6 @@ class GKEDeployment
   end
 
   def authorize_gcloud
-    ENV['GOOGLE_APPLICATION_CREDENTIALS'] = @key_file
     status = run_command('gcloud', '-q', 'auth', 'activate-service-account', '--key-file', @key_file)
     status = run_command('gcloud', '-q', 'config', 'set', 'project', @gcloud_project) if status
     raise FatalDeploymentError, "Failed to set gcloud project #{@gcloud_project}" unless status
@@ -157,6 +156,6 @@ deployment = GKEDeployment.new(
   environment: ENV['ENVIRONMENT'],
   template_folder: ENV['K8S_TEMPLATE_FOLDER'],
   current_sha: ENV['REVISION'],
-  key_dir: ENV['GCLOUD_CREDENTIALS_DIR']
+  key_file: ENV['GOOGLE_APPLICATION_CREDENTIALS']
 )
 deployment.run

data/lib/tasks/cron.rake

@@ -6,11 +6,18 @@ namespace :cron do
     Shipit::GithubStatus.refresh_status
   end
 
+  task hourly: [:rollup, :purge_deliveries, :refresh_users]
+
   desc "Rolls-up output chunks for completed deploys older than an hour"
   task rollup: :environment do
     Shipit::Task.due_for_rollup.find_each(&:schedule_rollup_chunks)
   end
 
+  desc "Delete old hook delivery records"
+  task purge_deliveries: :environment do
+    Shipit::Delivery.due_for_deletion.delete_all
+  end
+
   task refresh_users: :environment do
     Shipit::User.refresh_shard(Time.now.hour % 24, 24)
   end

data/test/dummy/config/environments/development.rb

@@ -38,6 +38,10 @@ Rails.application.configure do
 
   # Raises error for missing translations
   # config.action_view.raise_on_missing_translations = true
-
-  config.active_job.queue_adapter = :sucker_punch
+  if Rails.application.config_for(:database)['adapter'] == 'sqlite3'
+    Shipit::DeferredTouch.enabled = false
+  end
+  config.active_job.queue_adapter = :async
+  
+  Pubsubstub.use_persistent_connections = false
 end

data/test/dummy/config/environments/test.rb

@@ -39,4 +39,8 @@ Rails.application.configure do
 
   # Raises error for missing translations
   # config.action_view.raise_on_missing_translations = true
+
+  config.to_prepare do
+    Shipit::DeferredTouch.enabled = false
+  end
 end

data/test/dummy/db/schema.rb

@@ -1,4 +1,3 @@
-# encoding: UTF-8
 # This file is auto-generated from the current state of the database. Instead
 # of editing this file, please use the migrations feature of Active Record to
 # incrementally modify your database, and then regenerate this schema definition.
@@ -11,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20160822131405) do
+ActiveRecord::Schema.define(version: 20161206105318) do
 
   create_table "api_clients", force: :cascade do |t|
     t.text     "permissions", limit: 65535
@@ -20,10 +19,9 @@ ActiveRecord::Schema.define(version: 20160822131405) do
     t.datetime "updated_at",                             null: false
     t.string   "name",        limit: 255,   default: ""
     t.integer  "stack_id",    limit: 4
+    t.index ["creator_id"], name: "index_api_clients_on_creator_id"
   end
 
-  add_index "api_clients", ["creator_id"], name: "index_api_clients_on_creator_id"
-
   create_table "commit_deployment_statuses", force: :cascade do |t|
     t.integer  "commit_deployment_id"
     t.string   "status"
@@ -31,10 +29,9 @@ ActiveRecord::Schema.define(version: 20160822131405) do
     t.string   "api_url"
     t.datetime "created_at",           null: false
     t.datetime "updated_at",           null: false
+    t.index ["commit_deployment_id"], name: "index_commit_deployment_statuses_on_commit_deployment_id"
   end
 
-  add_index "commit_deployment_statuses", ["commit_deployment_id"], name: "index_commit_deployment_statuses_on_commit_deployment_id"
-
   create_table "commit_deployments", force: :cascade do |t|
     t.integer  "commit_id"
     t.integer  "task_id"
@@ -42,11 +39,10 @@ ActiveRecord::Schema.define(version: 20160822131405) do
     t.string   "api_url"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.index ["commit_id", "task_id"], name: "index_commit_deployments_on_commit_id_and_task_id", unique: true
+    t.index ["task_id"], name: "index_commit_deployments_on_task_id"
   end
 
-  add_index "commit_deployments", ["commit_id", "task_id"], name: "index_commit_deployments_on_commit_id_and_task_id", unique: true
-  add_index "commit_deployments", ["task_id"], name: "index_commit_deployments_on_task_id"
-
   create_table "commits", force: :cascade do |t|
     t.integer  "stack_id",     limit: 4,                     null: false
     t.integer  "author_id",    limit: 4,                     null: false
@@ -60,19 +56,18 @@ ActiveRecord::Schema.define(version: 20160822131405) do
     t.datetime "committed_at",                               null: false
     t.integer  "additions",    limit: 4
     t.integer  "deletions",    limit: 4
+    t.index ["author_id"], name: "index_commits_on_author_id"
+    t.index ["committer_id"], name: "index_commits_on_committer_id"
+    t.index ["created_at"], name: "index_commits_on_created_at"
+    t.index ["stack_id"], name: "index_commits_on_stack_id"
   end
 
-  add_index "commits", ["author_id"], name: "index_commits_on_author_id"
-  add_index "commits", ["committer_id"], name: "index_commits_on_committer_id"
-  add_index "commits", ["created_at"], name: "index_commits_on_created_at"
-  add_index "commits", ["stack_id"], name: "index_commits_on_stack_id"
-
   create_table "deliveries", force: :cascade do |t|
     t.integer  "hook_id",          limit: 4,                            null: false
-    t.string   "status",           limit: 255,      default: "pending", null: false
+    t.string   "status",           limit: 50,       default: "pending", null: false
     t.string   "url",              limit: 4096,                         null: false
     t.string   "content_type",     limit: 255,                          null: false
-    t.string   "event",            limit: 255,                          null: false
+    t.string   "event",            limit: 50,                           null: false
     t.text     "payload",          limit: 16777215,                     null: false
     t.integer  "response_code",    limit: 4
     t.text     "response_headers", limit: 65535
@@ -80,6 +75,9 @@ ActiveRecord::Schema.define(version: 20160822131405) do
     t.datetime "delivered_at"
     t.datetime "created_at",                                            null: false
     t.datetime "updated_at",                                            null: false
+    t.index ["created_at"], name: "index_deliveries_on_created_at"
+    t.index ["hook_id", "event", "status"], name: "index_deliveries_on_hook_id_and_event_and_status"
+    t.index ["status", "event"], name: "index_deliveries_on_status_and_event"
   end
 
   create_table "github_hooks", force: :cascade do |t|
@@ -92,11 +90,10 @@ ActiveRecord::Schema.define(version: 20160822131405) do
     t.string   "api_url",      limit: 255
     t.string   "type",         limit: 255
     t.string   "organization", limit: 39
+    t.index ["organization", "event"], name: "index_github_hooks_on_organization_and_event", unique: true
+    t.index ["stack_id", "event"], name: "index_github_hooks_on_stack_id_and_event", unique: true
   end
 
-  add_index "github_hooks", ["organization", "event"], name: "index_github_hooks_on_organization_and_event", unique: true
-  add_index "github_hooks", ["stack_id", "event"], name: "index_github_hooks_on_stack_id_and_event", unique: true
-
   create_table "hooks", force: :cascade do |t|
     t.integer  "stack_id",     limit: 4
     t.string   "delivery_url", limit: 4096,                  null: false
@@ -106,29 +103,26 @@ ActiveRecord::Schema.define(version: 20160822131405) do
     t.boolean  "insecure_ssl",              default: false,  null: false
     t.datetime "created_at",                                 null: false
     t.datetime "updated_at",                                 null: false
+    t.index ["stack_id"], name: "index_hooks_on_stack_id"
   end
 
-  add_index "hooks", ["stack_id"], name: "index_hooks_on_stack_id"
-
   create_table "memberships", force: :cascade do |t|
     t.integer  "team_id",    limit: 4
     t.integer  "user_id",    limit: 4
     t.datetime "created_at",           null: false
     t.datetime "updated_at",           null: false
+    t.index ["team_id", "user_id"], name: "index_memberships_on_team_id_and_user_id", unique: true
+    t.index ["user_id"], name: "index_memberships_on_user_id"
   end
 
-  add_index "memberships", ["team_id", "user_id"], name: "index_memberships_on_team_id_and_user_id", unique: true
-  add_index "memberships", ["user_id"], name: "index_memberships_on_user_id"
-
   create_table "output_chunks", force: :cascade do |t|
     t.integer  "task_id",    limit: 4
     t.text     "text",       limit: 16777215
     t.datetime "created_at"
     t.datetime "updated_at"
+    t.index ["task_id"], name: "index_output_chunks_on_task_id"
   end
 
-  add_index "output_chunks", ["task_id"], name: "index_output_chunks_on_task_id"
-
   create_table "stacks", force: :cascade do |t|
     t.string   "repo_name",                         limit: 100,                          null: false
     t.string   "repo_owner",                        limit: 39,                           null: false
@@ -148,10 +142,9 @@ ActiveRecord::Schema.define(version: 20160822131405) do
     t.integer  "estimated_deploy_duration",                       default: 1,            null: false
     t.datetime "continuous_delivery_delayed_since"
     t.datetime "locked_since"
+    t.index ["repo_owner", "repo_name", "environment"], name: "stack_unicity", unique: true
   end
 
-  add_index "stacks", ["repo_owner", "repo_name", "environment"], name: "stack_unicity", unique: true
-
   create_table "statuses", force: :cascade do |t|
     t.string   "state",       limit: 255
     t.string   "target_url",  limit: 255
@@ -160,10 +153,10 @@ ActiveRecord::Schema.define(version: 20160822131405) do
     t.integer  "commit_id",   limit: 4
     t.datetime "created_at"
     t.datetime "updated_at"
+    t.integer  "stack_id",                                      null: false
+    t.index ["commit_id"], name: "index_statuses_on_commit_id"
   end
 
-  add_index "statuses", ["commit_id"], name: "index_statuses_on_commit_id"
-
   create_table "tasks", force: :cascade do |t|
     t.integer  "stack_id",              limit: 4,                         null: false
     t.integer  "since_commit_id",       limit: 4,                         null: false
@@ -185,16 +178,15 @@ ActiveRecord::Schema.define(version: 20160822131405) do
     t.boolean  "allow_concurrency",                   default: false,     null: false
     t.datetime "started_at"
     t.datetime "ended_at"
+    t.index ["rolled_up", "created_at", "status"], name: "index_tasks_on_rolled_up_and_created_at_and_status"
+    t.index ["since_commit_id"], name: "index_tasks_on_since_commit_id"
+    t.index ["stack_id", "allow_concurrency", "status"], name: "index_active_tasks"
+    t.index ["type", "stack_id", "parent_id"], name: "index_tasks_by_stack_and_parent"
+    t.index ["type", "stack_id", "status"], name: "index_tasks_by_stack_and_status"
+    t.index ["until_commit_id"], name: "index_tasks_on_until_commit_id"
+    t.index ["user_id"], name: "index_tasks_on_user_id"
   end
 
-  add_index "tasks", ["rolled_up", "created_at", "status"], name: "index_tasks_on_rolled_up_and_created_at_and_status"
-  add_index "tasks", ["since_commit_id"], name: "index_tasks_on_since_commit_id"
-  add_index "tasks", ["stack_id", "allow_concurrency", "status"], name: "index_active_tasks"
-  add_index "tasks", ["type", "stack_id", "parent_id"], name: "index_tasks_by_stack_and_parent"
-  add_index "tasks", ["type", "stack_id", "status"], name: "index_tasks_by_stack_and_status"
-  add_index "tasks", ["until_commit_id"], name: "index_tasks_on_until_commit_id"
-  add_index "tasks", ["user_id"], name: "index_tasks_on_user_id"
-
   create_table "teams", force: :cascade do |t|
     t.integer  "github_id",    limit: 4
     t.string   "api_url",      limit: 255
@@ -203,10 +195,9 @@ ActiveRecord::Schema.define(version: 20160822131405) do
     t.string   "organization", limit: 39
     t.datetime "created_at",               null: false
     t.datetime "updated_at",               null: false
+    t.index ["organization", "slug"], name: "index_teams_on_organization_and_slug", unique: true
   end
 
-  add_index "teams", ["organization", "slug"], name: "index_teams_on_organization_and_slug", unique: true
-
   create_table "users", force: :cascade do |t|
     t.integer  "github_id",                        limit: 4
     t.string   "name",                             limit: 255, null: false
@@ -218,8 +209,7 @@ ActiveRecord::Schema.define(version: 20160822131405) do
     t.string   "avatar_url",                       limit: 255
     t.string   "encrypted_github_access_token"
     t.string   "encrypted_github_access_token_iv"
+    t.index ["login"], name: "index_users_on_login"
   end
 
-  add_index "users", ["login"], name: "index_users_on_login"
-
 end