shipit-engine 0.5.2 → 0.6.0

data/app/helpers/shipit/shipit_helper.rb

@@ -0,0 +1,64 @@
+module Shipit
+  module ShipitHelper
+    def include_plugins(stack)
+      stack.plugins.flat_map do |plugin, config|
+        plugin_tags(plugin, config)
+      end.join.html_safe
+    end
+
+    def plugin_tags(plugin, config)
+      tags = []
+      tags << tag('meta', name: "#{plugin}-config", content: config.to_json) if config
+      tags << javascript_include_tag("plugins/#{plugin}")
+      tags << stylesheet_link_tag("plugins/#{plugin}")
+      tags
+    end
+
+    def missing_github_oauth_message
+      (<<-MESSAGE).html_safe
+        Shipit requires a GitHub application to authenticate users.
+        If you haven't created an application on GitHub yet, you can do so in the
+        #{link_to 'Settings', Shipit.github_url('/settings/applications/new'), target: '_blank'}
+        section of your profile. You can also create applications for organizations.
+      MESSAGE
+    end
+
+    def missing_github_oauth_id_message
+      (<<-MESSAGE).html_safe
+        Copy the Client ID from your GitHub application,
+        and paste it into the secrets.yml file under <code>github_oauth.id</code>.
+       MESSAGE
+    end
+
+    def missing_github_oauth_secret_message
+      (<<-MESSAGE).html_safe
+        Copy the Client Secret from your GitHub application,
+        and paste it into the secrets.yml file under <code>github_oauth.secret</code>.
+       MESSAGE
+    end
+
+    def missing_github_api_credentials_message
+      (<<-MESSAGE).html_safe
+        Shipit needs API access to GitHub. You can
+        #{link_to 'create an access token', Shipit.github_url('/settings/tokens'), target: '_blank'}
+        with the following permissions:
+        <code>admin:repo_hook</code>, <code>admin:org_hook</code> and <code>repo</code>
+        and add it to the secrets.yml file under the key <code>github_api.access_token</code>.
+      MESSAGE
+    end
+
+    def missing_redis_url_message
+      (<<-MESSAGE).html_safe
+        Shipit needs a Redis server. Please configure the Redis URL in the secrets.yml file of your app,
+        under the key <code>redis_url</code>.
+      MESSAGE
+    end
+
+    def missing_host_message
+      (<<-MESSAGE).html_safe
+        Shipit needs the host of the application before generating links in background jobs.
+        Add the host name to the secrets.yml file, under the <code>host</code> key.
+      MESSAGE
+    end
+  end
+end

data/app/helpers/shipit/stacks_helper.rb

@@ -0,0 +1,78 @@
+module Shipit
+  module StacksHelper
+    COMMIT_TITLE_LENGTH = 79
+
+    def redeploy_button(commit)
+      url = new_stack_deploy_path(@stack, sha: commit.sha)
+      classes = %W(btn btn--primary deploy-action #{commit.state})
+
+      unless commit.stack.deployable?
+        classes.push(ignore_lock? ? 'btn--warning' : 'btn--disabled')
+      end
+
+      caption = 'Redeploy'
+      caption = 'Locked' if commit.stack.locked? && !ignore_lock?
+      caption = 'Deploy in progress...' if commit.stack.deploying?
+
+      link_to(caption, url, class: classes)
+    end
+
+    def ignore_lock?
+      params[:force].present?
+    end
+
+    def deploy_button(commit)
+      url = new_stack_deploy_path(@stack, sha: commit.sha)
+      classes = %W(btn btn--primary deploy-action #{commit.state})
+      if deploy_button_disabled?(commit)
+        classes.push(params[:force].present? ? 'btn--warning' : 'btn--disabled')
+      end
+
+      link_to(deploy_button_caption(commit), url, class: classes)
+    end
+
+    def github_change_url(commit)
+      commit.pull_request_url || github_commit_url(commit)
+    end
+
+    def render_commit_message(commit)
+      message = commit.pull_request_title || commit.message
+      content_tag(:span, message.truncate(COMMIT_TITLE_LENGTH), class: 'event-message')
+    end
+
+    def render_commit_message_with_link(commit)
+      link_to(render_commit_message(commit), github_change_url(commit), target: '_blank')
+    end
+
+    def render_commit_id_link(commit)
+      if commit.pull_request?
+        pull_request_link(commit) + " (#{render_raw_commit_id_link(commit)})".html_safe
+      else
+        render_raw_commit_id_link(commit)
+      end
+    end
+
+    def pull_request_link(commit)
+      link_to("##{commit.pull_request_id}", commit.pull_request_url, target: '_blank', class: 'number')
+    end
+
+    def render_raw_commit_id_link(commit)
+      link_to(commit.short_sha, github_commit_url(commit), target: '_blank', class: 'number')
+    end
+
+    private
+
+    def deploy_button_disabled?(commit)
+      !commit.deployable? || !commit.stack.deployable?
+    end
+
+    def deploy_button_caption(commit)
+      state = commit.status.state
+      state = 'locked' if commit.stack.locked? && !ignore_lock?
+      if commit.deployable?
+        state = commit.stack.deploying? ? 'deploying' : 'enabled'
+      end
+      t("deploy_button.caption.#{state}")
+    end
+  end
+end

data/app/helpers/shipit/tasks_helper.rb

@@ -0,0 +1,11 @@
+module Shipit
+  module TasksHelper
+    def task_description(task)
+      if task.class == Task
+        task.definition.action
+      else
+        t("#{task.class.name.underscore.pluralize}.description", sha: task.until_commit.short_sha)
+      end
+    end
+  end
+end

data/app/jobs/shipit/background_job.rb

@@ -0,0 +1,24 @@
+module Shipit
+  class BackgroundJob < ActiveJob::Base
+    class << self
+      attr_accessor :timeout
+    end
+
+    def perform(*)
+      with_timeout do
+        super
+      end
+    end
+
+    private
+
+    def with_timeout(&block)
+      return yield unless timeout
+      Timeout.timeout(timeout, &block)
+    end
+
+    def logger
+      Rails.logger
+    end
+  end
+end

data/app/jobs/shipit/background_job/unique.rb

@@ -0,0 +1,28 @@
+module Shipit
+  class BackgroundJob
+    module Unique
+      extend ActiveSupport::Concern
+
+      DEFAULT_TIMEOUT = 10
+
+      included do
+        around_perform { |job, block| job.acquire_lock(&block) }
+        cattr_accessor :lock_timeout
+      end
+
+      def acquire_lock(&block)
+        mutex = Redis::Lock.new(
+          lock_key(*arguments),
+          Shipit.redis,
+          expiration: self.class.timeout || DEFAULT_TIMEOUT,
+          timeout: self.class.lock_timeout || 0,
+        )
+        mutex.lock(&block)
+      end
+
+      def lock_key(*args)
+        ActiveJob::Arguments.serialize([self.class.name] + args).join('-')
+      end
+    end
+  end
+end

data/app/jobs/shipit/cache_deploy_spec_job.rb

@@ -0,0 +1,12 @@
+module Shipit
+  class CacheDeploySpecJob < BackgroundJob
+    include BackgroundJob::Unique
+
+    def perform(stack)
+      commands = Commands.for(stack)
+      commands.with_temporary_working_directory(commit: stack.commits.last) do |path|
+        stack.update!(cached_deploy_spec: DeploySpec::FileSystem.new(path, stack.environment))
+      end
+    end
+  end
+end

data/app/jobs/shipit/chunk_rollup_job.rb

@@ -0,0 +1,21 @@
+module Shipit
+  class ChunkRollupJob < BackgroundJob
+    include BackgroundJob::Unique
+
+    queue_as :default
+
+    def perform(task)
+      unless task.finished?
+        logger.error("Task ##{task.id} is not finished (current state: #{task.status}). Aborting.")
+        return
+      end
+
+      if task.rolled_up?
+        logger.error("Task ##{task.id} has already been rolled up. Aborting.")
+        return
+      end
+
+      task.rollup_chunks
+    end
+  end
+end

data/app/jobs/shipit/clear_git_cache_job.rb

@@ -0,0 +1,9 @@
+module Shipit
+  class ClearGitCacheJob < BackgroundJob
+    queue_as :default
+
+    def perform(stack)
+      stack.clear_git_cache!
+    end
+  end
+end

data/app/jobs/shipit/deliver_hook_job.rb

@@ -0,0 +1,9 @@
+module Shipit
+  class DeliverHookJob < BackgroundJob
+    queue_as :hooks
+
+    def perform(delivery)
+      delivery.send!
+    end
+  end
+end

data/app/jobs/shipit/destroy_stack_job.rb

@@ -0,0 +1,9 @@
+module Shipit
+  class DestroyStackJob < BackgroundJob
+    queue_as :default
+
+    def perform(stack)
+      stack.destroy!
+    end
+  end
+end

data/app/jobs/shipit/emit_event_job.rb

@@ -0,0 +1,10 @@
+module Shipit
+  class EmitEventJob < BackgroundJob
+    queue_as :hooks
+
+    def perform(params)
+      event, stack_id, payload = params.with_indifferent_access.values_at('event', 'stack_id', 'payload')
+      Hook.deliver(event, stack_id, JSON.load(payload))
+    end
+  end
+end

data/app/jobs/shipit/fetch_commit_stats_job.rb

@@ -0,0 +1,9 @@
+module Shipit
+  class FetchCommitStatsJob < BackgroundJob
+    queue_as :default
+
+    def perform(commit)
+      commit.fetch_stats!
+    end
+  end
+end

data/app/jobs/shipit/fetch_deployed_revision_job.rb

@@ -0,0 +1,23 @@
+module Shipit
+  class FetchDeployedRevisionJob < BackgroundJob
+    queue_as :default
+
+    def perform(stack)
+      return if stack.deploying?
+
+      commands = StackCommands.new(stack)
+
+      begin
+        sha = commands.fetch_deployed_revision
+      rescue DeploySpec::Error
+      end
+
+      return unless sha.present?
+
+      begin
+        stack.update_deployed_revision(sha)
+      rescue ActiveRecord::RecordNotFound
+      end
+    end
+  end
+end

data/app/jobs/shipit/git_mirror_update_job.rb

@@ -0,0 +1,12 @@
+module Shipit
+  class GitMirrorUpdateJob < BackgroundJob
+    queue_as :default
+
+    def perform(stack)
+      commands = StackCommands.new(stack)
+      stack.acquire_git_cache_lock do
+        commands.fetch.run
+      end
+    end
+  end
+end

data/app/jobs/shipit/github_sync_job.rb

@@ -0,0 +1,55 @@
+module Shipit
+  class GithubSyncJob < BackgroundJob
+    include BackgroundJob::Unique
+
+    MAX_FETCHED_COMMITS = 10
+    queue_as :default
+
+    self.timeout = 60
+    self.lock_timeout = 20
+
+    def perform(params)
+      @stack = Stack.find(params[:stack_id])
+
+      handle_github_errors do
+        new_commits, shared_parent = fetch_missing_commits { @stack.github_commits }
+
+        @stack.transaction do
+          shared_parent.try!(:detach_children!)
+          new_commits.each do |gh_commit|
+            @stack.commits.create_from_github!(gh_commit)
+          end
+        end
+      end
+      CacheDeploySpecJob.perform_later(@stack)
+    end
+
+    def fetch_missing_commits(&block)
+      commits = []
+      iterator = Shipit::FirstParentCommitsIterator.new(&block)
+      iterator.each_with_index do |commit, index|
+        break if index >= MAX_FETCHED_COMMITS
+
+        if shared_parent = lookup_commit(commit.sha)
+          return commits, shared_parent
+        end
+        commits.unshift(commit)
+      end
+      return commits, nil
+    end
+
+    protected
+
+    def handle_github_errors
+      yield
+    rescue Octokit::NotFound
+      @stack.mark_as_inaccessible!
+    else
+      @stack.mark_as_accessible!
+    end
+
+    def lookup_commit(sha)
+      @stack.commits.find_by_sha(sha)
+    end
+  end
+end

data/app/jobs/shipit/perform_commit_checks_job.rb

@@ -0,0 +1,7 @@
+module Shipit
+  class PerformCommitChecksJob < BackgroundJob
+    def perform(commit:)
+      commit.checks.run
+    end
+  end
+end

data/app/jobs/shipit/perform_task_job.rb

@@ -0,0 +1,57 @@
+module Shipit
+  class PerformTaskJob < BackgroundJob
+    queue_as :deploys
+
+    def perform(task)
+      @task = task
+      @commands = Commands.for(@task)
+      unless @task.pending?
+        logger.error("Task ##{@task.id} already in `#{@task.status}` state. Aborting.")
+        return
+      end
+      run
+    ensure
+      @task.clear_working_directory
+    end
+
+    def run
+      @task.run!
+      checkout_repository
+      perform_task
+      @task.complete!
+    rescue Command::Error => error
+      @task.report_failure!(error)
+    rescue StandardError => error
+      @task.report_error!(error)
+    end
+
+    def perform_task
+      Bundler.with_clean_env do
+        capture_all @commands.install_dependencies
+        capture_all @commands.perform
+      end
+    end
+
+    def checkout_repository
+      @task.acquire_git_cache_lock do
+        capture @commands.fetch
+        capture @commands.clone
+      end
+      capture @commands.checkout(@task.until_commit)
+    end
+
+    def capture_all(commands)
+      commands.map { |c| capture(c) }
+    end
+
+    def capture(command)
+      command.start
+      @task.write("$ #{command}\npid: #{command.pid}\n")
+      @task.pid = command.pid
+      command.stream! do |line|
+        @task.write(line)
+      end
+      @task.write("\n")
+    end
+  end
+end