shipitron 0.1.0 → 0.2.0

data/lib/shipitron/client/register_ecs_task_definitions.rb

@@ -0,0 +1,75 @@
+require 'shipitron'
+require 'shipitron/ecs_client'
+require 'shipitron/mustache_yaml_parser'
+
+module Shipitron
+  module Client
+    class RegisterEcsTaskDefinitions
+      include Metaractor
+      include EcsClient
+
+      required :region
+      required :task_def_directory
+
+      def call
+        Logger.info 'Creating ECS task definitions'
+
+        task_defs = Pathname.new(task_def_directory)
+        unless task_defs.directory?
+          fail_with_error!(
+            message: "task definition directory '#{task_def_directory}' does not exist"
+          )
+        end
+
+        task_defs.find do |path|
+          next if path.directory?
+
+          task_def = Smash.load(
+            path.to_s,
+            parser: MustacheYamlParser.new(
+              context: {
+                tag: 'latest'
+              }
+            )
+          )
+
+          begin
+            ecs_client(region: region).describe_task_definition(
+              task_definition: task_def.family
+            )
+          rescue Aws::ECS::Errors::ClientException => e
+            raise if e.message != 'Unable to describe task definition.'
+
+            Logger.info "Creating task definition '#{task_def.family}'"
+            Logger.debug "Task definition: #{task_def.to_h}"
+            ecs_client(region: region).register_task_definition(
+              task_def.to_h
+            )
+          else
+            Logger.info "Task definition '#{task_def.family}' already exists."
+          end
+        end
+
+        Logger.info 'Done'
+      rescue Aws::ECS::Errors::ServiceError => e
+        fail_with_errors!(messages: [
+          "Error: #{e.message}",
+          e.backtrace.join("\n")
+        ])
+      end
+
+      private
+      def region
+        context.region
+      end
+
+      def cluster_name
+        context.cluster_name
+      end
+
+      def task_def_directory
+        context.task_def_directory
+      end
+    end
+  end
+end

data/lib/shipitron/client/run_ecs_tasks.rb

@@ -0,0 +1,141 @@
+require 'shipitron'
+require 'shipitron/ecs_client'
+require 'shellwords'
+require 'base64'
+
+module Shipitron
+  module Client
+    class RunEcsTasks
+      include Metaractor
+      include EcsClient
+
+      required :application
+      required :clusters
+      required :shipitron_task
+      required :repository_url
+      required :s3_cache_bucket
+      required :image_name
+      required :ecs_task_defs
+      optional :ecs_task_def_templates
+      required :ecs_services
+      optional :ecs_service_templates
+      optional :build_script
+      optional :post_builds
+      optional :simulate
+
+      before do
+        context.post_builds ||= []
+        context.ecs_task_def_templates ||= []
+        context.ecs_service_templates ||= []
+      end
+
+      def call
+        Logger.info "Skipping ECS run_task calls due to --simulate" if simulate?
+
+        clusters.each do |cluster|
+          begin
+            if simulate?
+              command_args(cluster)
+              next
+            end
+
+            response = ecs_client(region: cluster.region).run_task(
+              cluster: cluster.name,
+              task_definition: shipitron_task,
+              overrides: {
+                container_overrides: [
+                  {
+                    name: 'shipitron',
+                    command: command_args(cluster)
+                  }
+                ]
+              },
+              count: 1,
+              started_by: 'shipitron'
+            )
+
+            if !response.failures.empty?
+              response.failures.each do |failure|
+                fail_with_error! message: "ECS run_task failure: #{failure.arn}: #{failure.reason}"
+              end
+            end
+
+          rescue Aws::ECS::Errors::ServiceError => e
+            fail_with_errors!(messages: [
+              "Error: #{e.message}",
+              e.backtrace.join("\n")
+            ])
+          end
+        end
+      end
+
+      private
+      def application
+        context.application
+      end
+
+      def clusters
+        context.clusters
+      end
+
+      def shipitron_task
+        context.shipitron_task
+      end
+
+      def escape(str)
+        Shellwords.escape(str)
+      end
+
+      def escaped(sym)
+        escape(context[sym])
+      end
+
+      def command_args(cluster)
+        [
+          'server_deploy',
+          '--name', escaped(:application),
+          '--repository', escaped(:repository_url),
+          '--bucket', escaped(:s3_cache_bucket),
+          '--image-name', escaped(:image_name),
+          '--region', escape(cluster.region),
+          '--cluster-name', escape(cluster.name),
+        ].tap do |ary|
+          ary << '--ecs-task-defs'
+          ary.concat(context.ecs_task_defs.each {|s| escape(s)})
+
+          ary << '--ecs-services'
+          ary.concat(context.ecs_services.each {|s| escape(s)})
+
+          if context.build_script != nil
+            ary.concat ['--build-script', escaped(:build_script)]
+          end
+
+          if !context.post_builds.empty?
+            ary << '--post-builds'
+            ary.concat(context.post_builds.map(&:to_s).each {|s| escape(s)})
+          end
+
+          if !context.ecs_task_def_templates.empty?
+            ary << '--ecs-task-def-templates'
+            ary.concat(context.ecs_task_def_templates.map {|t| Base64.urlsafe_encode64(t)})
+          end
+
+          if !context.ecs_service_templates.empty?
+            ary << '--ecs-service-templates'
+            ary.concat(context.ecs_service_templates.map {|t| Base64.urlsafe_encode64(t)})
+          end
+
+          if simulate?
+            Logger.info "server_deploy command: #{ary.join(' ')}"
+          else
+            Logger.debug "server_deploy command: #{ary.join(' ')}"
+          end
+        end
+      end
+
+      def simulate?
+        context.simulate == true
+      end
+    end
+  end
+end

data/lib/shipitron/consul_keys.rb

@@ -0,0 +1,34 @@
+require 'shipitron'
+require 'diplomat'
+
+module Shipitron
+  module ConsulKeys
+    extend self
+
+    def configure_consul_client!
+      if ENV['CONSUL_HOST'].nil?
+        raise 'Environment variable CONSUL_HOST required'
+      end
+
+      Diplomat.configure do |config|
+        config.url = "http://#{ENV['CONSUL_HOST']}:8500"
+      end
+    end
+
+    def fetch_key(key:)
+      Logger.debug "Fetching key #{key}"
+      value = Diplomat::Kv.get(key, {}, :return)
+      value = nil if value == ''
+      value
+    end
+
+    def fetch_key!(key:)
+      fetch_key(key: key).tap do |value|
+        if value.nil?
+          raise "Key #{key} not found in consul!"
+        end
+      end
+    end
+
+  end
+end

data/lib/shipitron/consul_lock.rb

@@ -0,0 +1,28 @@
+require 'diplomat'
+
+module Shipitron
+  module ConsulLock
+    extend self
+
+    class UnableToLock < StandardError; end
+
+    def with_lock(key:)
+      sessionid = nil
+      locked = false
+      sessionid = Diplomat::Session.create(Name: "#{key}.lock")
+      locked = Diplomat::Lock.acquire(key, sessionid)
+
+      if locked
+        yield
+      else
+        raise UnableToLock
+      end
+    ensure
+      if sessionid != nil
+        Diplomat::Lock.release(key, sessionid) if locked
+        Diplomat::Session.destroy(sessionid)
+      end
+    end
+
+  end
+end

data/lib/shipitron/docker_image.rb

@@ -0,0 +1,23 @@
+require 'shipitron'
+
+module Shipitron
+  class DockerImage < Hashie::Dash
+    property :name
+    property :tag
+
+    def name_with_tag(tag_override = nil)
+      tag_str = [tag_override, tag, ''].find {|str| !str.nil? }
+      tag_str = tag_str.to_s
+
+      if !tag_str.empty? && !tag_str.start_with?(':')
+        tag_str = tag_str.dup.prepend(':')
+      end
+
+      "#{name}#{tag_str}"
+    end
+
+    def to_s
+      name_with_tag
+    end
+  end
+end

data/lib/shipitron/ecs_client.rb

@@ -0,0 +1,22 @@
+require 'aws-sdk'
+
+module Shipitron
+  module EcsClient
+    def ecs_client(region:)
+      @ecs_clients ||= {}
+      @ecs_clients[region] ||= generate_ecs_client(region: region)
+    end
+
+    def generate_ecs_client(region:)
+      options = {region: region}
+      if Shipitron.config.aws_access_key_id? && Shipitron.config.aws_secret_access_key
+        options.merge!(
+          access_key_id: Shipitron.config.aws_access_key_id,
+          secret_access_key: Shipitron.config.aws_secret_access_key
+        )
+      end
+
+      Aws::ECS::Client.new(options)
+    end
+  end
+end

data/lib/shipitron/ecs_task_def.rb

@@ -0,0 +1,17 @@
+require 'shipitron'
+
+module Shipitron
+  class EcsTaskDef < Hashie::Dash
+    property :name
+    property :revision
+    property :params
+
+    def name_with_revision
+      "#{name}:#{revision}"
+    end
+
+    def to_s
+      name_with_revision
+    end
+  end
+end

data/lib/shipitron/fetch_bucket.rb

@@ -0,0 +1,26 @@
+require 'fog/aws'
+require 'fog/local' if ENV['FOG_LOCAL']
+
+module Shipitron
+  class FetchBucket
+    include Metaractor
+
+    required :name
+
+    def call
+      if ENV['FOG_LOCAL']
+        Logger.debug 'Using fog local storage'
+        storage = Fog::Storage.new provider: 'Local', local_root: '/fog'
+        context.bucket = storage.directories.create(key: name)
+      else
+        storage = Fog::Storage.new provider: 'AWS', use_iam_profile: true
+        context.bucket = storage.directories.get(name)
+      end
+    end
+
+    private
+    def name
+      context.name
+    end
+  end
+end

data/lib/shipitron/logger.rb

@@ -0,0 +1,33 @@
+require 'logger'
+
+module Shipitron
+  class Logger
+    class << self
+      %i[debug info warn error fatal].each do |sym|
+        define_method(sym) do |message|
+          logger.send(sym, "#{Thread.current[:logger_tag]}#{message}")
+        end
+      end
+    end
+
+    def self.tagged(tag)
+      existing_tag = Thread.current[:logger_tag]
+      Thread.current[:logger_tag] = "[#{tag}] "
+      yield
+    ensure
+      Thread.current[:logger_tag] = existing_tag
+    end
+
+    def self.logger
+      Thread.current[:logger] ||= ::Logger.new(STDOUT)
+    end
+
+    def self.level
+      logger.level
+    end
+
+    def self.level=(new_level)
+      logger.level = new_level
+    end
+  end
+end

data/lib/shipitron/mustache_yaml_parser.rb

@@ -0,0 +1,22 @@
+require 'mustache'
+require 'yaml'
+
+module Shipitron
+  class MustacheYamlParser
+    def initialize(context:nil, view:nil)
+      if (context.nil? && view.nil?) || (!context.nil? && !view.nil?)
+        raise ArgumentError, 'Either context or view required'
+      end
+
+      @context = context
+      @view = view
+
+      @view ||= Mustache
+    end
+
+    def perform(file_path)
+      file_path = file_path.is_a?(Pathname) ? file_path.to_s : file_path
+      YAML.load(@view.render(File.read(file_path), @context))
+    end
+  end
+end

data/lib/shipitron/parse_templates.rb

@@ -0,0 +1,32 @@
+require 'shipitron'
+require 'yaml'
+require 'mustache'
+
+module Shipitron
+  module Server
+    class ParseTemplates
+      include Metaractor
+
+      required :templates
+      required :template_context
+
+      def call
+        parsed = []
+        templates.each do |template|
+          parsed << Smash.new(YAML.load(Mustache.render(template, template_context)))
+        end
+
+        context.parsed_templates = parsed
+      end
+
+      private
+      def templates
+        context.templates
+      end
+
+      def template_context
+        context.template_context
+      end
+    end
+  end
+end

data/lib/shipitron/post_build.rb

@@ -0,0 +1,32 @@
+require 'shipitron'
+
+module Shipitron
+  class PostBuild < Hashie::Dash
+    property :ecs_task
+    property :container_name
+    property :command
+
+    # String is of the format:
+    # 'ecs_task:task,container_name:name,command:command
+    def self.parse(str)
+      PostBuild.new.tap do |post_build|
+        str.split(',').each do |part|
+          part.match(/([^:]+):(.+)/) do |m|
+            prop = m[1].to_sym
+            if property?(prop)
+              post_build[prop] = m[2]
+            end
+          end
+        end
+
+        properties.each do |prop|
+          raise "post build argument missing '#{prop}'" if post_build[prop].nil?
+        end
+      end
+    end
+
+    def to_s
+      "ecs_task:#{ecs_task},container_name:#{container_name},command:#{command}"
+    end
+  end
+end

data/lib/shipitron/server/deploy_application.rb

@@ -0,0 +1,75 @@
+require 'shipitron'
+require 'shipitron/consul_lock'
+require 'shipitron/server/git/pull_repo'
+require 'shipitron/server/docker/configure'
+require 'shipitron/server/docker/build_image'
+require 'shipitron/server/docker/push_image'
+require 'shipitron/server/update_ecs_task_definitions'
+require 'shipitron/server/run_post_build'
+require 'shipitron/server/update_ecs_services'
+
+module Shipitron
+  module Server
+    class DeployApplication
+      include Metaractor
+      include Interactor::Organizer
+      include ConsulLock
+
+      required :application
+      required :repository_url
+      required :s3_cache_bucket
+      required :docker_image
+      required :region
+      required :cluster_name
+      required :ecs_task_defs
+      optional :ecs_task_def_templates
+      required :ecs_services
+      optional :ecs_service_templates
+      optional :build_script
+      optional :post_builds
+      optional :repository_branch
+
+      around do |interactor|
+        if ENV['CONSUL_HOST'].nil?
+          fail_with_error!(message: 'Environment variable CONSUL_HOST required')
+        end
+
+        Diplomat.configure do |config|
+          config.url = "http://#{ENV['CONSUL_HOST']}:8500"
+        end
+
+        begin
+          with_lock(key: "shipitron/#{application}/deploy_lock") do
+            interactor.call
+          end
+        rescue UnableToLock
+          fail_with_errors!(messages: [
+            'Shipitron says: THERE CAN BE ONLY ONE',
+            'Unable to acquire deploy lock.'
+          ])
+        end
+      end
+
+      organize [
+        Git::PullRepo,
+        Docker::Configure,
+        Docker::BuildImage,
+        Docker::PushImage,
+        UpdateEcsTaskDefinitions,
+        RunPostBuild,
+        UpdateEcsServices
+      ]
+
+      def call
+        Logger.info "==> Deploying #{application} (server-side)"
+        super
+        Logger.info "==> Done"
+      end
+
+      private
+      def application
+        context.application
+      end
+    end
+  end
+end

data/lib/shipitron/server/docker/build_image.rb

@@ -0,0 +1,25 @@
+require 'shipitron'
+require 'shipitron/server/download_build_cache'
+require 'shipitron/server/docker/run_build_script'
+require 'shipitron/server/upload_build_cache'
+
+module Shipitron
+  module Server
+    module Docker
+      class BuildImage
+        include Metaractor
+        include Interactor::Organizer
+
+        required :application
+        required :docker_image
+        required :git_sha
+
+        organize [
+          DownloadBuildCache,
+          Docker::RunBuildScript,
+          UploadBuildCache
+        ]
+      end
+    end
+  end
+end

data/lib/shipitron/server/docker/configure.rb

@@ -0,0 +1,35 @@
+require 'shipitron'
+require 'shipitron/consul_keys'
+
+module Shipitron
+  module Server
+    module Docker
+      class Configure
+        include Metaractor
+        include ConsulKeys
+
+        required :application
+
+        before do
+          configure_consul_client!
+        end
+
+        def call
+          docker_auth = fetch_key!(key: "shipitron/#{application}/docker_auth")
+          auth_file = Pathname.new('/home/shipitron/.docker/config.json')
+          auth_file.parent.mkpath
+          auth_file.open('wb') do |file|
+            file.puts(docker_auth.to_s)
+            file.chmod(0600)
+          end
+        end
+
+        private
+        def application
+          context.application
+        end
+
+      end
+    end
+  end
+end

data/lib/shipitron/server/docker/push_image.rb

@@ -0,0 +1,37 @@
+require 'shipitron'
+
+module Shipitron
+  module Server
+    module Docker
+      class PushImage
+        include Metaractor
+
+        required :docker_image
+
+        def call
+          Logger.info "Pushing docker image #{docker_image} and #{docker_image.name_with_tag(:latest)}"
+
+          Logger.info `docker tag #{docker_image} #{docker_image.name_with_tag(:latest)}`
+          if $? != 0
+            fail_with_error!(message: 'Docker tag failed.')
+          end
+
+          Logger.info `docker push #{docker_image}`
+          if $? != 0
+            fail_with_error!(message: 'Docker push failed.')
+          end
+
+          Logger.info `docker push #{docker_image.name_with_tag(:latest)}`
+          if $? != 0
+            fail_with_error!(message: 'Docker push (latest) failed.')
+          end
+        end
+
+        private
+        def docker_image
+          context.docker_image
+        end
+      end
+    end
+  end
+end