shipitron 0.1.0 → 0.2.0

data/lib/shipitron/server/docker/run_build_script.rb

@@ -0,0 +1,53 @@
+require 'shipitron'
+
+module Shipitron
+  module Server
+    module Docker
+      class RunBuildScript
+        include Metaractor
+
+        required :application
+        required :docker_image
+        required :git_sha
+        optional :build_script
+
+        before do
+          context.build_script ||= 'shipitron/build.sh'
+        end
+
+        def call
+          Logger.info 'Building docker image'
+
+          docker_image.tag = git_sha
+
+          FileUtils.cd("/home/shipitron/#{application}") do
+            unless Pathname.new(build_script).exist?
+              fail_with_error!(message: "#{build_script} does not exist")
+            end
+            Logger.info `#{build_script} #{docker_image}`
+            if $? != 0
+              fail_with_error!(message: "build script exited with non-zero code: #{$?}")
+            end
+          end
+        end
+
+        private
+        def application
+          context.application
+        end
+
+        def docker_image
+          context.docker_image
+        end
+
+        def git_sha
+          context.git_sha
+        end
+
+        def build_script
+          context.build_script
+        end
+      end
+    end
+  end
+end

data/lib/shipitron/server/download_build_cache.rb

@@ -0,0 +1,44 @@
+require 'shipitron'
+require 'shipitron/fetch_bucket'
+
+module Shipitron
+  module Server
+    class DownloadBuildCache
+      include Metaractor
+
+      required :application
+      required :s3_cache_bucket
+
+      def call
+        Logger.info "Downloading build cache from bucket #{s3_cache_bucket}"
+
+        s3_file = bucket.files.get("#{application}.build-cache.tar.gz")
+        if s3_file.nil?
+          Logger.warn 'Build cache not found.'
+          return
+        end
+
+        build_cache = Pathname.new("/home/shipitron/#{application}/tmp/build-cache.tar.gz")
+        build_cache.parent.mkpath
+        build_cache.open('wb') do |local_file|
+          local_file.write(s3_file.body)
+        end
+
+        Logger.info 'Download complete.'
+      end
+
+      private
+      def application
+        context.application
+      end
+
+      def s3_cache_bucket
+        context.s3_cache_bucket
+      end
+
+      def bucket
+        @bucket ||= FetchBucket.call!(name: s3_cache_bucket).bucket
+      end
+    end
+  end
+end

data/lib/shipitron/server/ecs_task_defs/map_parsed_templates.rb

@@ -0,0 +1,31 @@
+require 'shipitron'
+
+module Shipitron
+  module Server
+    module EcsTaskDefs
+      class MapParsedTemplates
+        include Metaractor
+
+        required :ecs_task_defs
+        required :parsed_templates
+
+        def call
+          parsed_templates.each do |parsed|
+            task_def = ecs_task_defs.find {|t| t.name == parsed.family }
+            next if task_def.nil?
+            task_def.params = parsed
+          end
+        end
+
+        private
+        def ecs_task_defs
+          context.ecs_task_defs
+        end
+
+        def parsed_templates
+          context.parsed_templates
+        end
+      end
+    end
+  end
+end

data/lib/shipitron/server/ecs_task_defs/update_from_params.rb

@@ -0,0 +1,42 @@
+require 'shipitron'
+require 'shipitron/ecs_client'
+
+module Shipitron
+  module Server
+    module EcsTaskDefs
+      class UpdateFromParams
+        include Metaractor
+        include EcsClient
+
+        required :region
+        required :ecs_task_defs
+
+        def call
+          ecs_task_defs.each do |ecs_task_def|
+            next if ecs_task_def.params.nil?
+
+            ecs_task_def.revision = ecs_client(region: region).register_task_definition(
+              ecs_task_def.params.to_h
+            ).task_definition.revision
+
+            Logger.info "Created task definition #{ecs_task_def}"
+          end
+        rescue Aws::ECS::Errors::ServiceError => e
+          fail_with_errors!(messages: [
+            "Error: #{e.message}",
+            e.backtrace.join("\n")
+          ])
+        end
+
+        private
+        def region
+          context.region
+        end
+
+        def ecs_task_defs
+          context.ecs_task_defs
+        end
+      end
+    end
+  end
+end

data/lib/shipitron/server/ecs_task_defs/update_in_place.rb

@@ -0,0 +1,68 @@
+require 'shipitron'
+require 'shipitron/ecs_client'
+
+module Shipitron
+  module Server
+    module EcsTaskDefs
+      class UpdateInPlace
+        include Metaractor
+        include EcsClient
+
+        required :region
+        required :ecs_task_defs
+
+        def call
+          ecs_task_defs.each do |ecs_task_def|
+            next if ecs_task_def.params != nil
+
+            existing_task = ecs_client(region: region).describe_task_definition(
+              task_definition: ecs_task_def.name
+            ).task_definition
+
+            updated_image = false
+            existing_task.container_definitions.each do |container_def|
+              container_def.image.match(/([^:]+)(?::.+)?/) do |m|
+                if m[1] == docker_image.name
+                  container_def.image = docker_image.name_with_tag
+                  updated_image = true
+                end
+              end
+            end
+
+            unless updated_image
+              fail_with_error!(
+                message: "Unable to update ECS task definition; #{docker_image.name} not found in task family #{ecs_task_def.name}."
+              )
+            end
+
+            existing_task = existing_task.to_h
+
+            ecs_task_def.revision = ecs_client(region: region).register_task_definition(
+              [
+                :family,
+                :container_definitions,
+                :volumes
+              ].each_with_object({}) { |k, hash| hash[k] = existing_task[k] if existing_task.has_key?(k) }
+            ).task_definition.revision
+
+            Logger.info "Created task definition #{ecs_task_def}"
+          end
+        rescue Aws::ECS::Errors::ServiceError => e
+          fail_with_errors!(messages: [
+            "Error: #{e.message}",
+            e.backtrace.join("\n")
+          ])
+        end
+
+        private
+        def region
+          context.region
+        end
+
+        def ecs_task_defs
+          context.ecs_task_defs
+        end
+      end
+    end
+  end
+end

data/lib/shipitron/server/git/clone_local_copy.rb

@@ -0,0 +1,46 @@
+require 'shipitron'
+require 'shellwords'
+
+module Shipitron
+  module Server
+    module Git
+      class CloneLocalCopy
+        include Metaractor
+
+        required :application
+        required :repository_url
+        optional :repository_branch
+
+        before do
+          context.repository_branch ||= 'master'
+        end
+
+        def call
+          Logger.info "Using this branch: #{repository_branch}"
+          FileUtils.cd('/home/shipitron') do
+            `git clone git-cache #{Shellwords.escape application} --recursive --branch #{Shellwords.escape repository_branch}`
+          end
+
+          Logger.info 'Using this git commit:'
+          FileUtils.cd("/home/shipitron/#{application}") do
+            context.git_sha = `git rev-parse --short=12 HEAD`.chomp
+            Logger.info `git --no-pager log --format='%aN (%h): %s' -n 1`.chomp
+          end
+        end
+
+        private
+        def application
+          context.application
+        end
+
+        def repository_url
+          context.repository_url
+        end
+
+        def repository_branch
+          context.repository_branch
+        end
+      end
+    end
+  end
+end

data/lib/shipitron/server/git/configure.rb

@@ -0,0 +1,40 @@
+require 'shipitron'
+require 'shipitron/consul_keys'
+
+module Shipitron
+  module Server
+    module Git
+      class Configure
+        include Metaractor
+        include ConsulKeys
+
+        required :application
+        required :repository_url
+        required :s3_cache_bucket
+
+        before do
+          configure_consul_client!
+        end
+
+        def call
+          host_key = fetch_key!(key: "shipitron/#{application}/git_host_key")
+          Pathname.new('/home/shipitron/.ssh/known_hosts').open('a') do |file|
+            file.puts(host_key.to_s)
+          end
+
+          deploy_key = fetch_key!(key: "shipitron/#{application}/git_deploy_key")
+          Pathname.new('/home/shipitron/.ssh/id_rsa').open('w') do |file|
+            file.puts(deploy_key.to_s)
+            file.chmod(0600)
+          end
+        end
+
+        private
+        def application
+          context.application
+        end
+
+      end
+    end
+  end
+end

data/lib/shipitron/server/git/download_cache.rb

@@ -0,0 +1,56 @@
+require 'shipitron'
+require 'shipitron/fetch_bucket'
+require 'archive/tar/minitar'
+
+module Shipitron
+  module Server
+    module Git
+      class DownloadCache
+        include Metaractor
+
+        required :application
+        required :s3_cache_bucket
+
+        def call
+          Logger.info "Downloading git cache from bucket #{s3_cache_bucket}"
+
+          s3_file = bucket.files.get("#{application}.git.tar.gz")
+          if s3_file.nil?
+            Logger.warn 'Git cache not found.'
+            return
+          end
+
+          Pathname.new("/tmp/#{application}.git.tar.gz").open('wb') do |local_file|
+            local_file.write(s3_file.body)
+          end
+
+          extract_tarball(filename: "/tmp/#{application}.git.tar.gz", directory: '/home/shipitron')
+
+          Logger.info 'Download complete.'
+        end
+
+        private
+        def application
+          context.application
+        end
+
+        def s3_cache_bucket
+          context.s3_cache_bucket
+        end
+
+        def bucket
+          @bucket ||= FetchBucket.call!(name: s3_cache_bucket).bucket
+        end
+
+        def extract_tarball(filename:, directory:)
+          Pathname.new(filename).open('rb') do |tarball|
+            Zlib::GzipReader.wrap(tarball) do |gz|
+              Archive::Tar::Minitar.unpack(gz, directory)
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/shipitron/server/git/pull_repo.rb

@@ -0,0 +1,30 @@
+require 'shipitron'
+require 'shipitron/server/git/configure'
+require 'shipitron/server/git/download_cache'
+require 'shipitron/server/git/update_cache'
+require 'shipitron/server/git/upload_cache'
+require 'shipitron/server/git/clone_local_copy'
+
+module Shipitron
+  module Server
+    module Git
+      class PullRepo
+        include Metaractor
+        include Interactor::Organizer
+
+        required :application
+        required :repository_url
+        required :s3_cache_bucket
+        optional :repository_branch
+
+        organize [
+          Configure,
+          DownloadCache,
+          UpdateCache,
+          UploadCache,
+          CloneLocalCopy
+        ]
+      end
+    end
+  end
+end

data/lib/shipitron/server/git/update_cache.rb

@@ -0,0 +1,52 @@
+require 'shipitron'
+require 'shellwords'
+
+module Shipitron
+  module Server
+    module Git
+      class UpdateCache
+        include Metaractor
+
+        required :application
+        required :repository_url
+        required :s3_cache_bucket
+        optional :repository_branch
+
+        before do
+          context.repository_branch ||= 'master'
+        end
+
+        def call
+          if !Pathname.new('/home/shipitron/git-cache/objects').directory?
+            Logger.info 'Cloning the git repository'
+            FileUtils.cd('/home/shipitron') do
+              `git clone --bare #{Shellwords.escape repository_url} git-cache`
+            end
+          else
+            Logger.info 'Fetching new git commits'
+            FileUtils.cd('/home/shipitron/git-cache') do
+              `git fetch #{Shellwords.escape repository_url} #{Shellwords.escape repository_branch}:#{Shellwords.escape repository_branch}`
+            end
+          end
+        end
+
+        private
+        def application
+          context.application
+        end
+
+        def repository_url
+          context.repository_url
+        end
+
+        def s3_cache_bucket
+          context.s3_cache_bucket
+        end
+
+        def repository_branch
+          context.repository_branch
+        end
+      end
+    end
+  end
+end

data/lib/shipitron/server/git/upload_cache.rb

@@ -0,0 +1,61 @@
+require 'shipitron'
+require 'shipitron/fetch_bucket'
+require 'find'
+require 'archive/tar/minitar'
+
+module Shipitron
+  module Server
+    module Git
+      class UploadCache
+        include Metaractor
+
+        required :application
+        required :s3_cache_bucket
+
+        def call
+          Logger.info "Uploading git cache to bucket #{s3_cache_bucket}"
+          create_tarball(filename: "/tmp/#{application}.git.tar.gz", directory: '/home/shipitron/git-cache')
+
+          Pathname.new("/tmp/#{application}.git.tar.gz").open('rb') do |local_file|
+            bucket.files.create(
+              key: "#{application}.git.tar.gz",
+              body: local_file.read
+            )
+          end
+        end
+
+        private
+        def application
+          context.application
+        end
+
+        def s3_cache_bucket
+          context.s3_cache_bucket
+        end
+
+        def create_tarball(filename:, directory:)
+          base_dir = Pathname.new(directory).parent
+          Logger.debug 'Creating tarball'
+          FileUtils.cd(base_dir) do
+            Pathname.new(filename).open('wb') do |tarball|
+              Zlib::GzipWriter.wrap(tarball) do |gz|
+                Archive::Tar::Minitar::Output.open(gz) do |tar|
+                  Find.find(directory) do |path|
+                    pn = Pathname.new(path)
+                    name = pn.relative_path_from(base_dir)
+                    Logger.debug(name)
+                    Archive::Tar::Minitar.pack_file(name.to_s, tar)
+                  end
+                end
+              end
+            end
+          end
+        end
+
+        def bucket
+          @bucket ||= FetchBucket.call!(name: s3_cache_bucket).bucket
+        end
+      end
+    end
+  end
+end

data/lib/shipitron/server/run_post_build.rb

@@ -0,0 +1,79 @@
+require 'shipitron'
+require 'shipitron/ecs_client'
+
+module Shipitron
+  module Server
+    class RunPostBuild
+      include Metaractor
+      include EcsClient
+
+      required :region
+      required :cluster_name
+      optional :post_builds
+
+      def call
+        return if post_builds.nil? || post_builds.empty?
+
+        Logger.info 'Running post build commands'
+
+        begin
+          post_builds.each do |post_build|
+            Logger.info "Running #{post_build.command}"
+            response = ecs_client(region: region).run_task(
+              cluster: cluster_name,
+              task_definition: post_build.ecs_task,
+              overrides: {
+                container_overrides: [
+                  {
+                    name: post_build.container_name,
+                    command: [post_build.command]
+                  }
+                ]
+              },
+              count: 1,
+              started_by: 'shipitron'
+            )
+
+            if !response.failures.empty?
+              response.failures.each do |failure|
+                fail_with_error! message: "ECS run_task failure: #{failure.arn}: #{failure.reason}"
+              end
+            end
+
+            task_arn = response.tasks.first.task_arn
+
+            Logger.info 'Waiting for task to finish'
+            loop do
+              response = ecs_client(region: region).describe_tasks(
+                cluster: cluster_name,
+                tasks: [task_arn]
+              )
+              Logger.info "Task status: #{response.tasks.first.last_status}"
+              break if response.tasks.first.last_status == 'STOPPED'.freeze
+              sleep 1
+            end
+          end
+
+        rescue Aws::ECS::Errors::ServiceError => e
+          fail_with_errors!(messages: [
+            "Error: #{e.message}",
+            e.backtrace.join("\n")
+          ])
+        end
+      end
+
+      private
+      def post_builds
+        context.post_builds
+      end
+
+      def region
+        context.region
+      end
+
+      def cluster_name
+        context.cluster_name
+      end
+    end
+  end
+end

data/lib/shipitron/server/transform_cli_args.rb

@@ -0,0 +1,85 @@
+require 'shipitron'
+require 'shipitron/docker_image'
+require 'shipitron/ecs_task_def'
+require 'shipitron/post_build'
+require 'base64'
+
+module Shipitron
+  module Server
+    class TransformCliArgs
+      include Metaractor
+
+      required :application
+      required :repository_url
+      optional :repository_branch
+      required :s3_cache_bucket
+      required :image_name
+      required :region
+      required :cluster_name
+      required :ecs_task_defs
+      optional :ecs_task_def_templates
+      required :ecs_services
+      optional :ecs_service_templates
+      optional :build_script
+      optional :post_builds
+
+      before do
+        context.ecs_task_def_templates ||= []
+        context.ecs_service_templates ||= []
+      end
+
+      def call
+        cli_args = Smash.new
+
+        %i[
+          application
+          repository_url
+          repository_branch
+          s3_cache_bucket
+          region
+          cluster_name
+          ecs_services
+          build_script
+        ].each_with_object(cli_args) { |k, args| args[k] = context[k] }
+
+        cli_args.docker_image = DockerImage.new(name: context.image_name)
+
+        cli_args.ecs_task_defs = []
+        ecs_task_defs.each do |task_def|
+          cli_args.ecs_task_defs << EcsTaskDef.new(name: task_def)
+        end
+
+        cli_args.ecs_task_def_templates = []
+        context.ecs_task_def_templates.each do |template|
+          cli_args.ecs_task_def_templates << Base64.urlsafe_decode64(template)
+        end
+
+        cli_args.ecs_service_templates = []
+        context.ecs_service_templates.each do |template|
+          cli_args.ecs_service_templates << Base64.urlsafe_decode64(template)
+        end
+
+        Logger.debug "task_def_templates: #{cli_args.ecs_task_def_templates}"
+        Logger.debug "service_templates: #{cli_args.ecs_service_templates}"
+
+        if post_builds != nil && !post_builds.empty?
+          cli_args.post_builds = []
+          post_builds.each do |post_build_str|
+            cli_args.post_builds << PostBuild.parse(post_build_str)
+          end
+        end
+
+        context.cli_args = cli_args
+      end
+
+      private
+      def ecs_task_defs
+        context.ecs_task_defs
+      end
+
+      def post_builds
+        context.post_builds
+      end
+    end
+  end
+end