drunker 0.1.0

data/drunker.gemspec

@@ -0,0 +1,33 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'drunker/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "drunker"
+  spec.version       = Drunker::VERSION
+  spec.authors       = ["Kazuma Watanabe"]
+  spec.email         = ["watassbass@gmail.com"]
+
+  spec.summary       = %q{Distributed CLI runner on AWS CodeBuild}
+  spec.description   = %q{Distributed CLI runner on AWS CodeBuild}
+  spec.homepage      = "https://github.com/wata727/drunker"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "timecop", "~> 0.8"
+
+  spec.add_runtime_dependency "thor", "~> 0.19"
+  spec.add_runtime_dependency "rubyzip", "~> 1.2"
+  spec.add_runtime_dependency "aws-sdk", "~> 2"
+  spec.add_runtime_dependency "drunker-aggregator-pretty", "~> 0.1"
+end

data/exe/drunker

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH << File.join(__dir__, "../lib")
+require "drunker"
+
+Drunker::CLI.start(ARGV)

data/lib/drunker.rb

@@ -0,0 +1,20 @@
+require "thor"
+require "zip"
+require "pathname"
+require "aws-sdk"
+require "json"
+require "logger"
+require "erb"
+require "yaml"
+
+require "drunker/version"
+require "drunker/cli"
+require "drunker/source"
+require "drunker/artifact"
+require "drunker/artifact/layer"
+require "drunker/executor"
+require "drunker/executor/iam"
+require "drunker/executor/builder"
+require "drunker/config"
+require "drunker/aggregator"
+require "drunker/aggregator/base"

data/lib/drunker/aggregator.rb

@@ -0,0 +1,9 @@
+module Drunker
+  class Aggregator
+    def self.create(config)
+      require config.aggregator.name
+      klass = Object.const_get(config.aggregator.name.split("-").map(&:capitalize).join("::"))
+      klass.new
+    end
+  end
+end

data/lib/drunker/aggregator/base.rb

@@ -0,0 +1,13 @@
+module Drunker
+  class Aggregator
+    class Base
+      def run(layers)
+        raise NotImplementedError.new("You must implement #{self.class}##{__method__}")
+      end
+
+      def exit_status(layers)
+        raise NotImplementedError.new("You must implement #{self.class}##{__method__}")
+      end
+    end
+  end
+end

data/lib/drunker/artifact.rb

@@ -0,0 +1,73 @@
+module Drunker
+  class Artifact
+    attr_reader :bucket
+    attr_reader :stdout
+    attr_reader :stderr
+    attr_reader :exit_status
+
+    def initialize(config:, logger:)
+      timestamp = Time.now.to_i.to_s
+      s3 = Aws::S3::Resource.new(client: Aws::S3::Client.new(config.aws_client_options))
+
+      @bucket = s3.create_bucket(bucket: "drunker-artifact-store-#{timestamp}")
+      logger.info("Created artifact bucket: #{bucket.name}")
+      @name = "drunker_artifact_#{timestamp}"
+      @stdout = "drunker_artifact_#{timestamp}_stdout.txt"
+      @stderr = "drunker_artifact_#{timestamp}_stderr.txt"
+      @exit_status = "drunker_artifact_#{timestamp}_exit_status.txt"
+      @builds = []
+      @logger = logger
+    end
+
+    def to_h
+      {
+        type: "S3",
+        location: bucket.name,
+        namespace_type: "BUILD_ID",
+      }
+    end
+
+    def layers
+      @layers ||= builds.each_with_object([]) do |build, layers|
+        project_name, build_id = build.split(":")
+        layers << Layer.new(build_id: build).tap do |layer|
+          begin
+            layer.stdout = fetch_content("#{build_id}/#{project_name}/#{stdout}")
+            layer.stderr = fetch_content("#{build_id}/#{project_name}/#{stderr}")
+            layer.exit_status = fetch_content("#{build_id}/#{project_name}/#{exit_status}")
+          rescue Aws::S3::Errors::NoSuchKey
+            logger.debug("Artifact not found")
+            layer.invalid!
+          end
+        end
+      end
+    end
+
+    def set_build(build)
+      @builds << build
+      logger.debug("Set build: { build: #{build}, artifact: #{name} }")
+    end
+
+    def replace_build(before:, after:)
+      builds.delete(before)
+      logger.debug("Unset build: { build: #{before}, artifact: #{name} }")
+      set_build(after)
+    end
+
+    def delete
+      bucket.delete!
+      logger.info("Deleted bucket: #{bucket.name}")
+    end
+
+    private
+
+    attr_reader :builds
+    attr_reader :name
+    attr_reader :logger
+
+    def fetch_content(object_id)
+      logger.debug("Get artifact: #{object_id}")
+      bucket.object(object_id).get.body.string
+    end
+  end
+end

data/lib/drunker/artifact/layer.rb

@@ -0,0 +1,30 @@
+module Drunker
+  class Artifact
+    class Layer
+      attr_reader :build_id
+      attr_reader :exit_status
+      attr_accessor :stdout
+      attr_accessor :stderr
+
+      def initialize(build_id:, stdout: nil, stderr: nil, exit_status: nil)
+        @build_id = build_id
+        @stdout = stdout
+        @stderr = stderr
+        @exit_status = exit_status.to_i
+        @invalid = false
+      end
+
+      def exit_status=(exit_status)
+        @exit_status = exit_status.to_i
+      end
+
+      def invalid?
+        @invalid
+      end
+
+      def invalid!
+        @invalid = true
+      end
+    end
+  end
+end

data/lib/drunker/cli.rb

@@ -0,0 +1,74 @@
+module Drunker
+  class CLI < Thor
+    desc "run [IMAGE] [COMMAND]", "Run a command on CodeBuild"
+    method_option :config, :type => :string, :default => ".drunker.yml", :desc => "Location of config file"
+    method_option :concurrency, :type => :numeric, :default => 1, :desc => "Build concurrency"
+    method_option :compute_type, :type => :string, :default => "small", :enum => %w(small medium large), :desc => "Container compute type"
+    method_option :timeout, :type => :numeric, :default => 60, :desc => "Build timeout in minutes, should be between 5 and 480"
+    method_option :env, :type => :hash, :default => {}, :desc => "Environment variables in containers"
+    method_option :buildspec, :type => :string, :desc => "Location of custom buildspec"
+    method_option :file_pattern, :type => :string, :default => "**/*", :desc => "FILES target file pattern, can use glob to specify, but files beginning with a dot are ignored."
+    method_option :aggregator, :type => :string, :default => "pretty", :desc => "Aggregator name. If you want to use custom aggregator, please install that beforehand."
+    method_option :loglevel, :type => :string, :default => "info", :enum => %w(debug info warn error fatal), :desc => "Output log level"
+    method_option :debug, :type => :boolean, :default => false, :desc => "Enable debug mode. This mode does not delete the AWS resources created by Drunker"
+    method_option :access_key, :type => :string, :desc => "AWS access key token used by Drunker"
+    method_option :secret_key, :type => :string, :desc => "AWS secret key token used by Drunker"
+    method_option :region, :type => :string, :desc => "AWS region in which resources is created by Drunker"
+    method_option :profile_name, :type => :string, :desc => "AWS shared credentials profile name used by Drunker"
+    def _run(image, *commands)
+      loglevel = options[:debug] ? "DEBUG" : options[:loglevel].upcase
+      logger = Logger.new(STDERR).tap do |logger|
+        logger.level = Logger.const_get(loglevel)
+        logger.formatter = Proc.new { |severity, _datetime, _progname, message| "#{severity}: #{message}\n" } unless loglevel == "DEBUG"
+      end
+      config = Drunker::Config.new(image: image,
+                                   commands: commands,
+                                   config: options[:config],
+                                   concurrency: options[:concurrency],
+                                   compute_type: options[:compute_type],
+                                   timeout: options[:timeout],
+                                   env: options[:env],
+                                   buildspec: options[:buildspec],
+                                   file_pattern: options[:file_pattern],
+                                   aggregator: options[:aggregator],
+                                   debug: options[:debug],
+                                   access_key: options[:access_key],
+                                   secret_key: options[:secret_key],
+                                   region: options[:region],
+                                   profile_name: options[:profile_name],
+                                   logger: logger)
+
+      logger.info("Creating source....")
+      source = Drunker::Source.new(Pathname.pwd, config: config, logger: logger)
+
+      logger.info("Starting executor...")
+      artifact = Drunker::Executor.new(source: source, config: config, logger: logger).run
+
+      logger.info("Starting aggregator...")
+      aggregator = Drunker::Aggregator.create(config)
+      aggregator.run(artifact.layers)
+
+      unless config.debug?
+        logger.info("Deleting source...")
+        source.delete
+        logger.info("Deleting artifact...")
+        artifact.delete
+      end
+
+      exit aggregator.exit_status(artifact.layers)
+    rescue Drunker::Config::InvalidConfigException => exn
+      logger.error(exn.message)
+      exit 1
+    end
+    map "run" => "_run" # "run" is a Thor reserved word and cannot be defined as command
+
+    desc "version", "Show version"
+    def version
+      puts "Drunker #{VERSION}"
+    end
+
+    def self.exit_on_failure?
+      true
+    end
+  end
+end

data/lib/drunker/config.rb

@@ -0,0 +1,160 @@
+module Drunker
+  class Config
+    attr_reader :image
+    attr_reader :commands
+    attr_reader :concurrency
+    attr_reader :compute_type
+    attr_reader :timeout
+    attr_reader :environment_variables
+    attr_reader :buildspec
+    attr_reader :file_pattern
+    attr_reader :aggregator
+
+    class InvalidConfigException < StandardError; end
+
+    def initialize(image:,
+                   commands:,
+                   config:,
+                   concurrency:,
+                   compute_type:,
+                   timeout:,
+                   env:,
+                   buildspec:,
+                   file_pattern:,
+                   aggregator:,
+                   access_key:,
+                   secret_key:,
+                   region:,
+                   profile_name:,
+                   debug:,
+                   logger:)
+      @logger = logger
+      yaml = load!(config)
+
+      @image = image
+      @commands = commands
+      @concurrency = yaml["concurrency"] || concurrency
+      @compute_type = compute_name[ yaml["compute_type"] || compute_type ]
+      @timeout =  yaml["timeout"] || timeout
+      @environment_variables = codebuild_environments_format(yaml["environment_variables"] || env)
+      @buildspec = buildspec_body!(yaml["buildspec"] || buildspec)
+      @file_pattern = yaml["file_pattern"] || file_pattern
+      @aggregator = aggregator_gem!(yaml["aggregator"] || aggregator)
+      @credentials = aws_credentials(profile_name: yaml.dig("aws_credentials", "profile_name") || profile_name,
+                                     access_key: yaml.dig("aws_credentials", "access_key") || access_key,
+                                     secret_key: yaml.dig("aws_credentials", "secret_key") || secret_key)
+      @region = yaml.dig("aws_credentials", "region") || region
+      @debug = debug
+
+      validate!
+    end
+
+    def debug?
+      debug
+    end
+
+    def aws_client_options
+      { credentials: credentials, region: region }.delete_if { |_k, v| v.nil? }
+    end
+
+    private
+
+    attr_reader :credentials
+    attr_reader :region
+    attr_reader :debug
+    attr_reader :logger
+
+    def compute_name
+      {
+        "small" => "BUILD_GENERAL1_SMALL",
+        "medium" => "BUILD_GENERAL1_MEDIUM",
+        "large" => "BUILD_GENERAL1_LARGE"
+      }
+    end
+
+    def codebuild_environments_format(env)
+      env.map { |k, v| { name: k, value: v } }
+    end
+
+    def buildspec_body!(buildspec)
+      if buildspec
+        buildspec.is_a?(Hash) ? buildspec.to_yaml : Pathname.new(buildspec).read
+      else
+        Pathname.new(__dir__ + "/executor/buildspec.yml.erb").read
+      end
+    rescue Errno::ENOENT
+      raise InvalidConfigException.new("Invalid location of custom buildspec. got: #{buildspec}")
+    end
+
+    def aws_credentials(profile_name:, access_key:, secret_key:)
+      if profile_name
+        Aws::SharedCredentials.new(profile_name: profile_name)
+      elsif access_key && secret_key
+        Aws::Credentials.new(access_key, secret_key)
+      end
+    end
+
+    def aggregator_gem!(name)
+      gem = Gem::Specification.select { |gem| gem.name == "drunker-aggregator-#{name}" }.max { |a, b| a.version <=> b.version }
+      raise InvalidConfigException.new("Invalid aggregator. `drunker-aggregator-#{name}` is already installed?") unless gem
+      gem
+    end
+
+    def load!(config)
+      yaml = YAML.load_file(config)
+      validate_yaml!(yaml)
+      yaml
+    rescue Errno::ENOENT
+      if config == ".drunker.yml"
+        logger.debug("Config file not found. But it ignored because this is default config file.")
+        {}
+      else
+        raise InvalidConfigException.new("Config file not found. got: #{config}")
+      end
+    rescue Psych::SyntaxError => exn
+      raise InvalidConfigException.new("Invalid config file. message: #{exn.message}")
+    end
+
+    def validate_yaml!(yaml)
+      valid_toplevel_keys = %w(concurrency compute_type timeout file_pattern environment_variables buildspec aggregator aws_credentials)
+      invalid_keys = yaml.keys.reject { |k| valid_toplevel_keys.include?(k) }
+      raise InvalidConfigException.new("Invalid config file keys: #{invalid_keys.join(",")}") unless invalid_keys.empty?
+
+      if yaml["aws_credentials"]
+        valid_aws_credentials_keys = %w(profile_name access_key secret_key region)
+        invalid_keys = yaml["aws_credentials"].keys.reject { |k| valid_aws_credentials_keys.include?(k) }
+        raise InvalidConfigException.new("Invalid config file keys: #{invalid_keys.join(",")}") unless invalid_keys.empty?
+      end
+
+      message = case
+                when yaml["concurrency"] && !yaml["concurrency"].is_a?(Numeric)
+                  "Invalid concurrency. It should be number (Not string). got: #{yaml["concurrency"]}"
+                when yaml["compute_type"] && !%w(small medium large).include?(yaml["compute_type"])
+                  "Invalid compute type. It should be one of small, medium, large. got: #{yaml["compute_type"]}"
+                when yaml["timeout"] && !yaml["timeout"].is_a?(Numeric)
+                  "Invalid timeout. It should be number (Not string). got: #{yaml["timeout"]}"
+                when yaml["buildspec"] && !(yaml["buildspec"].is_a?(String) || yaml["buildspec"].is_a?(Hash))
+                  "Invalid buildspec. It should be string or hash. got: #{yaml["buildspec"]}"
+                when yaml["environment_variables"] && !yaml["environment_variables"]&.values.all? { |v| v.is_a?(String) || v.is_a?(Numeric) }
+                  "Invalid environment variables. It should be flatten hash. got: #{yaml["environment_variables"]}"
+                when yaml["file_pattern"] && !yaml["file_pattern"].is_a?(String)
+                  "Invalid file pattern. It should be string. got: #{yaml["file_pattern"]}"
+                when yaml["aggregator"] && !yaml["aggregator"].is_a?(String)
+                  "Invalid aggregator. It should be string. got: #{yaml["aggregator"]}"
+                end
+
+      raise InvalidConfigException.new(message) if message
+    end
+
+    def validate!
+      message = case
+                when concurrency <= 0
+                  "Invalid concurrency. It should be bigger than 0. got: #{concurrency}"
+                when !timeout.between?(5, 480)
+                  "Invalid timeout range. It should be 5 and 480. got: #{timeout}"
+                end
+
+      raise InvalidConfigException.new(message) if message
+    end
+  end
+end

data/lib/drunker/executor.rb

@@ -0,0 +1,122 @@
+module Drunker
+  class Executor
+    RETRY_LIMIT = 10
+
+    def initialize(source:, config:, logger:)
+      @project_name = "drunker-executor-#{Time.now.to_i.to_s}"
+      @source = source
+      logger.info("Creating artifact...")
+      @artifact = Drunker::Artifact.new(config: config, logger: logger)
+      @config = config
+      @client = Aws::CodeBuild::Client.new(config.aws_client_options)
+      @builders = []
+      @logger = logger
+    end
+
+    def run
+      setup_project do
+        @builders = parallel_build
+
+        loop do
+          builders.select(&:access_denied?).each do |builder|
+            failed_id = builder.build_id
+            if builder.retriable?
+              build_id = builder.retry
+              artifact.replace_build(before: failed_id ,after: build_id)
+            end
+          end
+
+          running, finished = builders.partition(&:running?)
+          finished.select(&:failed?).each do |failed|
+            failed.errors.each do |error|
+              logger.warn("Build failed: #{failed.build_id}")
+              logger.warn("\tphase_type: #{error[:phase_type]}")
+              logger.warn("\tphase_status: #{error[:phase_status]}")
+              logger.warn("\tstatus: #{error[:status]}")
+              logger.warn("\tmessage: #{error[:message]}")
+            end
+          end
+
+          break if running.count.zero?
+          logger.info("Waiting builder: #{finished.count}/#{builders.count}")
+          sleep 5
+          builders.each(&:refresh)
+        end
+        logger.info("Build is completed!")
+        artifact.layers # load artifact layers from S3
+      end
+
+      artifact
+    end
+
+    private
+
+    attr_reader :project_name
+    attr_reader :source
+    attr_reader :artifact
+    attr_reader :config
+    attr_reader :client
+    attr_reader :builders
+    attr_reader :logger
+
+    def setup_project
+      logger.info("Creating IAM resources...")
+      iam = IAM.new(source: source, artifact: artifact, config: config, logger: logger)
+      retry_count = 0
+
+      logger.info("Creating project...")
+      project_info = {
+        name: project_name,
+        source: source.to_h,
+        artifacts: artifact.to_h,
+        environment: {
+          type: "LINUX_CONTAINER",
+          image: config.image,
+          compute_type: config.compute_type,
+        },
+        service_role: iam.role.name,
+        timeout_in_minutes: config.timeout,
+      }
+      project_info[:environment][:environment_variables] = config.environment_variables unless config.environment_variables.empty?
+      begin
+        client.create_project(project_info)
+        logger.info("Created project: #{project_name}")
+      # Sometimes `CodeBuild is not authorized to perform: sts:AssumeRole` error occurs...
+      # We can solve this problem by retrying after a while.
+      rescue Aws::CodeBuild::Errors::InvalidInputException
+        if retry_count < RETRY_LIMIT
+          retry_count += 1
+          sleep 5
+          logger.info("Retrying...")
+          retry
+        else
+          raise
+        end
+      end
+
+      yield
+
+      unless config.debug?
+        logger.info("Deleting IAM resources...")
+        iam.delete
+        client.delete_project(name: project_name)
+        logger.info("Deleted project: #{project_name}")
+      end
+    end
+
+    def parallel_build
+      builders = []
+
+      files_list = source.target_files.each_slice(source.target_files.count.quo(config.concurrency).ceil).to_a
+      logger.info("Start parallel build: { files: #{source.target_files.count}, concurrency: #{config.concurrency}, real_concurrency: #{files_list.count} }")
+      files_list.to_a.each do |files|
+        builder = Builder.new(project_name: project_name, targets: files, artifact: artifact, config: config, logger: logger)
+        build_id = builder.run
+        artifact.set_build(build_id)
+        builders << builder
+      end
+
+      builders
+    end
+  end
+end