wrapbox 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c76da62eee815c3ab4d57a567a8a83810d9b7f20
-  data.tar.gz: 4c3df8f6ce34d0ae48f476d32e57d85170ce0028
+  metadata.gz: edc9210efdc2d7248e0b6eab05f04c89f7223b90
+  data.tar.gz: afe62fcbfbf5f5383b12e476709fd7d79a36af27
 SHA512:
-  metadata.gz: c2b799ee4e165c93dcf829659e513d610ad07f666377b8c3631c63209175bee00f7431b233dec3615cbd6dcba509895b4f4f7fa1e21b6b5174e7eb68d1673b22
-  data.tar.gz: 600bd6d4359e2534bf0c047621d61b2a7df7d5bcbe9143eb84e465b081f6714744027c9cf2ff989615c468cb6893b664b21b3aa6fb61322f558e71cc1048e0f1
+  metadata.gz: 8f856b7dc58a1776c662f4bd43d1165c043c4ec8a2e5ad0a12eac1b4c60a1162d9ef57541c1d54373471f59879f6257cd8ae1105c1b26e142ffc790839694f75
+  data.tar.gz: 0cc10bf941d6b73d320581ae722383c888f5cac1fe700bfccfd5b5ab117abd3b1347abb5f67229d8048d1ff784768224122dd8c767f7188888489b1877a0e559

data/Dockerfile

@@ -1,7 +1,7 @@
 FROM ruby:alpine
 
-RUN apk add --no-cache git
+RUN apk add --no-cache git gcc make g++ zlib-dev
 WORKDIR /app
 COPY . /app/
 
-RUN bundle install
+RUN bundle install -j3

data/exe/wrapbox

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require "wrapbox/cli"
+
+Wrapbox::Cli.start

data/lib/wrapbox/cli.rb

@@ -0,0 +1,10 @@
+require "thor/group"
+require "wrapbox/runner/docker"
+require "wrapbox/runner/ecs"
+
+module Wrapbox
+  class Cli < Thor
+    register(Wrapbox::Runner::Ecs::Cli, "ecs", "ecs [COMMAND]", "Commands for ECS")
+    register(Wrapbox::Runner::Docker::Cli, "docker", "docker [COMMAND]", "Commands for Docker")
+  end
+end

data/lib/wrapbox/config_repository.rb

@@ -1,4 +1,5 @@
 require 'yaml'
+require 'wrapbox/configuration'
 
 module Wrapbox
   class ConfigRepository

data/lib/wrapbox/configuration.rb

@@ -4,26 +4,32 @@ require "active_support/core_ext/string"
 module Wrapbox
   Configuration = Struct.new(
     :name,
+    :revision,
     :runner,
     :cluster,
     :region,
+    :retry,
+    :retry_interval,
+    :retry_interval_multiplier,
     :container_definition,
     :additional_container_definitions,
     :task_role_arn,
-    :use_sudo,
-    :rm
+    :keep_container
   ) do
     def self.load_config(config)
       new(
         config["name"],
+        config["revision"],
         config["runner"] ? config["runner"].to_sym : :docker,
         config["cluster"],
         config["region"],
+        config["retry"] || 0,
+        config["retry_interval"] || 1,
+        config["retry_interval_multiplier"] || 2,
         config["container_definition"].deep_symbolize_keys,
         config["additional_container_definitions"] || [],
         config["task_role_arn"],
-        config["use_sudo"].nil? ? false : config["use_sudo"],
-        config["rm"].nil? ? false : config["rm"]
+        config["keep_container"]
       )
     end
 
@@ -31,11 +37,11 @@ module Wrapbox
 
     def initialize(*args)
       super
-      raise "#{runner} is unsupported runner" unless AVAILABLE_RUNNERS.include?(runner)
-      require "wrapbox/runner/#{runner}"
     end
 
     def build_runner
+      raise "#{runner} is unsupported runner" unless AVAILABLE_RUNNERS.include?(runner.to_sym)
+      require "wrapbox/runner/#{runner}"
       Wrapbox::Runner.const_get(runner.to_s.camelcase).new(to_h)
     end
 

data/lib/wrapbox/runner/docker.rb

@@ -1,5 +1,7 @@
 require "open3"
 require "multi_json"
+require "docker"
+require "thor"
 
 module Wrapbox
   module Runner
@@ -9,86 +11,105 @@ module Wrapbox
       attr_reader \
         :name,
         :container_definition,
-        :rm,
-        :use_sudo
+        :keep_container
 
       def initialize(options)
         @name = options[:name]
         @container_definition = options[:container_definition]
-        @rm = options[:rm]
-        @use_sudo = options[:use_sudo]
+        @keep_container = options[:keep_container]
       end
 
       def run(class_name, method_name, args, container_definition_overrides: {}, environments: [])
         definition = container_definition
           .merge(container_definition_overrides)
 
-        cmdopt = build_cmdopt(definition)
-        cmdopt.concat(base_environments(class_name, method_name, args))
-        cmdopt.concat(extract_environments(environments))
-        cmdopt.concat([definition[:image], "bundle", "exec", "rake", "wrapbox:run"])
+        envs = base_environments(class_name, method_name, args)
+        envs.concat(extract_environments(environments))
 
-        exec_docker(*cmdopt)
+        exec_docker(definition: definition, cmd: ["bundle", "exec", "rake", "wrapbox:run"], environments: envs)
       end
 
       def run_cmd(*cmd,  container_definition_overrides: {}, environments: [])
         definition = container_definition
           .merge(container_definition_overrides)
 
-        cmdopt = build_cmdopt(definition)
-        cmdopt.concat(extract_environments(environments))
-        cmdopt.concat([definition[:image], *cmd])
+        environments = extract_environments(environments)
 
-        exec_docker(*cmdopt)
+        exec_docker(definition: definition, cmd: cmd, environments: environments)
       end
 
       private
 
-      def build_cmdopt(definition)
-        cmdopt = ["run"]
-        cmdopt.concat(["--rm"]) if rm
-        cmdopt.concat(["--cpu-shares", definition[:cpu].to_s]) if definition[:cpu]
-        cmdopt.concat(["--memory", "#{definition[:memory]}m"]) if definition[:memory]
-        cmdopt.concat(["--memory-reservation", "#{definition[:memory_reservation]}m"]) if definition[:memory_reservation]
-        cmdopt
-      end
-
       def base_environments(class_name, method_name, args)
-        ["-e", "#{CLASS_NAME_ENV}=#{class_name}", "-e", "#{METHOD_NAME_ENV}=#{method_name}", "-e", "#{METHOD_ARGS_ENV}=#{MultiJson.dump(args)}"]
+        ["#{CLASS_NAME_ENV}=#{class_name}", "#{METHOD_NAME_ENV}=#{method_name}", "#{METHOD_ARGS_ENV}=#{MultiJson.dump(args)}"]
       end
 
       def extract_environments(environments)
-        environments.flat_map do |e|
-          ["-e", "#{e[:name]}=#{e[:value]}"]
+        environments.map do |e|
+          "#{e[:name]}=#{e[:value]}"
+        end
+      end
+
+      def exec_docker(definition:, cmd:, environments: [])
+        ::Docker::Image.create("fromImage" => definition[:image])
+        options = {
+          "Image" => definition[:image],
+          "Cmd" => cmd,
+          "Env" => environments,
+        }
+        options["HostConfig"] = {}
+        options["HostConfig"]["Cpu"] = definition[:cpu] if definition[:cpu]
+        options["HostConfig"]["Memory"] = definition[:memory] * 1024 * 1024 if definition[:memory]
+        options["HostConfig"]["MemoryReservation"] = definition[:memory_reservation] * 1024 * 1024 if definition[:memory_reservation]
+        options["HostConfig"]["Links"] = definition[:links]
+        options["Entrypoint"] = definition[:entry_point] if definition[:entry_point]
+        options["WorkingDir"] = definition[:working_directory] if definition[:working_directory]
+
+        container = ::Docker::Container.create(options)
+
+        container.start!
+        output_container_logs(container)
+        resp = container.wait
+        output_container_logs(container)
+
+        unless resp["StatusCode"].zero?
+          raise ExecutionError, "exit_code=#{resp["StatusCode"]}"
         end
+      ensure
+        container.remove(force: true) if container && !keep_container
       end
 
-      def exec_docker(*args)
-        cmd = use_sudo ? "sudo docker" : "docker"
-
-        result = Open3.popen3(cmd, *args) do |stdin, stdout, stderr, wait_thr|
-          stdin.close_write
-          begin
-            loop do
-              rs, _ = IO.select([stdout, stderr])
-              rs.each do |io|
-                io.each do |line|
-                  next if line.nil? || line.empty?
-                  if io == stdout
-                    $stdout.puts(line)
-                  else
-                    $stderr.puts(line)
-                  end
-                end
-              end
-              break if stdout.eof? && stderr.eof?
-            end
-          rescue EOFError
+      def output_container_logs(container)
+        container.streaming_logs(stdout: true, stderr: true) do |stream, chunk|
+          if stream == "stdout"
+            $stdout.puts(chunk)
+          else
+            $stderr.puts(chunk)
           end
-          wait_thr.value
         end
+      end
 
-        raise ExecutionError, "exit_code=#{result.exitstatus}" unless result.success?
+      class Cli < Thor
+        namespace :docker
+
+        desc "run_cmd [shell command]", "Run shell on docker"
+        method_option :config, aliases: "-f", required: true, banner: "YAML_FILE", desc: "yaml file path"
+        method_option :config_name, aliases: "-n", required: true, default: "default"
+        method_option :environments, aliases: "-e"
+        def run_cmd(*args)
+          if args.size == 1
+            args = args[0].split(" ")
+          end
+
+          repo = Wrapbox::ConfigRepository.new.tap { |r| r.load_yaml(options[:config]) }
+          config = repo.get(options[:config_name])
+          config.runner = :docker
+          runner = config.build_runner
+          environments = options[:environments].to_s.split(/,\s*/).map { |kv| kv.split("=") }.map do |k, v|
+            {name: k, value: v}
+          end
+          runner.run_cmd(*args, environments: environments)
+        end
       end
     end
   end

data/lib/wrapbox/runner/ecs.rb

@@ -1,13 +1,22 @@
 require "aws-sdk"
 require "multi_json"
+require "thor"
+require "yaml"
+require "active_support/core_ext/hash"
+require "logger"
+
+require "wrapbox/config_repository"
+require "wrapbox/version"
 
 module Wrapbox
   module Runner
     class Ecs
       class ExecutionError < StandardError; end
+      class LaunchFailure < StandardError; end
 
       attr_reader \
         :name,
+        :revision,
         :cluster,
         :region,
         :container_definition,
@@ -16,28 +25,33 @@ module Wrapbox
 
       def initialize(options)
         @name = options[:name]
+        @revision = options[:revision]
         @cluster = options[:cluster]
         @region = options[:region]
         @container_definition = options[:container_definition]
         @additional_container_definitions = options[:additional_container_definitions]
         @task_role_arn = options[:task_role_arn]
+        @logger = Logger.new($stdout)
       end
 
-      def run(class_name, method_name, args, container_definition_overrides: {}, environments: [], task_role_arn: nil, cluster: nil, timeout: 3600 * 24, launch_timeout: 60 * 10, launch_retry: 10)
+      def run(class_name, method_name, args, container_definition_overrides: {}, environments: [], task_role_arn: nil, cluster: nil, timeout: 3600 * 24, launch_timeout: 60 * 10, launch_retry: 10, retry_interval: 1, retry_interval_multiplier: 2, max_retry_interval: 120)
         task_definition = register_task_definition(container_definition_overrides)
         run_task(
           task_definition.task_definition_arn, class_name, method_name, args,
           command: ["bundle", "exec", "rake", "wrapbox:run"],
           environments: environments,
-          task_role_arn: task_role_arn,
+          task_role_arn: task_role_arn || @task_role_arn,
           cluster: cluster,
           timeout: timeout,
           launch_timeout: launch_timeout,
           launch_retry: launch_retry,
+          retry_interval: retry_interval,
+          retry_interval_multiplier: retry_interval_multiplier,
+          max_retry_interval: max_retry_interval,
         )
       end
 
-      def run_cmd(*cmd, container_definition_overrides: {}, environments: [], task_role_arn: nil, cluster: nil, timeout: 3600 * 24, launch_timeout: 60 * 10, launch_retry: 10)
+      def run_cmd(*cmd, container_definition_overrides: {}, environments: [], task_role_arn: nil, cluster: nil, timeout: 3600 * 24, launch_timeout: 60 * 10, launch_retry: 10, retry_interval: 1, retry_interval_multiplier: 2, max_retry_interval: 120)
         task_definition = register_task_definition(container_definition_overrides)
 
         run_task(
@@ -49,52 +63,64 @@ module Wrapbox
           timeout: timeout,
           launch_timeout: launch_timeout,
           launch_retry: launch_retry,
+          retry_interval: retry_interval,
+          retry_interval_multiplier: retry_interval_multiplier,
+          max_retry_interval: max_retry_interval,
         )
       end
 
-      def run_task(task_definition_arn, class_name, method_name, args, command:, environments: [], task_role_arn: nil, cluster: nil, timeout: 3600 * 24, launch_timeout: 60 * 10, launch_retry: 10)
+      def run_task(task_definition_arn, class_name, method_name, args, command:, environments: [], task_role_arn: nil, cluster: nil, timeout: 3600 * 24, launch_timeout: 60 * 10, launch_retry: 10, retry_interval: 1, retry_interval_multiplier: 2, max_retry_interval: 120)
         cl = cluster || self.cluster
         args = Array(args)
 
-        task = client
-          .run_task(build_run_task_options(class_name, method_name, args, command, environments, cluster, task_definition_arn, task_role_arn))
-          .tasks[0]
-
         launch_try_count = 0
+        current_retry_interval = retry_interval
+        task = nil
+        exit_code = nil
         begin
-          launched_at = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
+          task = client
+            .run_task(build_run_task_options(class_name, method_name, args, command, environments, cluster, task_definition_arn, task_role_arn))
+            .tasks[0]
+          raise LaunchFailure unless task
+          @logger.debug("Create Task: #{task.task_arn}")
           client.wait_until(:tasks_running, cluster: cl, tasks: [task.task_arn]) do |w|
             if launch_timeout
+              w.delay = 5
+              w.max_attempts = launch_timeout / w.delay
+            else
               w.max_attempts = nil
-              w.before_wait do
-                throw :failure if Process.clock_gettime(Process::CLOCK_MONOTONIC, :second) - launched_at > launch_timeout
-              end
             end
           end
-        rescue Aws::Waiters::Errors::TooManyAttemptsError
-          if launch_try_count >= launch_retry
-            client.stop_task(
-              cluster: cl,
-              task: task.task_arn,
-              reason: "launch timeout"
-            )
-            raise
-          else
-            put_waiting_task_count_metric(cl)
-            launch_try_count += 1
-            retry
+        rescue Aws::Waiters::Errors::WaiterFailed, LaunchFailure
+          exit_code = task && fetch_exit_code(cl, task.task_arn)
+          unless exit_code
+            if launch_try_count >= launch_retry
+              client.stop_task(
+                cluster: cl,
+                task: task.task_arn,
+                reason: "launch timeout"
+              ) if task
+              raise
+            else
+              put_waiting_task_count_metric(cl)
+              launch_try_count += 1
+              @logger.debug("Retry Create Task after #{current_retry_interval} sec")
+              sleep current_retry_interval
+              current_retry_interval = [current_retry_interval * retry_interval_multiplier, max_retry_interval].min
+              retry
+            end
           end
-        rescue Aws::Waiters::Errors::WaiterFailed
         end
 
+        @logger.debug("Launch Task: #{task.task_arn}")
+
         begin
-          started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
           client.wait_until(:tasks_stopped, cluster: cl, tasks: [task.task_arn]) do |w|
             if timeout
+              w.delay = 5
+              w.max_attempts = timeout / w.delay
+            else
               w.max_attempts = nil
-              w.before_wait do
-                throw :failure if Process.clock_gettime(Process::CLOCK_MONOTONIC, :second) - started_at > timeout
-              end
             end
           end
         rescue Aws::Waiters::Errors::TooManyAttemptsError
@@ -103,12 +129,14 @@ module Wrapbox
             task: task.task_arn,
             reason: "process timeout",
           })
+          raise ExecutionError, "process timeout"
         end
 
-        task = client.describe_tasks(cluster: cl, tasks: [task.task_arn]).tasks[0]
-        container = task.containers.find { |c| c.name = task_definition_name }
-        unless container.exit_code == 0
-          raise ExecutionError, "Container #{task_definition_name} is failed. exit_code=#{container.exit_code}"
+        @logger.debug("Stop Task: #{task.task_arn}")
+
+        exit_code ||= fetch_exit_code(cl, task.task_arn)
+        unless exit_code == 0
+          raise ExecutionError, "Container #{task_definition_name} is failed. exit_code=#{exit_code}"
         end
       end
 
@@ -118,11 +146,25 @@ module Wrapbox
         "wrapbox_#{name}"
       end
 
+      def fetch_exit_code(cluster, task_arn)
+        task = client.describe_tasks(cluster: cluster, tasks: [task_arn]).tasks[0]
+        container = task.containers.find { |c| c.name = task_definition_name }
+        container.exit_code
+      end
+
       def register_task_definition(container_definition_overrides)
         definition = container_definition
           .merge(container_definition_overrides)
           .merge(name: task_definition_name)
         container_definitions = [definition, *additional_container_definitions]
+
+        if revision
+          begin
+            return client.describe_task_definition(task_definition: "#{task_definition_name}:#{revision}").task_definition
+          rescue
+          end
+        end
+
         client.register_task_definition({
           family: task_definition_name,
           container_definitions: container_definitions,
@@ -142,7 +184,7 @@ module Wrapbox
 
         options = {}
         options[:region] = region if region
-        @cloud_watch_client = Aws::CloudWatch::Client.new
+        @cloud_watch_client = Aws::CloudWatch::Client.new(options)
       end
 
       def put_waiting_task_count_metric(cluster)
@@ -197,6 +239,42 @@ module Wrapbox
           started_by: "wrapbox-#{Wrapbox::VERSION}",
         }
       end
+
+      class Cli < Thor
+        namespace :ecs
+
+        desc "run_cmd [shell command]", "Run shell on ECS"
+        method_option :config, aliases: "-f", required: true, banner: "YAML_FILE", desc: "yaml file path"
+        method_option :config_name, aliases: "-n", required: true, default: "default"
+        method_option :cluster, aliases: "-c"
+        method_option :environments, aliases: "-e"
+        method_option :task_role_arn
+        method_option :timeout, type: :numeric
+        method_option :launch_timeout, type: :numeric
+        method_option :launch_retry, type: :numeric
+        method_option :max_retry_interval, type: :numeric
+        def run_cmd(*args)
+          if args.size == 1
+            args = args[0].split(" ")
+          end
+
+          repo = Wrapbox::ConfigRepository.new.tap { |r| r.load_yaml(options[:config]) }
+          config = repo.get(options[:config_name])
+          config.runner = :ecs
+          runner = config.build_runner
+          environments = options[:environments].to_s.split(/,\s*/).map { |kv| kv.split("=") }.map do |k, v|
+            {name: k, value: v}
+          end
+          run_options = {
+            task_role_arn: options[:task_role_arn],
+            timeout: options[:timeout],
+            launch_timeout: options[:launch_timeout],
+            launch_retry: options[:launch_retry],
+            max_retry_interval: options[:max_retry_interval]
+          }.reject { |_, v| v.nil? }
+          runner.run_cmd(*args, environments: environments, **run_options)
+        end
+      end
     end
   end
 end

data/lib/wrapbox/version.rb

@@ -1,3 +1,3 @@
 module Wrapbox
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/wrapbox.gemspec

@@ -22,7 +22,9 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency "aws-sdk", "~> 2.4"
   spec.add_runtime_dependency "activesupport", ">= 4"
+  spec.add_runtime_dependency "docker-api"
   spec.add_runtime_dependency "multi_json"
+  spec.add_runtime_dependency "thor"
 
   spec.add_development_dependency "bundler", "~> 1.13"
   spec.add_development_dependency "rake", "~> 10.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wrapbox
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - joker1007
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-10-31 00:00:00.000000000 Z
+date: 2017-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '4'
+- !ruby/object:Gem::Dependency
+  name: docker-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: multi_json
   requirement: !ruby/object:Gem::Requirement
@@ -52,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -139,7 +167,8 @@ dependencies:
 description: Ruby method runner on AWS ECS
 email:
 - kakyoin.hierophant@gmail.com
-executables: []
+executables:
+- wrapbox
 extensions: []
 extra_rdoc_files: []
 files:
@@ -153,7 +182,9 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- exe/wrapbox
 - lib/wrapbox.rb
+- lib/wrapbox/cli.rb
 - lib/wrapbox/config_repository.rb
 - lib/wrapbox/configuration.rb
 - lib/wrapbox/job.rb
@@ -181,7 +212,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.4
+rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
 summary: Ruby method runner on AWS ECS