wrapbox 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: edc9210efdc2d7248e0b6eab05f04c89f7223b90
-  data.tar.gz: afe62fcbfbf5f5383b12e476709fd7d79a36af27
+  metadata.gz: 8118b81067d91ea3535d8c3c5ca1ba55c412d345
+  data.tar.gz: 33fb0eb243ff53a6bc6ec7c5147a69ea3bbbc206
 SHA512:
-  metadata.gz: 8f856b7dc58a1776c662f4bd43d1165c043c4ec8a2e5ad0a12eac1b4c60a1162d9ef57541c1d54373471f59879f6257cd8ae1105c1b26e142ffc790839694f75
-  data.tar.gz: 0cc10bf941d6b73d320581ae722383c888f5cac1fe700bfccfd5b5ab117abd3b1347abb5f67229d8048d1ff784768224122dd8c767f7188888489b1877a0e559
+  metadata.gz: 5e9a9730bca6b1dddbe9a8c21a2660645dc0d3f6f3c71a74c24e175930a6bd01544e2f2b2efd6d04fc0b14bdbeeded4a4d5ad7abe38e37c4c5abe83593c1a37c
+  data.tar.gz: 74c5c6831768ed37e5107903426131a4158220b808b39fae6d08b74df14dcfcc2f9bd495ec1ce3f6157a66ee5602f08a5161e0c73b6535f1cefdf4ebbf14a388

data/README.md

@@ -42,6 +42,17 @@ docker:
     memory: 1024
 ```
 
+#### run by CLI
+
+```sh
+$ wrapbox ecs run_cmd -f config.yml \
+  -e "FOO=bar,HOGE=fuga" \
+  "bundle exec rspec spec/models" \
+  "bundle exec rspec spec/controllers" \
+```
+
+#### run by ruby
+
 ```ruby
 Wrapbox.configure do |c|
   c.load_yaml(File.expand_path("../config.yml", __FILE__))
@@ -53,10 +64,7 @@ Wrapbox.run("TestJob", :perform, ["arg1", ["arg2", "arg3"]], environments: [{nam
 Wrapbox.run("TestJob", :perform, ["arg1", ["arg2", "arg3"]], config_name: :docker, environments: [{name: "RAILS_ENV", value: "development"}]) # use docker config
 
 # runs ls . command in ECS container
-Wrapbox.run_cmd("ls", ".", environments: [{name: "RAILS_ENV", value: "development"}])
-
-# runs ls . command in local docker container
-Wrapbox.run_cmd("ls", ".", config_name: :docker, environments: [{name: "RAILS_ENV", value: "development"}])
+Wrapbox.run_cmd(["ls ."], environments: [{name: "RAILS_ENV", value: "development"}])
 ```
 
 ## Config
@@ -86,9 +94,9 @@ Wrapbox.run_cmd("ls", ".", config_name: :docker, environments: [{name: "RAILS_EN
 
 ## API
 
-#### `run(class_name, method_name, args, container_definition_overrides: {}, environments: [], task_role_arn: nil, cluster: nil, timeout: 3600 * 24, launch_timeout: 60 * 10, launch_retry: 10)`
+#### `run(class_name, method_name, args, container_definition_overrides: {}, environments: [], task_role_arn: nil, cluster: nil, timeout: 3600 * 24, launch_timeout: 60 * 10, launch_retry: 10, retry_interval: 1, retry_interval_multiplier: 2, max_retry_interval: 120, execution_retry: 0)`
 
-#### `run_cmd(*cmd, container_definition_overrides: {}, environments: [], task_role_arn: nil, cluster: nil, timeout: 3600 * 24, launch_timeout: 60 * 10, launch_retry: 10)`
+#### `run_cmd(*cmd, container_definition_overrides: {}, environments: [], task_role_arn: nil, cluster: nil, timeout: 3600 * 24, launch_timeout: 60 * 10, launch_retry: 10, retry_interval: 1, retry_interval_multiplier: 2, max_retry_interval: 120, execution_retry: 0)`
 
 following options is only for ECS.
 
@@ -98,7 +106,7 @@ following options is only for ECS.
 - launch_timeout
 - launch_retry
 
-If Wrapbox cannot launch task in `launch_timeout` seconds, it puts custom metric data to CloudWatch.
+If Wrapbox cannot create task, it puts custom metric data to CloudWatch.
 Custom metric data is `wrapbox/WaitingTaskCount` that has `ClusterName` dimension.
 And, it retry launching until retry count reach `launch_retry`.
 

data/lib/wrapbox/runner/docker.rb

@@ -29,13 +29,18 @@ module Wrapbox
         exec_docker(definition: definition, cmd: ["bundle", "exec", "rake", "wrapbox:run"], environments: envs)
       end
 
-      def run_cmd(*cmd,  container_definition_overrides: {}, environments: [])
+      def run_cmd(cmds,  container_definition_overrides: {}, environments: [])
         definition = container_definition
           .merge(container_definition_overrides)
 
         environments = extract_environments(environments)
 
-        exec_docker(definition: definition, cmd: cmd, environments: environments)
+        ths = cmds.map do |cmd|
+          Thread.new(cmd) do |c|
+            exec_docker(definition: definition, cmd: c.split(/\s+/), environments: environments)
+          end
+        end
+        ths.each(&:join)
       end
 
       private
@@ -67,7 +72,7 @@ module Wrapbox
 
         container = ::Docker::Container.create(options)
 
-        container.start!
+        container.start
         output_container_logs(container)
         resp = container.wait
         output_container_logs(container)
@@ -97,10 +102,6 @@ module Wrapbox
         method_option :config_name, aliases: "-n", required: true, default: "default"
         method_option :environments, aliases: "-e"
         def run_cmd(*args)
-          if args.size == 1
-            args = args[0].split(" ")
-          end
-
           repo = Wrapbox::ConfigRepository.new.tap { |r| r.load_yaml(options[:config]) }
           config = repo.get(options[:config_name])
           config.runner = :docker
@@ -108,7 +109,7 @@ module Wrapbox
           environments = options[:environments].to_s.split(/,\s*/).map { |kv| kv.split("=") }.map do |k, v|
             {name: k, value: v}
           end
-          runner.run_cmd(*args, environments: environments)
+          runner.run_cmd(args, environments: environments)
         end
       end
     end

data/lib/wrapbox/runner/ecs.rb

@@ -11,8 +11,15 @@ require "wrapbox/version"
 module Wrapbox
   module Runner
     class Ecs
-      class ExecutionError < StandardError; end
+      class ExecutionFailure < StandardError; end
+      class ContainerAbnormalEnd < StandardError; end
+      class ExecutionTimeout < StandardError; end
       class LaunchFailure < StandardError; end
+      class LackResource < StandardError; end
+
+      EXECUTION_RETRY_INTERVAL = 3
+      WAIT_DELAY = 5
+      HOST_TERMINATED_REASON_REGEXP = /Host EC2.*terminated/
 
       attr_reader \
         :name,
@@ -31,125 +38,206 @@ module Wrapbox
         @container_definition = options[:container_definition]
         @additional_container_definitions = options[:additional_container_definitions]
         @task_role_arn = options[:task_role_arn]
+        $stdout.sync = true
         @logger = Logger.new($stdout)
       end
 
-      def run(class_name, method_name, args, container_definition_overrides: {}, environments: [], task_role_arn: nil, cluster: nil, timeout: 3600 * 24, launch_timeout: 60 * 10, launch_retry: 10, retry_interval: 1, retry_interval_multiplier: 2, max_retry_interval: 120)
+      class Parameter
+        attr_reader \
+          :environments,
+          :task_role_arn,
+          :cluster,
+          :timeout,
+          :launch_timeout,
+          :launch_retry,
+          :retry_interval,
+          :retry_interval_multiplier,
+          :max_retry_interval,
+          :execution_retry
+
+        def initialize(environments: [], task_role_arn: nil, cluster: nil, timeout: 3600 * 24, launch_timeout: 60 * 10, launch_retry: 10, retry_interval: 1, retry_interval_multiplier: 2, max_retry_interval: 120, execution_retry: 0)
+          b = binding
+          method(:initialize).parameters.each do |param|
+            instance_variable_set("@#{param[1]}", b.local_variable_get(param[1]))
+          end
+        end
+      end
+
+      def run(class_name, method_name, args, container_definition_overrides: {}, **parameters)
         task_definition = register_task_definition(container_definition_overrides)
+        parameter = Parameter.new(**parameters)
+
         run_task(
           task_definition.task_definition_arn, class_name, method_name, args,
-          command: ["bundle", "exec", "rake", "wrapbox:run"],
-          environments: environments,
-          task_role_arn: task_role_arn || @task_role_arn,
-          cluster: cluster,
-          timeout: timeout,
-          launch_timeout: launch_timeout,
-          launch_retry: launch_retry,
-          retry_interval: retry_interval,
-          retry_interval_multiplier: retry_interval_multiplier,
-          max_retry_interval: max_retry_interval,
+          ["bundle", "exec", "rake", "wrapbox:run"],
+          parameter
         )
       end
 
-      def run_cmd(*cmd, container_definition_overrides: {}, environments: [], task_role_arn: nil, cluster: nil, timeout: 3600 * 24, launch_timeout: 60 * 10, launch_retry: 10, retry_interval: 1, retry_interval_multiplier: 2, max_retry_interval: 120)
+      def run_cmd(cmds, container_definition_overrides: {}, **parameters)
         task_definition = register_task_definition(container_definition_overrides)
+        parameter = Parameter.new(**parameters)
 
-        run_task(
-          task_definition.task_definition_arn, nil, nil, nil,
-          command: cmd,
-          environments: environments,
-          task_role_arn: task_role_arn,
-          cluster: cluster,
-          timeout: timeout,
-          launch_timeout: launch_timeout,
-          launch_retry: launch_retry,
-          retry_interval: retry_interval,
-          retry_interval_multiplier: retry_interval_multiplier,
-          max_retry_interval: max_retry_interval,
-        )
+        ths = cmds.map do |cmd|
+          Thread.new(cmd) do |c|
+            run_task(
+              task_definition.task_definition_arn, nil, nil, nil,
+              c.split(/\s+/),
+              parameter
+            )
+          end
+        end
+        ths.each(&:join)
+      end
+
+      private
+
+      def run_task(task_definition_arn, class_name, method_name, args, command, parameter)
+        cl = parameter.cluster || self.cluster
+        execution_try_count = 0
+
+        begin
+          task = create_task(task_definition_arn, class_name, method_name, args, command, parameter)
+
+          @logger.debug("Launch Task: #{task.task_arn}")
+
+          wait_task_stopped(cl, task.task_arn, parameter.timeout)
+
+          @logger.debug("Stop Task: #{task.task_arn}")
+
+          # Avoid container exit code fetch miss
+          sleep WAIT_DELAY
+
+          task_status = fetch_task_status(cl, task.task_arn)
+
+          # If exit_code is nil, Container is force killed or ECS failed to launch Container by Irregular situation
+          error_message = "Container #{task_definition_name} is failed. task=#{task.task_arn}, exit_code=#{task_status[:exit_code]}, reason=#{task_status[:stopped_reason]}"
+          raise ContainerAbnormalEnd, error_message unless task_status[:exit_code]
+          raise ExecutionFailure, error_message unless task_status[:exit_code] == 0
+
+          true
+        rescue ContainerAbnormalEnd
+          retry if task_status[:stopped_reason] =~ HOST_TERMINATED_REASON_REGEXP
+
+          if execution_try_count >= parameter.execution_retry
+            raise
+          else
+            execution_try_count += 1
+            @logger.debug("Retry Execution after #{EXECUTION_RETRY_INTERVAL} sec")
+            sleep EXECUTION_RETRY_INTERVAL
+            retry
+          end
+        end
+      end
+
+      def task_definition_name
+        "wrapbox_#{name}"
       end
 
-      def run_task(task_definition_arn, class_name, method_name, args, command:, environments: [], task_role_arn: nil, cluster: nil, timeout: 3600 * 24, launch_timeout: 60 * 10, launch_retry: 10, retry_interval: 1, retry_interval_multiplier: 2, max_retry_interval: 120)
-        cl = cluster || self.cluster
+      def create_task(task_definition_arn, class_name, method_name, args, command, parameter)
+        cl = parameter.cluster || self.cluster
         args = Array(args)
 
         launch_try_count = 0
-        current_retry_interval = retry_interval
-        task = nil
-        exit_code = nil
+        current_retry_interval = parameter.retry_interval
+
         begin
           task = client
-            .run_task(build_run_task_options(class_name, method_name, args, command, environments, cluster, task_definition_arn, task_role_arn))
+            .run_task(build_run_task_options(task_definition_arn, class_name, method_name, args, command, cl, parameter.environments, parameter.task_role_arn))
             .tasks[0]
-          raise LaunchFailure unless task
+
+          raise LackResource unless task # this case is almost lack of container resource.
+
           @logger.debug("Create Task: #{task.task_arn}")
-          client.wait_until(:tasks_running, cluster: cl, tasks: [task.task_arn]) do |w|
-            if launch_timeout
-              w.delay = 5
-              w.max_attempts = launch_timeout / w.delay
-            else
-              w.max_attempts = nil
-            end
-          end
-        rescue Aws::Waiters::Errors::WaiterFailed, LaunchFailure
-          exit_code = task && fetch_exit_code(cl, task.task_arn)
-          unless exit_code
-            if launch_try_count >= launch_retry
-              client.stop_task(
-                cluster: cl,
-                task: task.task_arn,
-                reason: "launch timeout"
-              ) if task
-              raise
-            else
-              put_waiting_task_count_metric(cl)
-              launch_try_count += 1
-              @logger.debug("Retry Create Task after #{current_retry_interval} sec")
-              sleep current_retry_interval
-              current_retry_interval = [current_retry_interval * retry_interval_multiplier, max_retry_interval].min
-              retry
-            end
-          end
-        end
 
-        @logger.debug("Launch Task: #{task.task_arn}")
+          # Wait ECS Task Status becomes stable
+          sleep WAIT_DELAY
 
-        begin
-          client.wait_until(:tasks_stopped, cluster: cl, tasks: [task.task_arn]) do |w|
-            if timeout
-              w.delay = 5
-              w.max_attempts = timeout / w.delay
+          begin
+            wait_task_running(cl, task.task_arn, parameter.launch_timeout)
+            task
+          rescue Aws::Waiters::Errors::TooManyAttemptsError
+            client.stop_task(
+              cluster: cl,
+              task: task.task_arn,
+              reason: "launch timeout"
+            )
+            raise
+          rescue Aws::Waiters::Errors::WaiterFailed
+            task_status = fetch_task_status(cl, task.task_arn)
+
+            case task_status[:last_status]
+            when "RUNNING"
+              return task
+            when "PENDING"
+              retry
             else
-              w.max_attempts = nil
+              if task_status[:exit_code]
+                return task
+              else
+                raise LaunchFailure
+              end
             end
           end
-        rescue Aws::Waiters::Errors::TooManyAttemptsError
-          client.stop_task({
-            cluster: cluster || self.cluster,
-            task: task.task_arn,
-            reason: "process timeout",
-          })
-          raise ExecutionError, "process timeout"
-        end
+        rescue LackResource
+          @logger.debug("Failed to create task, because of lack resource")
+          put_waiting_task_count_metric(cl)
 
-        @logger.debug("Stop Task: #{task.task_arn}")
-
-        exit_code ||= fetch_exit_code(cl, task.task_arn)
-        unless exit_code == 0
-          raise ExecutionError, "Container #{task_definition_name} is failed. exit_code=#{exit_code}"
+          if launch_try_count >= parameter.launch_retry
+            raise
+          else
+            launch_try_count += 1
+            @logger.debug("Retry Create Task after #{current_retry_interval} sec")
+            sleep current_retry_interval
+            current_retry_interval = [current_retry_interval * parameter.retry_interval_multiplier, parameter.max_retry_interval].min
+            retry
+          end
+        rescue LaunchFailure
+          if launch_try_count >= parameter.launch_retry
+            raise
+          else
+            launch_try_count += 1
+            @logger.debug("Retry Create Task after #{current_retry_interval} sec")
+            sleep current_retry_interval
+            current_retry_interval = [current_retry_interval * parameter.retry_interval_multiplier, parameter.max_retry_interval].min
+            retry
+          end
         end
       end
 
-      private
+      def wait_task_running(cluster, task_arn, launch_timeout)
+        client.wait_until(:tasks_running, cluster: cluster, tasks: [task_arn]) do |w|
+          if launch_timeout
+            w.delay = WAIT_DELAY
+            w.max_attempts = launch_timeout / w.delay
+          else
+            w.max_attempts = nil
+          end
+        end
+      end
 
-      def task_definition_name
-        "wrapbox_#{name}"
+      def wait_task_stopped(cluster, task_arn, timeout)
+        client.wait_until(:tasks_stopped, cluster: cluster, tasks: [task_arn]) do |w|
+          if timeout
+            w.delay = WAIT_DELAY
+            w.max_attempts = timeout / w.delay
+          else
+            w.max_attempts = nil
+          end
+        end
+      rescue Aws::Waiters::Errors::TooManyAttemptsError
+        client.stop_task({
+          cluster: cl,
+          task: task_arn,
+          reason: "process timeout",
+        })
+        raise ExecutionTimeout, "Container #{task_definition_name} is timeout. task=#{task_arn}, timeout=#{timeout}"
       end
 
-      def fetch_exit_code(cluster, task_arn)
+      def fetch_task_status(cluster, task_arn)
         task = client.describe_tasks(cluster: cluster, tasks: [task_arn]).tasks[0]
         container = task.containers.find { |c| c.name = task_definition_name }
-        container.exit_code
+        {last_status: task.last_status, exit_code: container.exit_code, stopped_reason: task.stopped_reason}
       end
 
       def register_task_definition(container_definition_overrides)
@@ -165,10 +253,18 @@ module Wrapbox
           end
         end
 
-        client.register_task_definition({
-          family: task_definition_name,
-          container_definitions: container_definitions,
-        }).task_definition
+        register_retry_count = 0
+        begin
+          client.register_task_definition({
+            family: task_definition_name,
+            container_definitions: container_definitions,
+          }).task_definition
+        rescue Aws::ECS::Errors::ClientException
+          raise if register_retry_count > 2
+          register_retry_count += 1
+          sleep 2
+          retry
+        end
       end
 
       def client
@@ -198,13 +294,14 @@ module Wrapbox
                 value: cluster || self.cluster,
               },
             ],
+            timestamp: Time.now,
             value: 1.0,
             unit: "Count",
           ]
         )
       end
 
-      def build_run_task_options(class_name, method_name, args, command, environments, cluster, task_definition_arn, task_role_arn)
+      def build_run_task_options(task_definition_arn, class_name, method_name, args, command, cluster, environments, task_role_arn)
         env = environments
         env += [
           {
@@ -252,12 +349,9 @@ module Wrapbox
         method_option :timeout, type: :numeric
         method_option :launch_timeout, type: :numeric
         method_option :launch_retry, type: :numeric
+        method_option :execution_retry, type: :numeric
         method_option :max_retry_interval, type: :numeric
         def run_cmd(*args)
-          if args.size == 1
-            args = args[0].split(" ")
-          end
-
           repo = Wrapbox::ConfigRepository.new.tap { |r| r.load_yaml(options[:config]) }
           config = repo.get(options[:config_name])
           config.runner = :ecs
@@ -270,9 +364,10 @@ module Wrapbox
             timeout: options[:timeout],
             launch_timeout: options[:launch_timeout],
             launch_retry: options[:launch_retry],
+            execution_retry: options[:execution_retry],
             max_retry_interval: options[:max_retry_interval]
           }.reject { |_, v| v.nil? }
-          runner.run_cmd(*args, environments: environments, **run_options)
+          runner.run_cmd(args, environments: environments, **run_options)
         end
       end
     end

data/lib/wrapbox/version.rb

@@ -1,3 +1,3 @@
 module Wrapbox
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wrapbox
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - joker1007
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-02-17 00:00:00.000000000 Z
+date: 2017-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk