wrapbox 0.9.0 → 0.10.0

data/lib/wrapbox/runner/ecs.rb

@@ -4,12 +4,15 @@ require "multi_json"
 require "thor"
 require "yaml"
 require "active_support/core_ext/hash"
-require "logger"
 require "pp"
 require "shellwords"
+require "thwait"
 
+require "wrapbox"
 require "wrapbox/config_repository"
 require "wrapbox/log_fetcher"
+require "wrapbox/runner/ecs/instance_manager"
+require "wrapbox/runner/ecs/task_waiter"
 require "wrapbox/version"
 
 module Wrapbox
@@ -29,13 +32,11 @@ module Wrapbox
       attr_reader \
         :name,
         :revision,
-        :cluster,
         :region,
         :container_definitions,
         :volumes,
         :placement_constraints,
         :placement_strategy,
-        :launch_type,
         :requires_compatibilities,
         :task_definition_name,
         :main_container_name,
@@ -43,8 +44,21 @@ module Wrapbox
         :network_configuration,
         :cpu,
         :memory,
-        :task_role_arn,
-        :execution_role_arn
+        :enable_ecs_managed_tags,
+        :tags,
+        :propagate_tags,
+        :enable_execute_command
+
+      def self.split_overridable_options_and_parameters(options)
+        opts = options.dup
+        overridable_options = {}
+        %i[cluster launch_type task_role_arn execution_role_arn tags propagate_tags].each do |key|
+          value = opts.delete(key)
+          overridable_options[key] = value if value
+        end
+
+        [overridable_options, opts]
+      end
 
       def initialize(options)
         @name = options[:name]
@@ -53,14 +67,23 @@ module Wrapbox
         @cluster = options[:cluster]
         @region = options[:region]
         @volumes = options[:volumes]
-        @placement_constraints = options[:placement_constraints]
+        @placement_constraints = options[:placement_constraints] || []
         @placement_strategy = options[:placement_strategy]
+        @capacity_provider_strategy = options[:capacity_provider_strategy] || []
         @launch_type = options[:launch_type]
         @requires_compatibilities = options[:requires_compatibilities]
         @network_mode = options[:network_mode]
         @network_configuration = options[:network_configuration]
         @cpu = options[:cpu]
         @memory = options[:memory]
+        @enable_ecs_managed_tags = options[:enable_ecs_managed_tags]
+        @tags = options[:tags]
+        @propagate_tags = options[:propagate_tags]
+        @enable_execute_command = options[:enable_execute_command]
+        if options[:launch_instances]
+          @instance_manager = Wrapbox::Runner::Ecs::InstanceManager.new(@cluster, @region, **options[:launch_instances])
+        end
+        @task_waiter = Wrapbox::Runner::Ecs::TaskWaiter.new(cluster: @cluster, region: @region, delay: WAIT_DELAY)
 
         @container_definitions = options[:container_definition] ? [options[:container_definition]] : options[:container_definitions] || []
         @container_definitions.concat(options[:additional_container_definitions]) if options[:additional_container_definitions] # deprecated
@@ -96,18 +119,14 @@ module Wrapbox
         @logger = Wrapbox.logger
         if options[:log_fetcher]
           type = options[:log_fetcher][:type]
-          @log_fetcher = LogFetcher.new(type, options[:log_fetcher])
+          @log_fetcher = LogFetcher.new(type, **options[:log_fetcher])
         end
       end
 
       class Parameter
         attr_reader \
           :environments,
-          :task_role_arn,
-          :execution_role_arn,
-          :cluster,
           :timeout,
-          :launch_type,
           :launch_timeout,
           :launch_retry,
           :retry_interval,
@@ -115,7 +134,7 @@ module Wrapbox
           :max_retry_interval,
           :execution_retry
 
-        def initialize(environments: [], task_role_arn: nil, cluster: nil, timeout: 3600 * 24, launch_type: "EC2", launch_timeout: 60 * 10, launch_retry: 10, retry_interval: 1, retry_interval_multiplier: 2, max_retry_interval: 120, execution_retry: 0)
+        def initialize(environments: [], timeout: 3600 * 24, launch_timeout: 60 * 10, launch_retry: 10, retry_interval: 1, retry_interval_multiplier: 2, max_retry_interval: 120, execution_retry: 0)
           b = binding
           method(:initialize).parameters.each do |param|
             instance_variable_set("@#{param[1]}", b.local_variable_get(param[1]))
@@ -127,30 +146,52 @@ module Wrapbox
         task_definition = prepare_task_definition(container_definition_overrides)
         parameter = Parameter.new(**parameters)
 
-        run_task(
-          task_definition.task_definition_arn, class_name, method_name, args,
-          ["bundle", "exec", "rake", "wrapbox:run"],
-          parameter
-        )
+        envs = parameters[:environments] || []
+        envs += [
+          {
+            name: CLASS_NAME_ENV,
+            value: class_name.to_s,
+          },
+          {
+            name: METHOD_NAME_ENV,
+            value: method_name.to_s,
+          },
+          {
+            name: METHOD_ARGS_ENV,
+            value: MultiJson.dump(args),
+          },
+        ]
+
+        if @instance_manager
+          Thread.new { @instance_manager.start_preparing_instances(1) }
+        end
+
+        run_task(task_definition.task_definition_arn, ["bundle", "exec", "rake", "wrapbox:run"], envs, parameter)
+      ensure
+        @instance_manager&.terminate_all_instances
       end
 
       def run_cmd(cmds, container_definition_overrides: {}, ignore_signal: false, **parameters)
         ths = []
 
         task_definition = prepare_task_definition(container_definition_overrides)
+        parameter = Parameter.new(**parameters)
 
         cmds << nil if cmds.empty?
+
+        if @instance_manager
+          Thread.new { @instance_manager.start_preparing_instances(cmds.size) }
+        end
+
         cmds.each_with_index do |cmd, idx|
           ths << Thread.new(cmd, idx) do |c, i|
             Thread.current[:cmd_index] = i
             envs = (parameters[:environments] || []) + [{name: "WRAPBOX_CMD_INDEX", value: i.to_s}]
-            run_task(
-              task_definition.task_definition_arn, nil, nil, nil,
-              c&.shellsplit,
-              Parameter.new(**parameters.merge(environments: envs))
-            )
+            run_task(task_definition.task_definition_arn, c&.shellsplit, envs, parameter)
           end
         end
+        ThreadsWait.all_waits(ths)
+        # Raise an error if some threads have an error
         ths.each(&:join)
 
         true
@@ -171,6 +212,8 @@ module Wrapbox
           end
         end
         nil
+      ensure
+        @instance_manager&.terminate_all_instances
       end
 
       private
@@ -179,24 +222,24 @@ module Wrapbox
         !!@task_definition_info
       end
 
-      def run_task(task_definition_arn, class_name, method_name, args, command, parameter)
-        cl = parameter.cluster || self.cluster
+      def run_task(task_definition_arn, command, environments, parameter)
         execution_try_count = 0
 
+        ec2_instance_id = @instance_manager&.pop_ec2_instance_id
         begin
-          task = create_task(task_definition_arn, class_name, method_name, args, command, parameter)
+          task = create_task(task_definition_arn, command, environments, parameter, ec2_instance_id)
           return unless task # only Task creation aborted by SignalException
 
-          @logger.debug("Launch Task: #{task.task_arn}")
+          @logger.info("#{log_prefix}Launch Task: #{task.task_arn}")
 
-          wait_task_stopped(cl, task.task_arn, parameter.timeout)
+          wait_task_stopped(task.task_arn, parameter.timeout)
 
-          @logger.debug("Stop Task: #{task.task_arn}")
+          @logger.info("#{log_prefix}Stop Task: #{task.task_arn}")
 
           # Avoid container exit code fetch miss
           sleep WAIT_DELAY
 
-          task_status = fetch_task_status(cl, task.task_arn)
+          task_status = fetch_task_status(task.task_arn)
 
           # If exit_code is nil, Container is force killed or ECS failed to launch Container by Irregular situation
           error_message = build_error_message(task_definition_name, task.task_arn, task_status)
@@ -211,18 +254,18 @@ module Wrapbox
             raise
           else
             execution_try_count += 1
-            @logger.warn("Retry Execution after #{EXECUTION_RETRY_INTERVAL} sec (#{execution_try_count}/#{parameter.execution_retry})")
+            @logger.warn("#{log_prefix}Retry Execution after #{EXECUTION_RETRY_INTERVAL} sec (#{execution_try_count}/#{parameter.execution_retry})")
             sleep EXECUTION_RETRY_INTERVAL
             retry
           end
         rescue SignalException
           client.stop_task(
-            cluster: cl,
+            cluster: @cluster,
             task: task.task_arn,
             reason: "signal interrupted"
           )
-          wait_task_stopped(cl, task.task_arn, TERM_TIMEOUT)
-          @logger.debug("Stop Task: #{task.task_arn}")
+          wait_task_stopped(task.task_arn, TERM_TIMEOUT)
+          @logger.debug("#{log_prefix}Stop Task: #{task.task_arn}")
         ensure
           if @log_fetcher
             begin
@@ -231,29 +274,43 @@ module Wrapbox
               @logger.warn(e)
             end
           end
+          @instance_manager.terminate_instance(ec2_instance_id) if ec2_instance_id
         end
       end
 
-      def create_task(task_definition_arn, class_name, method_name, args, command, parameter)
-        cl = parameter.cluster || self.cluster
-        launch_type = parameter.launch_type || self.launch_type
+      def create_task(task_definition_arn, command, environments, parameter, ec2_instance_id)
         args = Array(args)
 
         launch_try_count = 0
         current_retry_interval = parameter.retry_interval
 
         begin
-          run_task_options = build_run_task_options(task_definition_arn, class_name, method_name, args, command, cl, launch_type, parameter.environments, parameter.task_role_arn)
-          @logger.debug("Task Options: #{run_task_options}")
-          resp = client.run_task(run_task_options)
+          run_task_options = build_run_task_options(task_definition_arn, command, environments, ec2_instance_id)
+          @logger.debug("#{log_prefix}Task Options: #{run_task_options}")
+
+          begin
+            resp = client.run_task(run_task_options)
+          rescue Aws::ECS::Errors::ThrottlingException
+            @logger.warn("#{log_prefix}Failure: Rate exceeded.")
+            raise LaunchFailure
+          rescue Aws::ECS::Errors::InvalidParameterException => e
+            # ec2:DescribeSecurityGroups is called in ecs:RunTask when awsvpc mode is used,
+            # and some errors like "Request limit exceeded", "InternalError" etc. caused by
+            # ec2:DescribeSecurityGroups are retriable.
+            # cf. https://github.com/reproio/wrapbox/issues/32, https://github.com/reproio/wrapbox/issues/43
+            raise if !e.message.include?("Request limit exceeded") && !e.message.include?("InternalError")
+
+            @logger.warn("#{log_prefix}Failure: #{e.message}")
+            raise LaunchFailure
+          end
           task = resp.tasks[0]
 
           resp.failures.each do |failure|
-            @logger.warn("Failure: Arn=#{failure.arn}, Reason=#{failure.reason}")
+            @logger.warn("#{log_prefix}Failure: Arn=#{failure.arn}, Reason=#{failure.reason}")
           end
           raise LackResource unless task # this case is almost lack of container resource.
 
-          @logger.debug("Create Task: #{task.task_arn}")
+          @logger.debug("#{log_prefix}Create Task: #{task.task_arn}")
 
           @log_fetcher.run(task: task) if @log_fetcher
 
@@ -261,17 +318,17 @@ module Wrapbox
           sleep WAIT_DELAY
 
           begin
-            wait_task_running(cl, task.task_arn, parameter.launch_timeout)
+            wait_task_running(task.task_arn, parameter.launch_timeout)
             task
-          rescue Aws::Waiters::Errors::TooManyAttemptsError
+          rescue Wrapbox::Runner::Ecs::TaskWaiter::WaitTimeout
             client.stop_task(
-              cluster: cl,
+              cluster: @cluster,
               task: task.task_arn,
               reason: "launch timeout"
             )
             raise
-          rescue Aws::Waiters::Errors::WaiterFailed
-            task_status = fetch_task_status(cl, task.task_arn)
+          rescue Wrapbox::Runner::Ecs::TaskWaiter::WaitFailure
+            task_status = fetch_task_status(task.task_arn)
 
             case task_status[:last_status]
             when "RUNNING"
@@ -287,71 +344,62 @@ module Wrapbox
             end
           end
         rescue LackResource
-          @logger.warn("Failed to create task, because of lack resource")
-          put_waiting_task_count_metric(cl)
+          @logger.warn("#{log_prefix}Failed to create task, because of lack resource")
+          put_waiting_task_count_metric
 
           if launch_try_count >= parameter.launch_retry
             raise
           else
             launch_try_count += 1
-            @logger.warn("Retry Create Task after #{current_retry_interval} sec (#{launch_try_count}/#{parameter.launch_retry})")
-            sleep current_retry_interval
+            retry_interval = current_retry_interval/2 + rand(current_retry_interval/2)
+            @logger.warn("#{log_prefix}Retry Create Task after #{retry_interval} sec (#{launch_try_count}/#{parameter.launch_retry})")
+            sleep retry_interval
             current_retry_interval = [current_retry_interval * parameter.retry_interval_multiplier, parameter.max_retry_interval].min
             retry
           end
         rescue LaunchFailure
           if launch_try_count >= parameter.launch_retry
-            task_status = fetch_task_status(cl, task.task_arn)
+            task_status = fetch_task_status(task.task_arn)
             raise LaunchFailure, build_error_message(task_definition_name, task.task_arn, task_status)
           else
             launch_try_count += 1
-            @logger.warn("Retry Create Task after #{current_retry_interval} sec (#{launch_try_count}/#{parameter.launch_retry})")
-            sleep current_retry_interval
+            retry_interval = current_retry_interval/2 + rand(current_retry_interval/2)
+            @logger.warn("#{log_prefix}Retry Create Task after #{retry_interval} sec (#{launch_try_count}/#{parameter.launch_retry})")
+            sleep retry_interval
             current_retry_interval = [current_retry_interval * parameter.retry_interval_multiplier, parameter.max_retry_interval].min
             retry
           end
         rescue SignalException
           if task
             client.stop_task(
-              cluster: cl,
+              cluster: @cluster,
               task: task.task_arn,
               reason: "signal interrupted"
             )
-            wait_task_stopped(cl, task.task_arn, TERM_TIMEOUT)
-            @logger.debug("Stop Task: #{task.task_arn}")
+            wait_task_stopped(task.task_arn, TERM_TIMEOUT)
+            @logger.debug("#{log_prefix}Stop Task: #{task.task_arn}")
             nil
           end
         end
       end
 
-      def wait_until_with_timeout(cluster, task_arn, timeout, waiter_name)
-        client.wait_until(waiter_name, cluster: cluster, tasks: [task_arn]) do |w|
-          if timeout
-            w.delay = WAIT_DELAY
-            w.max_attempts = timeout / w.delay
-          else
-            w.max_attempts = nil
-          end
-        end
+      def wait_task_running(task_arn, launch_timeout)
+        @task_waiter.wait_task_running(task_arn, timeout: launch_timeout)
       end
 
-      def wait_task_running(cluster, task_arn, launch_timeout)
-        wait_until_with_timeout(cluster, task_arn, launch_timeout, :tasks_running)
-      end
-
-      def wait_task_stopped(cluster, task_arn, execution_timeout)
-        wait_until_with_timeout(cluster, task_arn, execution_timeout, :tasks_stopped)
-      rescue Aws::Waiters::Errors::TooManyAttemptsError
+      def wait_task_stopped(task_arn, execution_timeout)
+        @task_waiter.wait_task_stopped(task_arn, timeout: execution_timeout)
+      rescue Wrapbox::Runner::Ecs::TaskWaiter::WaitTimeout
         client.stop_task({
-          cluster: cluster,
+          cluster: @cluster,
           task: task_arn,
           reason: "process timeout",
         })
         raise ExecutionTimeout, "Task #{task_definition_name} is timeout. task=#{task_arn}, timeout=#{execution_timeout}"
       end
 
-      def fetch_task_status(cluster, task_arn)
-        task = client.describe_tasks(cluster: cluster, tasks: [task_arn]).tasks[0]
+      def fetch_task_status(task_arn)
+        task = client.describe_tasks(cluster: @cluster, tasks: [task_arn]).tasks[0]
         container = task.containers.find { |c| c.name == main_container_name }
         {
           last_status: task.last_status,
@@ -384,7 +432,7 @@ module Wrapbox
           end
         end
 
-        @logger.debug("Container Definitions: #{overrided_container_definitions}")
+        @logger.debug("#{log_prefix}Container Definitions: #{overrided_container_definitions}")
         register_retry_count = 0
         begin
           client.register_task_definition({
@@ -395,10 +443,11 @@ module Wrapbox
             container_definitions: overrided_container_definitions,
             volumes: volumes,
             requires_compatibilities: requires_compatibilities,
-            task_role_arn: task_role_arn,
-            execution_role_arn: execution_role_arn
+            task_role_arn: @task_role_arn,
+            execution_role_arn: @execution_role_arn,
+            tags: tags,
           }).task_definition
-        rescue Aws::ECS::Errors::ClientException
+        rescue Aws::ECS::Errors::ClientException, Aws::ECS::Errors::ThrottlingException
           raise if register_retry_count > 2
           register_retry_count += 1
           sleep 2
@@ -406,6 +455,14 @@ module Wrapbox
         end
       end
 
+      def cmd_index
+        Thread.current[:cmd_index]
+      end
+
+      def log_prefix
+        cmd_index ? "##{cmd_index} " : ""
+      end
+
       def client
         return @client if @client
 
@@ -422,7 +479,7 @@ module Wrapbox
         @cloud_watch_client = Aws::CloudWatch::Client.new(options)
       end
 
-      def put_waiting_task_count_metric(cluster)
+      def put_waiting_task_count_metric
         cloud_watch_client.put_metric_data(
           namespace: "wrapbox",
           metric_data: [
@@ -430,7 +487,7 @@ module Wrapbox
             dimensions: [
               {
                 name: "ClusterName",
-                value: cluster || self.cluster,
+                value: @cluster,
               },
             ],
             timestamp: Time.now,
@@ -440,54 +497,53 @@ module Wrapbox
         )
       end
 
-      def build_run_task_options(task_definition_arn, class_name, method_name, args, command, cluster, launch_type, environments, task_role_arn)
-        env = environments
-        env += [
-          {
-            name: CLASS_NAME_ENV,
-            value: class_name.to_s,
-          },
-          {
-            name: METHOD_NAME_ENV,
-            value: method_name.to_s,
-          },
-          {
-            name: METHOD_ARGS_ENV,
-            value: MultiJson.dump(args),
-          },
-        ] if class_name && method_name && args
+      def build_run_task_options(task_definition_arn, command, environments, ec2_instance_id)
         overrides = {
           container_overrides: [
             {
               name: main_container_name,
-              environment: env,
+              environment: environments,
             }.tap { |o| o[:command] = command if command },
             *container_definitions.drop(1).map do |c|
               {
                 name: c[:name],
-                environment: env,
+                environment: environments,
               }
             end
           ],
         }
-        role_arn = task_role_arn || self.task_role_arn
-        overrides[:task_role_arn] = role_arn if role_arn
+        overrides[:task_role_arn] = @task_role_arn if @task_role_arn
 
-        {
-          cluster: cluster || self.cluster,
+        additional_placement_constraints = []
+        if ec2_instance_id
+          additional_placement_constraints << { type: "memberOf", expression: "ec2InstanceId == #{ec2_instance_id}" }
+        end
+        options = {
+          cluster: @cluster,
+          enable_execute_command: enable_execute_command,
           task_definition: task_definition_arn,
           overrides: overrides,
           placement_strategy: placement_strategy,
-          placement_constraints: placement_constraints,
-          launch_type: launch_type,
+          placement_constraints: placement_constraints + additional_placement_constraints,
           network_configuration: network_configuration,
           started_by: "wrapbox-#{Wrapbox::VERSION}",
+          enable_ecs_managed_tags: enable_ecs_managed_tags,
+          propagate_tags: propagate_tags,
         }
+        if @capacity_provider_strategy.empty?
+          options[:launch_type] = @launch_type if @launch_type
+        else
+          if @launch_type
+            @logger.warn("#{log_prefix}Ignore --launch_type and launch_type in the configuration file when specified capacity_provider_strategy in the configuration file")
+          end
+          options[:capacity_provider_strategy] = @capacity_provider_strategy
+        end
+        options
       end
 
       def build_error_message(task_definition_name, task_arn, task_status)
         error_message = "Task #{task_definition_name} is failed. task=#{task_arn}, "
-        error_message << "cmd_index=#{Thread.current[:cmd_index]}, " if Thread.current[:cmd_index]
+        error_message << "cmd_index=#{cmd_index}, " if cmd_index
         error_message << "exit_code=#{task_status[:exit_code]}, task_stopped_reason=#{task_status[:stopped_reason]}, container_stopped_reason=#{task_status[:container_stopped_reason]}"
         error_message
       end
@@ -501,15 +557,18 @@ module Wrapbox
         method_option :cluster, aliases: "-c"
         method_option :cpu, type: :numeric
         method_option :memory, type: :numeric
+        method_option :working_directory, aliases: "-w", type: :string
         method_option :environments, aliases: "-e"
         method_option :task_role_arn
         method_option :timeout, type: :numeric
-        method_option :launch_type, type: :string, default: "EC2", enum: ["EC2", "FARGATE"]
+        method_option :launch_type, type: :string, enum: ["EC2", "FARGATE"]
         method_option :launch_timeout, type: :numeric
         method_option :launch_retry, type: :numeric
         method_option :execution_retry, type: :numeric
         method_option :max_retry_interval, type: :numeric
         method_option :ignore_signal, type: :boolean, default: false, desc: "Even if receive a signal (like TERM, INT, QUIT), ECS Tasks continue running"
+        method_option :tags, type: :string, aliases: "-t", repeatable: true
+        method_option :propagate_tags, type: :string, enum: ["TASK_DEFINITION", "SERVICE"]
         method_option :verbose, aliases: "-v", type: :boolean, default: false, desc: "Verbose mode"
         def run_cmd(*args)
           Wrapbox.logger.level = :debug if options[:verbose]
@@ -518,6 +577,10 @@ module Wrapbox
           environments = options[:environments].to_s.split(/,\s*/).map { |kv| kv.split("=") }.map do |k, v|
             {name: k, value: v}
           end
+          tags = options.fetch(:tags, []).map do |kv|
+            k, v = kv.split("=", 2)
+            {key: k, value: v}
+          end.presence
           run_options = {
             cluster: options[:cluster],
             task_role_arn: options[:task_role_arn],
@@ -528,9 +591,11 @@ module Wrapbox
             execution_retry: options[:execution_retry],
             max_retry_interval: options[:max_retry_interval],
             ignore_signal: options[:ignore_signal],
+            tags: tags,
+            propagate_tags: options[:propagate_tags],
           }.reject { |_, v| v.nil? }
-          if options[:cpu] || options[:memory]
-            container_definition_overrides = {cpu: options[:cpu], memory: options[:memory]}.reject { |_, v| v.nil? }
+          if options[:cpu] || options[:memory] || options[:working_directory]
+            container_definition_overrides = {cpu: options[:cpu], memory: options[:memory], working_directory: options[:working_directory]}.reject { |_, v| v.nil? }
           else
             container_definition_overrides = {}
           end

data/lib/wrapbox/version.rb

@@ -1,3 +1,3 @@
 module Wrapbox
-  VERSION = "0.9.0"
+  VERSION = "0.10.0"
 end

data/lib/wrapbox.rb

@@ -20,14 +20,19 @@ module Wrapbox
       yield configs
     end
 
-    def run(*args, runner: nil, config_name: nil, **options)
-      config = @configs.get(config_name)
-      config.run(*args, **options)
+    def run(*args, config_name: nil, **options)
+      get_config(config_name).run(*args, **options)
     end
 
-    def run_cmd(*args, runner: nil, config_name: nil, **options)
-      config = @configs.get(config_name)
-      config.run_cmd(*args, **options)
+    def run_cmd(*args, config_name: nil, **options)
+      get_config(config_name).run_cmd(*args, **options)
+    end
+
+    private
+
+    def get_config(config_name)
+      @configs.get(config_name) or
+        raise RuntimeError, %Q{The configuration "#{config_name}" is not registered}
     end
   end
 

data/wrapbox.gemspec

@@ -20,15 +20,17 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_runtime_dependency "aws-sdk-ec2", "~> 1"
   spec.add_runtime_dependency "aws-sdk-ecs", "~> 1"
   spec.add_runtime_dependency "aws-sdk-cloudwatch", "~> 1"
   spec.add_runtime_dependency "activesupport", ">= 4"
   spec.add_runtime_dependency "docker-api"
   spec.add_runtime_dependency "multi_json"
-  spec.add_runtime_dependency "thor"
+  spec.add_runtime_dependency "thor", ">= 1"
+  spec.add_runtime_dependency "thwait"
 
-  spec.add_development_dependency "bundler", "~> 1.13"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "webmock"
   spec.add_development_dependency "tapp"