wrapbox 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ec530a0b68981cc7a90bcd451c4a54d371d275dd
-  data.tar.gz: 17e6212341e23126993ebde697b9ddccf4ecc397
+  metadata.gz: c24c21fbb7bbac8241ec5750714e5ea7d28f4755
+  data.tar.gz: 29fe85200b1df44f85528bdda6eae40b67c23fe8
 SHA512:
-  metadata.gz: eccc15faf35450e79202e8b134b9fd304ca66bb3a8179e976b6e181f83cc69d2e4a613326ab2fd872eb268c9485dbc2eb7feb0bf2147985d4b913feaabda0523
-  data.tar.gz: 2f3623dbb0aa16b3b37b22e0e06e0eacf985f21be5c12b8d1bf217234c3c9e36773bf53cc53429f76b9ae4ff3a5401da89548369183750e1a1eedb8b7ab702af
+  metadata.gz: a34af96a5e4ffb23f48cb04bb19bd63baeb8ec6a5b17be701da6182becde100b1937680a34f0e890e859e0f9668b95fdfc37618cd482e4c00d147c8d6f2a74f0
+  data.tar.gz: df635e95edceff7169e4fa213dc57722698cf873f67e1e204e988ea0557a95454d103b7a6b0a7c0cca4cd10afee28628310be7c93f606994dfb1f2ec93b97719

data/README.md

@@ -24,7 +24,7 @@ Write config.yml
 
 ```yaml
 default:
-  cluster: ecsr-test
+  cluster: wrapbox
   runner: ecs
   region: ap-northeast-1
   container_definition:
@@ -40,6 +40,15 @@ docker:
     image: joker1007/wrapbox
     cpu: 512
     memory: 1024
+
+ecs2:
+  cluster: wrapbox
+  runner: ecs
+  region: ap-northeast-1
+  # Use already existing task definition
+  task_definition:
+    task_definition_name: foo_task:1
+    main_container_name: container_name_where_command_is_executed
 ```
 
 #### run by CLI
@@ -53,12 +62,14 @@ $ wrapbox ecs run_cmd -f config.yml \
 
 #### run by ruby
 
+Run `rake wrapbox:run` with `CLASS_NAME_ENV` and `METHOD_NAME_ENV` and `METHOD_ARGS_ENV`
+
 ```ruby
 Wrapbox.configure do |c|
   c.load_yaml(File.expand_path("../config.yml", __FILE__))
 end
 
-# runs TestJob#perform in ECS container
+# runs TestJob#perform("arg1", ["arg2", "arg3"]) in ECS container via `rake wrapbox:run`
 Wrapbox.run("TestJob", :perform, ["arg1", ["arg2", "arg3"]], environments: [{name: "RAILS_ENV", value: "development"}]) # use default config
 # runs TestJob#perform in local docker container (Use docker cli)
 Wrapbox.run("TestJob", :perform, ["arg1", ["arg2", "arg3"]], config_name: :docker, environments: [{name: "RAILS_ENV", value: "development"}]) # use docker config

data/lib/wrapbox/configuration.rb

@@ -12,9 +12,21 @@ module Wrapbox
     :retry_interval,
     :retry_interval_multiplier,
     :container_definition,
+    :container_definitions,
+    :volumes,
+    :placement_constraints,
+    :placement_strategy,
+    :launch_type,
+    :requires_compatibilities,
+    :task_definition,
     :additional_container_definitions,
+    :network_mode,
+    :network_configuration,
+    :cpu,
+    :memory,
     :task_role_arn,
-    :keep_container
+    :keep_container,
+    :log_fetcher
   ) do
     def self.load_config(config)
       new(
@@ -26,10 +38,22 @@ module Wrapbox
         config["retry"] || 0,
         config["retry_interval"] || 1,
         config["retry_interval_multiplier"] || 2,
-        config["container_definition"].deep_symbolize_keys,
-        config["additional_container_definitions"] || [],
+        config["container_definition"]&.deep_symbolize_keys,
+        config["container_definitions"]&.map(&:deep_symbolize_keys) || [],
+        config["volumes"]&.map(&:deep_symbolize_keys) || [],
+        config["placement_constraints"]&.map(&:deep_symbolize_keys) || [],
+        config["placement_strategy"]&.map(&:deep_symbolize_keys) || [],
+        config["launch_type"],
+        config["requires_compatibilities"] || ["EC2"],
+        config["task_definition"]&.deep_symbolize_keys,
+        config["additional_container_definitions"]&.map(&:deep_symbolize_keys) || [],
+        config["network_mode"],
+        config["network_configuration"]&.deep_symbolize_keys,
+        config["cpu"]&.to_s,
+        config["memory"]&.to_s,
         config["task_role_arn"],
-        config["keep_container"]
+        config["keep_container"],
+        config["log_fetcher"]&.deep_symbolize_keys
       )
     end
 
@@ -39,10 +63,11 @@ module Wrapbox
       super
     end
 
-    def build_runner
-      raise "#{runner} is unsupported runner" unless AVAILABLE_RUNNERS.include?(runner.to_sym)
-      require "wrapbox/runner/#{runner}"
-      Wrapbox::Runner.const_get(runner.to_s.camelcase).new(to_h)
+    def build_runner(overrided_runner = nil)
+      r = overrided_runner || runner
+      raise "#{r} is unsupported runner" unless AVAILABLE_RUNNERS.include?(r.to_sym)
+      require "wrapbox/runner/#{r}"
+      Wrapbox::Runner.const_get(r.to_s.camelcase).new(to_h)
     end
 
     def run(class_name, method_name, args, **options)

data/lib/wrapbox/log_fetcher/papertrail.rb

@@ -0,0 +1,75 @@
+module Wrapbox
+  module LogFetcher
+    class Papertrail
+      STOP_WAIT_TIMELIMIT = 10
+
+      def initialize(query: nil, delay: 2, **options)
+        begin
+          require 'papertrail/cli'
+        rescue LoadError
+          $stderr.puts "Require papertrail gem"
+          exit 1
+        end
+
+        # see. https://github.com/papertrail/papertrail-cli/blob/master/lib/papertrail/cli.rb
+        @query = query
+        @delay = delay
+        @options = options.reject { |_, v| v.nil? }
+      end
+
+      def run
+        @started_at = Time.now
+        @loop_thread = Thread.start(&method(:main_loop))
+      end
+
+      def stop
+        @stop = true
+        @loop_thread.join(STOP_WAIT_TIMELIMIT)
+      end
+
+      def main_loop
+        papertrail = ::Papertrail::Cli.new
+        connection_options = papertrail.options.merge(@options).merge(follow: true)
+        connection = ::Papertrail::Connection.new(connection_options)
+
+        query_options = {}
+
+        if @options[:system]
+          query_options[:system_id] = connection.find_id_for_source(@options[:system])
+          unless query_options[:system_id]
+            $stderr.puts "System \"#{@options[:system]}\" not found"
+          end
+        end
+
+        if @options[:group]
+          query_options[:group_id] = connection.find_id_for_group(@options[:group])
+          unless query_options[:group_id]
+            $stderr.puts "Group \"#{@options[:group]}\" not found"
+          end
+        end
+
+        if @options[:search]
+          search = connection.find_search(@options[:search], @query_options[:group_id])
+          unless search
+            $stderr.puts "Search \"#{@options[:search]}\" not found"
+          end
+
+          query_options[:group_id] ||= search['group_id']
+          @query = search['query']
+        end
+
+        @query ||= ''
+
+        search_query = ::Papertrail::SearchQuery.new(connection, @query, query_options)
+
+        until @stop do
+          search_query.next_results_page.events.each do |event|
+            next if event.received_at < @started_at
+            papertrail.display_result(event)
+          end
+          sleep @delay
+        end
+      end
+    end
+  end
+end

data/lib/wrapbox/log_fetcher.rb

@@ -0,0 +1,14 @@
+require "active_support/core_ext/string"
+
+# LogFetcher Implementation requires two methods.
+# - run (start log fetching asynchronously)
+# - stop (stop log fetching)
+module Wrapbox
+  module LogFetcher
+    def self.new(type, **options)
+      raise "log_fetcher config needs `type`" unless type
+      require "wrapbox/log_fetcher/#{type}"
+      self.const_get(type.classify).new(**options)
+    end
+  end
+end

data/lib/wrapbox/runner/docker.rb

@@ -2,6 +2,7 @@ require "open3"
 require "multi_json"
 require "docker"
 require "thor"
+require "shellwords"
 
 module Wrapbox
   module Runner
@@ -15,7 +16,14 @@ module Wrapbox
 
       def initialize(options)
         @name = options[:name]
-        @container_definition = options[:container_definition]
+        @container_definitions = options[:container_definition] ? [options[:container_definition]] : options[:container_definitions][0]
+
+        if @container_definitions.size >= 2
+          raise "Docker runner does not support multi container currently"
+        end
+
+        @container_definition = @container_definitions[0]
+
         @keep_container = options[:keep_container]
       end
 
@@ -35,10 +43,15 @@ module Wrapbox
 
         environments = extract_environments(environments)
 
+        cmds << nil if cmds.empty?
         ths = cmds.map.with_index do |cmd, idx|
           Thread.new(cmd, idx) do |c, i|
             envs = environments + ["WRAPBOX_CMD_INDEX=#{idx}"]
-            exec_docker(definition: definition, cmd: c.split(/\s+/), environments: envs)
+            exec_docker(
+              definition: definition,
+              cmd: c ? Shellwords.shellsplit(c) : nil,
+              environments: envs
+            )
           end
         end
         ths.each(&:join)
@@ -60,9 +73,8 @@ module Wrapbox
         ::Docker::Image.create("fromImage" => definition[:image])
         options = {
           "Image" => definition[:image],
-          "Cmd" => cmd,
           "Env" => environments,
-        }
+        }.tap { |o| o["Cmd"] = cmd if cmd }
         options["HostConfig"] = {}
         options["HostConfig"]["Cpu"] = definition[:cpu] if definition[:cpu]
         options["HostConfig"]["Memory"] = definition[:memory] * 1024 * 1024 if definition[:memory]

data/lib/wrapbox/runner/ecs.rb

@@ -5,8 +5,10 @@ require "yaml"
 require "active_support/core_ext/hash"
 require "logger"
 require "pp"
+require "shellwords"
 
 require "wrapbox/config_repository"
+require "wrapbox/log_fetcher"
 require "wrapbox/version"
 
 module Wrapbox
@@ -27,8 +29,18 @@ module Wrapbox
         :revision,
         :cluster,
         :region,
-        :container_definition,
-        :additional_container_definitions,
+        :container_definitions,
+        :volumes,
+        :placement_constraints,
+        :placement_strategy,
+        :launch_type,
+        :requires_compatibilities,
+        :task_definition_name,
+        :main_container_name,
+        :network_mode,
+        :network_configuration,
+        :cpu,
+        :memory,
         :task_role_arn
 
       def initialize(options)
@@ -36,11 +48,52 @@ module Wrapbox
         @revision = options[:revision]
         @cluster = options[:cluster]
         @region = options[:region]
-        @container_definition = options[:container_definition]
-        @additional_container_definitions = options[:additional_container_definitions]
+        @volumes = options[:volumes]
+        @placement_constraints = options[:placement_constraints]
+        @placement_strategy = options[:placement_strategy]
+        @launch_type = options[:launch_type]
+        @requires_compatibilities = options[:requires_compatibilities]
+        @network_mode = options[:network_mode]
+        @network_configuration = options[:network_configuration]
+        @cpu = options[:cpu]
+        @memory = options[:memory]
+
+        if (options[:container_definition] || options[:container_definitions].empty?.!) && options[:task_definition]
+          raise "Please set only one of `container_definition` and `task_definition`"
+        end
+
+        if options[:additional_container_definitions].empty?.!
+          warn "`additional_container_definitions` is deprecated parameter, Use `container_definitions` instead of it"
+        end
+
+        @container_definitions = options[:container_definition] ? [options[:container_definition]] : options[:container_definitions]
+        @container_definitions.concat(options[:additional_container_definitions]) # deprecated
+
+        @task_definition_info = options[:task_definition]
+
+        if !@container_definitions.empty?
+          @task_definition_name = "wrapbox_#{@name}"
+          @main_container_name = @container_definitions[0][:name] || @task_definition_name
+        elsif @task_definition_info
+          @task_definition_name = @task_definition_info[:task_definition_name]
+          @main_container_name = @task_definition_info[:main_container_name]
+          unless @main_container_name
+            raise "Please set `task_definition[:main_container_name]`"
+          end
+        end
+
+        @container_definitions.each do |d|
+          d[:docker_labels]&.stringify_keys!
+          d.dig(:log_configuration, :options)&.stringify_keys!
+        end
+
         @task_role_arn = options[:task_role_arn]
         $stdout.sync = true
         @logger = Logger.new($stdout)
+        if options[:log_fetcher]
+          type = options[:log_fetcher].delete(:type)
+          @log_fetcher = LogFetcher.new(type, options[:log_fetcher])
+        end
       end
 
       class Parameter
@@ -65,7 +118,7 @@ module Wrapbox
       end
 
       def run(class_name, method_name, args, container_definition_overrides: {}, **parameters)
-        task_definition = register_task_definition(container_definition_overrides)
+        task_definition = prepare_task_definition(container_definition_overrides)
         parameter = Parameter.new(**parameters)
 
         run_task(
@@ -76,15 +129,16 @@ module Wrapbox
       end
 
       def run_cmd(cmds, container_definition_overrides: {}, **parameters)
-        task_definition = register_task_definition(container_definition_overrides)
+        task_definition = prepare_task_definition(container_definition_overrides)
 
+        cmds << nil if cmds.empty?
         ths = cmds.map.with_index do |cmd, idx|
           Thread.new(cmd, idx) do |c, i|
             Thread.current[:cmd_index] = i
             envs = (parameters[:environments] || []) + [{name: "WRAPBOX_CMD_INDEX", value: i.to_s}]
             run_task(
               task_definition.task_definition_arn, nil, nil, nil,
-              c.split(/\s+/),
+              c ? Shellwords.shellsplit(c) : nil,
               Parameter.new(**parameters.merge(environments: envs))
             )
           end
@@ -94,12 +148,17 @@ module Wrapbox
 
       private
 
+      def use_existing_task_definition?
+        !!@task_definition_info
+      end
+
       def run_task(task_definition_arn, class_name, method_name, args, command, parameter)
         cl = parameter.cluster || self.cluster
         execution_try_count = 0
 
         begin
           task = create_task(task_definition_arn, class_name, method_name, args, command, parameter)
+          @log_fetcher.run if @log_fetcher
 
           @logger.debug("Launch Task: #{task.task_arn}")
 
@@ -129,13 +188,17 @@ module Wrapbox
             sleep EXECUTION_RETRY_INTERVAL
             retry
           end
+        ensure
+          if @log_fetcher
+            begin
+              @log_fetcher.stop
+            rescue => e
+              @logger.warn(e)
+            end
+          end
         end
       end
 
-      def task_definition_name
-        "wrapbox_#{name}"
-      end
-
       def create_task(task_definition_arn, class_name, method_name, args, command, parameter)
         cl = parameter.cluster || self.cluster
         args = Array(args)
@@ -210,19 +273,8 @@ module Wrapbox
         end
       end
 
-      def wait_task_running(cluster, task_arn, launch_timeout)
-        client.wait_until(:tasks_running, cluster: cluster, tasks: [task_arn]) do |w|
-          if launch_timeout
-            w.delay = WAIT_DELAY
-            w.max_attempts = launch_timeout / w.delay
-          else
-            w.max_attempts = nil
-          end
-        end
-      end
-
-      def wait_task_stopped(cluster, task_arn, timeout)
-        client.wait_until(:tasks_stopped, cluster: cluster, tasks: [task_arn]) do |w|
+      def wait_until_with_timeout(cluster, task_arn, timeout, waiter_name)
+        client.wait_until(waiter_name, cluster: cluster, tasks: [task_arn]) do |w|
           if timeout
             w.delay = WAIT_DELAY
             w.max_attempts = timeout / w.delay
@@ -230,18 +282,26 @@ module Wrapbox
             w.max_attempts = nil
           end
         end
+      end
+
+      def wait_task_running(cluster, task_arn, launch_timeout)
+        wait_until_with_timeout(cluster, task_arn, launch_timeout, :tasks_running)
+      end
+
+      def wait_task_stopped(cluster, task_arn, execution_timeout)
+        wait_until_with_timeout(cluster, task_arn, execution_timeout, :tasks_stopped)
       rescue Aws::Waiters::Errors::TooManyAttemptsError
         client.stop_task({
-          cluster: cl,
+          cluster: cluster,
           task: task_arn,
           reason: "process timeout",
         })
-        raise ExecutionTimeout, "Container #{task_definition_name} is timeout. task=#{task_arn}, timeout=#{timeout}"
+        raise ExecutionTimeout, "Task #{task_definition_name} is timeout. task=#{task_arn}, timeout=#{execution_timeout}"
       end
 
       def fetch_task_status(cluster, task_arn)
         task = client.describe_tasks(cluster: cluster, tasks: [task_arn]).tasks[0]
-        container = task.containers.find { |c| c.name == task_definition_name }
+        container = task.containers.find { |c| c.name == main_container_name }
         {
           last_status: task.last_status,
           exit_code: container.exit_code,
@@ -250,11 +310,21 @@ module Wrapbox
         }
       end
 
+      def prepare_task_definition(container_definition_overrides)
+        if use_existing_task_definition?
+          client.describe_task_definition(task_definition: task_definition_name).task_definition
+        else
+          register_task_definition(container_definition_overrides)
+        end
+      end
+
       def register_task_definition(container_definition_overrides)
-        definition = container_definition
+        main_container_definition = container_definitions[0]
+        main_container_definition = main_container_definition
           .merge(container_definition_overrides)
-          .merge(name: task_definition_name)
-        container_definitions = [definition, *additional_container_definitions]
+          .merge(name: main_container_name)
+
+        overrided_container_definitions = [main_container_definition, *(container_definitions.drop(1))]
 
         if revision
           begin
@@ -263,12 +333,17 @@ module Wrapbox
           end
         end
 
-        @logger.debug("Container Definitions: #{container_definitions}")
+        @logger.debug("Container Definitions: #{overrided_container_definitions}")
         register_retry_count = 0
         begin
           client.register_task_definition({
             family: task_definition_name,
-            container_definitions: container_definitions,
+            cpu: cpu,
+            memory: memory,
+            network_mode: network_mode,
+            container_definitions: overrided_container_definitions,
+            volumes: volumes,
+            requires_compatibilities: requires_compatibilities,
           }).task_definition
         rescue Aws::ECS::Errors::ClientException
           raise if register_retry_count > 2
@@ -331,10 +406,9 @@ module Wrapbox
         overrides = {
           container_overrides: [
             {
-              name: task_definition_name,
-              command: command,
+              name: main_container_name,
               environment: env,
-            },
+            }.tap { |o| o[:command] = command if command },
           ],
         }
         role_arn = task_role_arn || self.task_role_arn
@@ -344,12 +418,16 @@ module Wrapbox
           cluster: cluster || self.cluster,
           task_definition: task_definition_arn,
           overrides: overrides,
+          placement_strategy: placement_strategy,
+          placement_constraints: placement_constraints,
+          launch_type: launch_type,
+          network_configuration: network_configuration,
           started_by: "wrapbox-#{Wrapbox::VERSION}",
         }
       end
 
       def build_error_message(task_definition_name, task_arn, task_status)
-        error_message = "Container #{task_definition_name} is failed. task=#{task_arn}, "
+        error_message = "Task #{task_definition_name} is failed. task=#{task_arn}, "
         error_message << "cmd_index=#{Thread.current[:cmd_index]}, " if Thread.current[:cmd_index]
         error_message << "exit_code=#{task_status[:exit_code]}, task_stopped_reason=#{task_status[:stopped_reason]}, container_stopped_reason=#{task_status[:container_stopped_reason]}"
         error_message

data/lib/wrapbox/version.rb

@@ -1,3 +1,3 @@
 module Wrapbox
-  VERSION = "0.4.1"
+  VERSION = "0.5.0"
 end

data/wrapbox.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "aws-sdk", "~> 2.4"
+  spec.add_runtime_dependency "aws-sdk", "~> 2.10", ">= 2.10.109"
   spec.add_runtime_dependency "activesupport", ">= 4"
   spec.add_runtime_dependency "docker-api"
   spec.add_runtime_dependency "multi_json"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wrapbox
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - joker1007
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-07-27 00:00:00.000000000 Z
+date: 2017-12-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -16,14 +16,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.4'
+        version: '2.10'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.10.109
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.4'
+        version: '2.10'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.10.109
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -188,6 +194,8 @@ files:
 - lib/wrapbox/config_repository.rb
 - lib/wrapbox/configuration.rb
 - lib/wrapbox/job.rb
+- lib/wrapbox/log_fetcher.rb
+- lib/wrapbox/log_fetcher/papertrail.rb
 - lib/wrapbox/runner/docker.rb
 - lib/wrapbox/runner/ecs.rb
 - lib/wrapbox/tasks/run.rake
@@ -212,7 +220,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Ruby method runner on AWS ECS