wrapbox 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12b4d996dbd5a6dd05a700e8654c369b670246eaaed96514e9783e94be18a506
-  data.tar.gz: 06db857821782f32d3cbf3bbcd679182287735224c17256b85f57644b13ea69f
+  metadata.gz: 207f24c9cc5e7fc32e69652b7ff9f012b32da2203ac5c044b6adb755fa34ba6c
+  data.tar.gz: 6a1dcfd5cd2e09cba65449184f2d736074e5436419abe20b54dcb1117c8af81a
 SHA512:
-  metadata.gz: 05f3a8b158b5c54b0f010695fd020a3ea381140a88da9b57488cd469091f66da63215b36b6973e0736b3422200495a1f6dd1d199be07ccc94f32c985f3c17762
-  data.tar.gz: 10a3aecb71fb45e22b9ffe3123e592142ae4fab5ef5e70bb372c328f8c8b302ffcf8caac592305e9f3466e80570752421ed398936a5f3aabf5764faac40334ce
+  metadata.gz: a02a9ab56bf3f0d4bdd2d24fab5acef9d41a128bbdd477f0e4cab16f2a00d90cb20d23c93bc6ec5c7416faa059eacb3dd9a46a050559b994ce772b28f1dabaeb
+  data.tar.gz: c0e5595b908a11924854830867af67312f851430d4b5dadf058c273178d111d63d8164a4573efc3a20bbbff8b2f6386dfca8bcb1bd829d1624004b96aa87a853

data/lib/wrapbox.rb

@@ -4,6 +4,10 @@ module Wrapbox
   METHOD_ARGS_ENV = "WRAPBOX_METHOD_ARGS".freeze
 
   class << self
+    def load_config(filename)
+      configs.load_yaml(filename)
+    end
+
     def configs
       @configs ||= ConfigRepository.new
     end
@@ -12,12 +16,12 @@ module Wrapbox
       yield configs
     end
 
-    def run(*args, config_name: nil, **options)
+    def run(*args, runner: nil, config_name: nil, **options)
       config = @configs.get(config_name)
       config.run(*args, **options)
     end
 
-    def run_cmd(*args, config_name: nil, **options)
+    def run_cmd(*args, runner: nil, config_name: nil, **options)
       config = @configs.get(config_name)
       config.run_cmd(*args, **options)
     end

data/lib/wrapbox/cli.rb

@@ -1,4 +1,5 @@
 require "thor/group"
+require "wrapbox"
 require "wrapbox/runner/docker"
 require "wrapbox/runner/ecs"
 

data/lib/wrapbox/configuration.rb

@@ -4,6 +4,7 @@ require "active_support/core_ext/string"
 module Wrapbox
   Configuration = Struct.new(
     :name,
+    :task_definition_name,
     :revision,
     :runner,
     :cluster,
@@ -32,6 +33,7 @@ module Wrapbox
     def self.load_config(config)
       new(
         config["name"],
+        config["task_definition_name"],
         config["revision"],
         config["runner"] ? config["runner"].to_sym : :docker,
         config["cluster"],
@@ -72,12 +74,12 @@ module Wrapbox
       Wrapbox::Runner.const_get(r.to_s.camelcase).new(to_h)
     end
 
-    def run(class_name, method_name, args, **options)
-      build_runner.run(class_name, method_name, args, **options)
+    def run(class_name, method_name, args, runner: nil, **options)
+      build_runner(runner).run(class_name, method_name, args, **options)
     end
 
-    def run_cmd(*cmd, **options)
-      build_runner.run_cmd(*cmd, **options)
+    def run_cmd(*cmd, runner: nil, **options)
+      build_runner(runner).run_cmd(*cmd, **options)
     end
   end
 end

data/lib/wrapbox/log_fetcher.rb

@@ -8,7 +8,15 @@ module Wrapbox
     def self.new(type, **options)
       raise "log_fetcher config needs `type`" unless type
       require "wrapbox/log_fetcher/#{type}"
-      self.const_get(type.classify).new(**options)
+      self.const_get(type.camelize).new(**options)
+    end
+
+    def run(task:)
+      raise NotImplementedError
+    end
+
+    def stop
+      raise NotImplementedError
     end
   end
 end

data/lib/wrapbox/log_fetcher/awslogs.rb

@@ -0,0 +1,104 @@
+module Wrapbox
+  module LogFetcher
+    class Awslogs
+      STOP_WAIT_TIMELIMIT = 10
+
+      def initialize(log_group:, log_stream_prefix:, filter_pattern: nil, region: nil, access_key_id: nil, secret_access_key: nil, timestamp_format: "%Y-%m-%d %H:%M:%S.%3N", delay: 2, **options)
+        begin
+          require 'aws-sdk-cloudwatchlogs'
+        rescue LoadError
+          $stderr.puts "Require aws-sdk-cloudwatchlogs gem"
+          exit 1
+        end
+
+        @log_group = log_group
+        @log_stream_prefix = log_stream_prefix
+        @filter_pattern = filter_pattern
+        @region = region
+        @access_key_id = access_key_id
+        @secret_access_key = secret_access_key
+        @timestamp_format = timestamp_format
+        @delay = delay
+        @options = options.reject { |_, v| v.nil? }
+        @displayed_log_stream_names = {}
+        @displayed_log_stream_number = 0
+        @displayed_event_ids = {}
+      end
+
+      def run(task:)
+        @loop_thread = Thread.start do
+          main_loop(task)
+        end
+      end
+
+      def stop
+        @stop = true
+        @loop_thread&.join(STOP_WAIT_TIMELIMIT)
+      end
+
+      def main_loop(task)
+        task_id = task.task_arn.split("/").last
+        log_stream_names = task.containers.map do |container|
+          [@log_stream_prefix, container.name, task_id].join("/")
+        end
+        filter_log_opts = {
+          log_group_name: @log_group,
+          log_stream_names: log_stream_names,
+          filter_pattern: @filter_pattern,
+          interleaved: true,
+        }.compact
+        @max_timestamp = ((Time.now.to_f - 120) * 1000).round
+
+        until @stop do
+          filter_log_opts[:start_time] = @max_timestamp + 1
+          resp = client.filter_log_events(filter_log_opts) rescue nil
+          resp&.each do |r|
+            r.events.each do |ev|
+              next if @displayed_event_ids.member?(ev.event_id)
+              display_message(ev)
+              @displayed_event_ids[ev.event_id] = ev.timestamp
+              @max_timestamp = ev.timestamp if @max_timestamp < ev.timestamp
+            end
+          end
+          Thread.start do
+            @displayed_event_ids.each do |event_id, ts|
+              if ts < (Time.now.to_f - 600) * 1000
+                @displayed_event_ids.delete(event_id)
+              end
+            end
+          end.tap do
+            sleep @delay 
+          end.join
+        end
+      end
+
+      COLOR_ESCAPE_SEQUENCES = [33, 31, 32, 34, 35, 36]
+      def display_message(ev, output: $stdout)
+        num = @displayed_log_stream_names.fetch(ev.log_stream_name) do |key|
+          current = @displayed_log_stream_number
+          @displayed_log_stream_names[key] = current
+          @displayed_log_stream_number += 1
+          current
+        end
+
+        sequence_number = COLOR_ESCAPE_SEQUENCES[num % COLOR_ESCAPE_SEQUENCES.length]
+
+        time = Time.at(ev.timestamp / 1000.0)
+        output.puts("\e[#{sequence_number}m#{time.strftime(@timestamp_format)} #{ev.log_stream_name}\e[0m #{ev.message}")
+      end
+
+      private
+
+      def client
+        return @client if @client
+
+        options = {
+          region: @region,
+          access_key_id: @access_key_id,
+          secret_access_key: @secret_access_key,
+        }.compact
+        @client = Aws::CloudWatchLogs::Client.new(**options)
+      end
+    end
+  end
+end

data/lib/wrapbox/log_fetcher/papertrail.rb

@@ -17,7 +17,7 @@ module Wrapbox
         @options = options.reject { |_, v| v.nil? }
       end
 
-      def run
+      def run(task:)
         @started_at = Time.now
         @loop_thread = Thread.start(&method(:main_loop))
       end

data/lib/wrapbox/runner/docker.rb

@@ -139,10 +139,8 @@ module Wrapbox
         method_option :environments, aliases: "-e"
         method_option :ignore_signal, type: :boolean, default: false, desc: "Even if receive a signal (like TERM, INT, QUIT), Docker container continue running"
         def run_cmd(*args)
-          repo = Wrapbox::ConfigRepository.new.tap { |r| r.load_yaml(options[:config]) }
-          config = repo.get(options[:config_name])
-          config.runner = :docker
-          runner = config.build_runner
+          Wrapbox.load_config(options[:config])
+          config = Wrapbox.configs[options[:config_name]]
           environments = options[:environments].to_s.split(/,\s*/).map { |kv| kv.split("=") }.map do |k, v|
             {name: k, value: v}
           end
@@ -151,7 +149,7 @@ module Wrapbox
           else
             container_definition_overrides = {}
           end
-          unless runner.run_cmd(args, environments: environments, container_definition_overrides: container_definition_overrides, ignore_signal: options[:ignore_signal])
+          unless config.run_cmd(args, runner: "docker", environments: environments, container_definition_overrides: container_definition_overrides, ignore_signal: options[:ignore_signal])
             exit 1
           end
         end

data/lib/wrapbox/runner/ecs.rb

@@ -48,6 +48,7 @@ module Wrapbox
 
       def initialize(options)
         @name = options[:name]
+        @task_definition_name = options[:task_definition_name]
         @revision = options[:revision]
         @cluster = options[:cluster]
         @region = options[:region]
@@ -75,7 +76,7 @@ module Wrapbox
         @task_definition_info = options[:task_definition]
 
         if !@container_definitions.empty?
-          @task_definition_name = "wrapbox_#{@name}"
+          @task_definition_name ||= "wrapbox_#{@name}"
           @main_container_name = @container_definitions[0][:name] || @task_definition_name
         elsif @task_definition_info
           @task_definition_name = @task_definition_info[:task_definition_name]
@@ -107,6 +108,7 @@ module Wrapbox
           :execution_role_arn,
           :cluster,
           :timeout,
+          :launch_type,
           :launch_timeout,
           :launch_retry,
           :retry_interval,
@@ -114,7 +116,7 @@ module Wrapbox
           :max_retry_interval,
           :execution_retry
 
-        def initialize(environments: [], task_role_arn: nil, cluster: nil, timeout: 3600 * 24, launch_timeout: 60 * 10, launch_retry: 10, retry_interval: 1, retry_interval_multiplier: 2, max_retry_interval: 120, execution_retry: 0)
+        def initialize(environments: [], task_role_arn: nil, cluster: nil, timeout: 3600 * 24, launch_type: "EC2", launch_timeout: 60 * 10, launch_retry: 10, retry_interval: 1, retry_interval_multiplier: 2, max_retry_interval: 120, execution_retry: 0)
           b = binding
           method(:initialize).parameters.each do |param|
             instance_variable_set("@#{param[1]}", b.local_variable_get(param[1]))
@@ -186,8 +188,6 @@ module Wrapbox
           task = create_task(task_definition_arn, class_name, method_name, args, command, parameter)
           return unless task # only Task creation aborted by SignalException
 
-          @log_fetcher.run if @log_fetcher
-
           @logger.debug("Launch Task: #{task.task_arn}")
 
           wait_task_stopped(cl, task.task_arn, parameter.timeout)
@@ -237,16 +237,18 @@ module Wrapbox
 
       def create_task(task_definition_arn, class_name, method_name, args, command, parameter)
         cl = parameter.cluster || self.cluster
+        launch_type = parameter.launch_type || self.launch_type
         args = Array(args)
 
         launch_try_count = 0
         current_retry_interval = parameter.retry_interval
 
         begin
-          run_task_options = build_run_task_options(task_definition_arn, class_name, method_name, args, command, cl, parameter.environments, parameter.task_role_arn)
+          run_task_options = build_run_task_options(task_definition_arn, class_name, method_name, args, command, cl, launch_type, parameter.environments, parameter.task_role_arn)
           @logger.debug("Task Options: #{run_task_options}")
           resp = client.run_task(run_task_options)
           task = resp.tasks[0]
+          @log_fetcher.run(task: task) if @log_fetcher
 
           resp.failures.each do |failure|
             @logger.debug("Failure: Arn=#{failure.arn}, Reason=#{failure.reason}")
@@ -438,7 +440,7 @@ module Wrapbox
         )
       end
 
-      def build_run_task_options(task_definition_arn, class_name, method_name, args, command, cluster, environments, task_role_arn)
+      def build_run_task_options(task_definition_arn, class_name, method_name, args, command, cluster, launch_type, environments, task_role_arn)
         env = environments
         env += [
           {
@@ -502,16 +504,15 @@ module Wrapbox
         method_option :environments, aliases: "-e"
         method_option :task_role_arn
         method_option :timeout, type: :numeric
+        method_option :launch_type, type: :string, default: "EC2", enum: ["EC2", "FARGATE"]
         method_option :launch_timeout, type: :numeric
         method_option :launch_retry, type: :numeric
         method_option :execution_retry, type: :numeric
         method_option :max_retry_interval, type: :numeric
         method_option :ignore_signal, type: :boolean, default: false, desc: "Even if receive a signal (like TERM, INT, QUIT), ECS Tasks continue running"
         def run_cmd(*args)
-          repo = Wrapbox::ConfigRepository.new.tap { |r| r.load_yaml(options[:config]) }
-          config = repo.get(options[:config_name])
-          config.runner = :ecs
-          runner = config.build_runner
+          Wrapbox.load_config(options[:config])
+          config = Wrapbox.configs[options[:config_name]]
           environments = options[:environments].to_s.split(/,\s*/).map { |kv| kv.split("=") }.map do |k, v|
             {name: k, value: v}
           end
@@ -519,6 +520,7 @@ module Wrapbox
             cluster: options[:cluster],
             task_role_arn: options[:task_role_arn],
             timeout: options[:timeout],
+            launch_type: options[:launch_type],
             launch_timeout: options[:launch_timeout],
             launch_retry: options[:launch_retry],
             execution_retry: options[:execution_retry],
@@ -530,7 +532,7 @@ module Wrapbox
           else
             container_definition_overrides = {}
           end
-          unless runner.run_cmd(args, environments: environments, container_definition_overrides: container_definition_overrides, **run_options)
+          unless config.run_cmd(args, runner: "ecs", environments: environments, container_definition_overrides: container_definition_overrides, **run_options)
             exit 1
           end
         end

data/lib/wrapbox/version.rb

@@ -1,3 +1,3 @@
 module Wrapbox
-  VERSION = "0.7.0"
+  VERSION = "0.8.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wrapbox
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - joker1007
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-07-10 00:00:00.000000000 Z
+date: 2018-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-ecs
@@ -203,6 +203,7 @@ files:
 - lib/wrapbox/configuration.rb
 - lib/wrapbox/job.rb
 - lib/wrapbox/log_fetcher.rb
+- lib/wrapbox/log_fetcher/awslogs.rb
 - lib/wrapbox/log_fetcher/papertrail.rb
 - lib/wrapbox/runner/docker.rb
 - lib/wrapbox/runner/ecs.rb