kobanzame 0.0.8

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 159e7d623735c2eecf51458f18b1bbfe0c5b2f3588693ca23876710265691e85
+  data.tar.gz: 2546fea926c29d499b6ba7e22f2e3ee1a106c137059f307f67636129182a38d2
+SHA512:
+  metadata.gz: d01b930e5042038c7f2809bd584e704802bdf5feb1ddfccb0306e93126ae26b71f91356d94c26a9714c2da50041d9cd687810e22b1d3f6525b0227d06ba31d1e
+  data.tar.gz: 91a656c17d523b5faf2f917dc8679af62e06206cee035000786999569f084e8855d087bb5f3a879e2263f85a01d6c0b5e2bedd7dec31cc54c3170fcff87f4100

data/.circleci/config.yml

@@ -0,0 +1,29 @@
+version: 2
+
+jobs:
+  build:
+    docker:
+      # specify the version you desire here
+      - image: cimg/ruby:2.6.5
+    working_directory: ~/repo
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "Gemfile.lock" }}
+            # fallback to using the latest cache if no exact match is found
+            - v1-dependencies-
+      - run:
+          name: install dependencies
+          command: |
+            gem install bundler -v '1.17'
+            bundle install --jobs=4 --retry=3 --path vendor/bundle
+      - save_cache:
+          paths:
+            - ./vendor/bundle
+          key: v1-dependencies-{{ checksum "Gemfile.lock" }}
+      # run tests!
+      - run:
+          name: Run tests
+          command: |
+            bundle exec rake spec

data/.gitignore

@@ -0,0 +1,18 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/docs/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+*.pid
+*.log
+.envrc
+.ruby-version
+vendor/
+bk/

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in kobanzame.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,57 @@
+PATH
+  remote: .
+  specs:
+    kobanzame (0.0.8)
+      aws-sdk-cloudwatch
+      aws-sdk-cloudwatchlogs
+      serverengine
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws-eventstream (1.1.0)
+    aws-partitions (1.329.0)
+    aws-sdk-cloudwatch (1.39.1)
+      aws-sdk-core (~> 3, >= 3.99.0)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-cloudwatchlogs (1.32.1)
+      aws-sdk-core (~> 3, >= 3.99.0)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-core (3.99.2)
+      aws-eventstream (~> 1, >= 1.0.2)
+      aws-partitions (~> 1, >= 1.239.0)
+      aws-sigv4 (~> 1.1)
+      jmespath (~> 1.0)
+    aws-sigv4 (1.1.4)
+      aws-eventstream (~> 1.0, >= 1.0.2)
+    diff-lcs (1.3)
+    jmespath (1.4.0)
+    rake (13.0.1)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    serverengine (2.2.1)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (>= 1.9, < 3.0)
+  kobanzame!
+  rake (>= 12.3.3)
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   1.17.2

data/README.md

@@ -0,0 +1,64 @@
+# Kobanzame [![CircleCI](https://circleci.com/gh/inokappa/kobanzame.svg?style=svg)](https://circleci.com/gh/inokappa/kobanzame)
+
+![](https://raw.githubusercontent.com/inokappa/kobanzame/master/docs/images/fish_kobanzame.png)
+
+## About
+
+Resource monitoring tool for ECS Task. It sticks to a Docker container like a `Kobanzame (black shark)` and gets information about the container.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'kobanzame'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install kobanzame
+
+## Usage
+
+Write kobanzame.json.
+
+```json
+{
+  "container": {
+    "name": "batch-worker",
+    "check_interval": 1,
+    "report_format": "text"
+  },
+  "metrics": {
+      "name": "cloudwatch",
+      "namespace": "Custom/Kobanzame"
+  },
+  "outputs": [
+    {
+      "name": "cloudwatch_logs",
+      "log_group_name": "kobanzame-sample",
+      "log_stream_prefix": "kobanzame"
+    }
+  ]
+}
+```
+
+Starting kobanzame.
+
+```sh
+$ kobanzame --config=kobanzame.json
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/kobanzame.

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "kobanzame"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/kobanzame

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'kobanzame'
+require_relative '../lib/kobanzame/cli'

data/kobanzame.gemspec

@@ -0,0 +1,29 @@
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'kobanzame/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'kobanzame'
+  spec.version       = Kobanzame::VERSION
+  spec.authors       = ['inokappa']
+  spec.email         = ['inokara@gmail.com']
+  spec.summary       = %q{kobanzame collects resources for ecs task.}
+  spec.description   = %q{kobanzame collects resources for ecs task.}
+  spec.homepage      = 'https://github.com/inokappa/kobanzame'
+
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|example|docs)/}) }
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '>= 1.9', '< 3.0'
+  spec.add_development_dependency 'rake', '>= 12.3.3'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+
+  spec.add_runtime_dependency 'aws-sdk-cloudwatchlogs'
+  spec.add_runtime_dependency 'aws-sdk-cloudwatch'
+  spec.add_runtime_dependency 'serverengine'
+end

data/kobanzame.sample.json

@@ -0,0 +1,26 @@
+{
+  "container": {
+    "name": "batch-worker",
+    "check_interval": 1,
+    "report_format": "json"
+  },
+  "metrics": {
+      "name": "cloudwatch",
+      "namespace": "Custom/Kobanzame"
+  },
+  "outputs": [
+    {
+      "name": "cloudwatch_logs",
+      "log_group_name": "kobanzame-sample",
+      "log_stream_prefix": "kobanzame",
+      "log_stream_name": "batch-worker-resource-statistics"
+    },
+    {
+      "name": "slack",
+      "webhook_url": "",
+      "title": "Kobanzame Sample Output",
+      "user_name": "kobanzame-sample",
+      "icon_emoji": ":shark:"
+    }
+  ]
+}

data/lib/kobanzame.rb

@@ -0,0 +1,16 @@
+require 'net/http'
+require 'serverengine'
+require 'kobanzame/config'
+require 'kobanzame/ext'
+require 'kobanzame/metrics'
+require 'kobanzame/outputs'
+require 'kobanzame/report'
+require 'kobanzame/supervisor'
+require 'kobanzame/utilities'
+require 'kobanzame/version'
+require 'kobanzame/worker'
+
+module Kobanzame
+  class Error < StandardError; end
+  # Your code goes here...
+end

data/lib/kobanzame/cli.rb

@@ -0,0 +1,44 @@
+require 'optparse'
+require 'kobanzame'
+
+op = OptionParser.new
+op.version = Kobanzame::VERSION
+
+opts = {
+  config: 'kobanzame.json',
+  pid_file: 'kobanzame.pid',
+  log_file: STDOUT,
+  daemonize: false,
+  debug: false
+}
+
+op.on('-c', '--config PATH', "config file path (default: #{opts[:config]})") {|v|
+  opts[:config] = v
+}
+
+op.on('-d', '--daemonize', "enable daemonize (default: #{opts[:daemonize]})") {|v|
+  opts[:daemonize] = v
+}
+
+op.on('-p', '--pid-file PATH', "pid file path (default: #{opts[:pid_file]})") {|v|
+  opts[:pid_file] = v
+}
+
+op.on('-l', '--log-file PATH', "log file path (default: STDOUT)") {|v|
+  opts[:log_file] = v
+}
+
+op.on('-D', '--debug', "start with debug mode (default: #{opts[:debug]})") {|v|
+  opts[:debug] = v
+}
+
+begin
+  op.parse(ARGV)
+rescue OptionParser::InvalidOption => ex
+  puts op.to_s
+  puts "error: #{ex.message}"
+  exit 1
+end
+
+k = Kobanzame::Supervisor.new(opts)
+k.start

data/lib/kobanzame/config.rb

@@ -0,0 +1,40 @@
+module Kobanzame
+  class Config
+    def self.load_config(opts)
+      cfg = read_json(opts[:config])
+      se_config = { 
+        debug: opts[:debug],
+        daemonize: opts[:daemonize],
+        pid_path: opts[:pid_file],
+        log: opts[:log_file],
+        log_level: 'debug',
+        kobanzame_config: cfg
+      }
+      se_config
+    end
+
+    def self.read_json(path)
+      config_hash = ''
+      File.open(path) do |file|
+        config_hash = JSON.load(file)
+      end
+      config_hash
+    end
+
+    def self.select_conf(config, key)
+      konf = config[:kobanzame_config]
+      konf[key]
+    end
+
+    def self.generate_task_params(container_info)
+      params = {}
+      params['task_id'] = (0...32).map{ (65 + rand(26)).chr }.join
+      if container_info.dig('Labels', 'com.amazonaws.ecs.task-arn')
+        params['task_id'] = container_info.dig('Labels', 'com.amazonaws.ecs.task-arn').split('/')[-1]
+      end
+      params['docker_id'] = container_info.dig('DockerId')
+      params['docker_name'] = container_info.dig('DockerName')
+      params
+    end
+  end
+end

data/lib/kobanzame/ext.rb

@@ -0,0 +1,11 @@
+class Array
+  def average
+    (self.inject(:+) / self.length).round(3)
+  end
+end
+
+class String
+  def camelize
+    self.split('_').map(&:capitalize).join
+  end
+end

data/lib/kobanzame/metrics.rb

@@ -0,0 +1,38 @@
+module Kobanzame
+  class Metrics
+    def initialize(conf, params)
+      @metrics_conf = Kobanzame::Config.select_conf(conf, 'metrics') 
+      @params = params
+      @conf = conf
+    end
+
+    def logger
+      @logger ||= ServerEngine::DaemonLogger.new(@conf[:log] || STDOUT, @conf)
+    end
+
+    def publish(result)
+      load_output_modules.each do |mod|
+        begin
+          require "kobanzame/metrics/#{mod}"
+          class_name = "Kobanzame::Metric::#{mod.camelize}"
+          Object.const_get(class_name).new(@metrics_conf, @params).publish(result)
+          logger.info("Sent information to #{mod}")
+        rescue LoadError => ex
+          logger.warn("Could not load #{mod} module.")
+        rescue => ex
+          logger.warn("Could not send information to #{mod}. Reason: #{ex.message}")
+        end
+      end
+    end
+
+    private
+
+    def load_output_modules
+      modules = []
+      Dir.glob(File.dirname(__FILE__) + '/metrics/*').each do |r|
+        modules << File.basename(r, '.rb') unless r.include?('stub')
+      end
+      modules.sort
+    end
+  end
+end

data/lib/kobanzame/metrics/cloudwatch.rb

@@ -0,0 +1,70 @@
+require 'time'
+require 'aws-sdk-cloudwatch'
+
+module Kobanzame
+  module Metric
+    class Cloudwatch
+      def initialize(metric_conf, params)
+        @namespace = metric_conf['namespace']
+        @task_id = params['task_id']
+      end
+
+      def cw
+        @cw ||= Aws::CloudWatch::Client.new
+      end
+
+      def publish(result)
+        data = write_metric_data(result)
+        begin
+          cw.put_metric_data(data)
+        rescue Aws::CloudWatch::Errors::ServiceError
+          raise $!.message
+        end
+      end
+
+      private
+
+      # Input Data: [[timestamp, CPU Usage, Memory Used], [1591710550364, 0.0, 0.621], [1591710553452, 0.0, 0.621]]       
+      def write_metric_data(datas)
+        metric = {}
+        metric_datas = []
+        metric[:namespace] = @namespace
+        datas.each do |data|
+          metric_data = [
+            {
+              metric_name: 'CPU Usage',
+              dimensions: dimentions,
+              timestamp: timestamp(data[0]),
+              value: data[1],
+              unit: 'Percent'
+            },
+            {
+              metric_name: 'Memory Used',
+              dimensions: dimentions,
+              timestamp: timestamp(data[0]),
+              value: data[2],
+              unit: 'Megabits'
+            }
+          ]
+          metric_datas << metric_data
+        end
+        metric[:metric_data] = metric_datas.flatten
+        metric
+      end
+
+      def dimentions
+        dims = [
+          {
+            name: 'TaskId',
+            value: @task_id,
+          }
+        ]
+        dims
+      end
+
+      def timestamp(t)
+        DateTime.strptime((t.to_i/1000).to_s, '%s').iso8601
+      end
+    end
+  end
+end

data/lib/kobanzame/outputs.rb

@@ -0,0 +1,44 @@
+module Kobanzame
+  class Outputs 
+    def initialize(conf, params)
+      @outputs_conf = Kobanzame::Config.select_conf(conf, 'outputs') 
+      @container_conf = Kobanzame::Config.select_conf(conf, 'container') 
+      @params = params
+      @conf = conf
+    end
+
+    def logger
+      @logger ||= ServerEngine::DaemonLogger.new(@conf[:log] || STDOUT, @conf)
+    end
+
+    def publish(result)
+      load_output_modules.each do |mod|
+        output_conf = @outputs_conf.select { |c| c['name'] == mod }.last
+        if output_conf.nil?
+          logger.warn("Skip output to #{mod}")
+          next
+        end
+        begin
+          require "kobanzame/outputs/#{mod}"
+          class_name = "Kobanzame::Output::#{mod.camelize}"
+          Object.const_get(class_name).new(output_conf, @container_conf, @params).publish(result)
+          logger.info("Sent information to #{mod}")
+        rescue LoadError
+          logger.warn("Could not load #{mod} module.")
+        rescue => ex
+          logger.warn("Could not send information to #{mod}. Reason: #{ex.message}")
+        end
+      end
+    end
+
+    private
+
+    def load_output_modules
+      modules = []
+      Dir.glob(File.dirname(__FILE__) + '/outputs/*').each do |r|
+        modules << File.basename(r, '.rb') unless r.include?('stub')
+      end
+      modules.sort
+    end
+  end
+end

data/lib/kobanzame/outputs/cloudwatch_logs.rb

@@ -0,0 +1,72 @@
+require 'aws-sdk-cloudwatchlogs'
+
+module Kobanzame
+  module Output
+    class CloudwatchLogs
+      def initialize(output_conf, container_conf, params)
+        @log_group_name = output_conf['log_group_name']
+        @log_stream_prefix = output_conf['log_stream_prefix']
+        @log_stream_name = output_conf['log_stream_name']
+        @container_name = container_conf['name']
+        @format = container_conf['report_format']
+        @task_id = params['task_id']
+      end
+
+      def cwlog
+        @cwlog ||= Aws::CloudWatchLogs::Client.new
+      end
+
+      def publish(result)
+        event = {
+          log_group_name: @log_group_name,
+          log_stream_name: log_stream_name,
+          log_events: [{
+            timestamp: (Time.now.utc.to_f.round(3) * 1000).to_i,
+            message: result.send(@format)
+          }]
+        }
+        event['sequence_token'] = upload_sequence_token if upload_sequence_token != ''
+        begin
+          cwlog.put_log_events(event)
+        rescue Aws::CloudWatchLogs::Errors::ServiceError
+          raise $!.message
+        end
+      end
+
+      private
+
+      def log_stream_name
+        return @task_id if @log_stream_prefix.nil? && @log_stream_name.nil?
+        return @log_stream_name if @log_stream_prefix.nil?
+        @log_stream_prefix + '/' + @container_name + '/' + @task_id
+      end
+
+      def upload_sequence_token
+        return '' if log_streams.empty?
+        log_streams[0]['upload_sequence_token']
+      end
+
+      def log_streams
+        begin
+          res = cwlog.describe_log_streams({
+            log_group_name: @log_group_name,
+            log_stream_name_prefix: log_stream_name
+          })
+        rescue Aws::CloudWatchLogs::Errors::ServiceError
+          raise $!.message
+        end
+
+        begin
+          cwlog.create_log_stream({
+            log_group_name: @log_group_name,
+            log_stream_name: log_stream_name
+          })
+          return []
+        rescue Aws::CloudWatchLogs::Errors::ServiceError
+          raise $!.message
+        end if res['log_streams'].empty?
+        res['log_streams']
+      end
+    end
+  end
+end

data/lib/kobanzame/outputs/slack.rb

@@ -0,0 +1,74 @@
+require 'json'
+require 'net/http'
+
+module Kobanzame
+  module Output
+    class Slack
+      def initialize(output_conf, container_conf, params)
+        @format = container_conf['report_format']
+        @output_conf = output_conf
+      end
+
+      def publish(result)
+        attachements = build_attachements(result.send(@format))
+        params = build_params(attachements)
+        send(params)
+      end
+
+      private
+
+      def build_attachements(result)
+        result = build_pretty_json(result) if result.start_with?('{')
+        value = <<-"VALUE"
+```
+#{result}
+```
+        VALUE
+        attachments = [
+          {
+            'fallback': '',
+            'title': @output_conf['title'],
+            'fields': [
+              {
+                'title': 'Output',
+                'value': value,
+                'short': false
+              },
+            ]
+          }
+        ]
+        attachments
+      end
+
+      def build_params(attachments)
+        params = {
+          username: @output_conf['user_name'],
+          icon_emoji: @output_conf['icon_emoji'] ,
+          text: '',
+          attachments: attachments
+        }
+        params
+      end
+
+      def build_pretty_json(json_string)
+        hash = JSON.parse(json_string)
+        JSON.pretty_generate(hash)
+      end
+
+      def send(build_params)
+        uri = URI.parse(ENV['SLACK_WEBHOOK_URL'])
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+        begin
+          http.start do
+            request = Net::HTTP::Post.new(uri.path)
+            request.set_form_data(payload: build_params.to_json)
+            http.request(request)
+          end
+        rescue => e
+          raise $!.message
+        end
+      end
+    end
+  end
+end

data/lib/kobanzame/outputs/stdout.rb

@@ -0,0 +1,13 @@
+module Kobanzame
+  module Output
+    class Stdout < Outputs
+      def initialize(output_conf, container_conf, params)
+        @format = container_conf['report_format']
+      end
+
+      def publish(result)
+        $stdout.puts result.send(@format)
+      end
+    end
+  end
+end

data/lib/kobanzame/report.rb

@@ -0,0 +1,67 @@
+module Kobanzame
+  class Report
+    def initialize(result, params)
+      @result = result
+      @params = params
+    end
+
+    def duration
+      times = @result.map { |a| a[0] }
+      return '' if times.nil?
+      duration = {}
+      duration['duration'] = (times[-1] - times[0])
+      duration['unit'] = 'ms'
+      duration
+    end
+
+    def cpu_usage
+      cpus = @result.map { |a| a[1] }
+      return '' if cpus.nil?
+      usage = {}
+      usage['average'] = cpus.average
+      usage['max'] = cpus.max
+      usage['unit'] = '%'
+      usage 
+    end
+
+    def memory_used
+      memories = @result.map { |a| a[2] }
+      return '' if memories.nil?
+      usage = {}
+      usage['average'] = memories.average
+      usage['max'] = memories.max
+      usage['unit'] = 'MiB'
+      usage
+    end
+
+    def text
+      m = "Memory Used: #{memory_used['average']}MiB(ave)/#{memory_used['max']}MiB(max)"
+      c = "CPU Usage: #{cpu_usage['average']}%(ave)/#{cpu_usage['max']}%(max)"
+      task_id = @params['task_id']
+    
+      "REPORT TaskID: #{task_id} Duration: #{duration['duration']}ms, #{m}, #{c}"
+    end
+
+    def json
+      report = {}
+      
+      report['duration'] = duration
+      report['memory_used'] = memory_used
+      report['cpu_usage'] = cpu_usage
+
+      res = {}
+      res['report'] = report
+      res['task_id'] = @params['task_id']
+      res['docker_name'] = @params['docker_name']
+      res.to_json
+    end
+
+    def csv
+      m = "#{memory_used['average']},#{memory_used['max']}"
+      c = "#{cpu_usage['average']},#{cpu_usage['max']}"
+      task_id = @params['task_id']
+    
+      "#{task_id},#{duration['duration']},#{m},#{c}"
+    end
+  end
+end

data/lib/kobanzame/supervisor.rb

@@ -0,0 +1,14 @@
+module Kobanzame
+  class Supervisor
+    def initialize(opts)
+      @opts = opts
+    end
+
+    def start
+      se = ServerEngine.create(nil, 
+                               Kobanzame::Worker,
+                               Kobanzame::Config.load_config(@opts))
+      se.run
+    end
+  end
+end

data/lib/kobanzame/utilities.rb

@@ -0,0 +1,13 @@
+module Kobanzame
+  class Utilities
+    def self.request(path)
+      url = URI.parse(ENV['ECS_CONTAINER_METADATA_URI'] + '/' + path)
+      req = Net::HTTP::Get.new(url.path)
+      res = Net::HTTP.start(url.host, url.port) {|http|
+        http.request(req)
+      }
+      return nil unless res.code == '200'
+      JSON.parse(res.body)
+    end
+  end
+end

data/lib/kobanzame/version.rb

@@ -0,0 +1,3 @@
+module Kobanzame
+  VERSION = '0.0.8'
+end

data/lib/kobanzame/worker.rb

@@ -0,0 +1,113 @@
+module Kobanzame
+  module Worker
+    def calc_cpu_percent(current, previous, interval = 1000000000)
+      # to Percent (x.x %)
+      # refer to: https://github.com/mackerelio/mackerel-container-agent/blob/ce7cb283a3cdbc690127c6f67c44bdce7b140619/platform/ecs/metric.go#L112-L115
+      ((current['cpu_usage']['total_usage'].to_f - previous['cpu_usage']['total_usage'].to_f) / interval.to_f * 100).round(3)
+    end
+
+    def calc_memory_size(memory_stat)
+      # to Megabyte (x.xx MiB)
+      # refer to: https://github.com/mackerelio/mackerel-container-agent/blob/ce7cb283a3cdbc690127c6f67c44bdce7b140619/platform/ecs/metric.go#L117-L119
+      ((memory_stat['usage'] - memory_stat['stats']['cache']).to_f / (1024.0 * 1024.0)).round(3)
+    end
+
+    def describe_target_container
+      # return Container Info
+      res = Kobanzame::Utilities.request('task')
+      return nil if res.nil?
+      res
+    end
+
+    def describe_target_container_metrics(container_id)
+      stat = nil
+      res = Kobanzame::Utilities.request('task/stats')
+      stat = res.dig(container_id) unless res.nil?
+    
+      return nil if stat.nil? || stat.dig('memory_stats').empty?
+      memory_usage = calc_memory_size(stat['memory_stats'])
+      cpu_usage = calc_cpu_percent(stat['cpu_stats'], stat['precpu_stats'])
+      
+      # Return format: ['Datetime', 'Used CPU unit', 'Used Memory size']
+      [Time.now.strftime('%s%L').to_i, cpu_usage, memory_usage]
+    end
+
+    def load_container_config
+      Kobanzame::Config.select_conf(config, 'container')
+    end
+
+    def load_metrics_config
+      Kobanzame::Config.select_conf(config, 'metrics')
+    end
+
+    def describe_container_info
+      cf = load_container_config
+      logger.info 'Waiting for starting target container.'
+      i = 1
+      container_info = ''
+      loop do
+        containers = describe_target_container
+        logger.warn 'Could not get all container information.' if containers.nil?
+        container_info = containers['Containers'].find { |c| c['Name'] == cf['name'] }
+        break if !container_info.nil? && container_info['DockerId'] != ''
+        # waiting for 120s
+        i += 1
+        if i > 5 then
+          logger.fatal 'Could not get the information of the batch container.'
+          exit 1
+        end
+        sleep cf['check_interval'].to_i
+      end
+      container_info['_check_interval'] = cf['check_interval'].to_i
+      container_info
+    end
+
+    def run
+      container_info = describe_container_info
+      mt = load_metrics_config
+      params = Kobanzame::Config.generate_task_params(container_info)
+      logger.info 'Collecting target container metrics.'
+
+      i = 1
+      result = []
+      nilbox = []
+      metbox = []
+      until @stop
+        metrics = describe_target_container_metrics(container_info.dig('DockerId')) 
+        # metrics が 3 回取れなかったら, ループを抜ける
+        nilbox << nil if metrics.nil?
+        if nilbox.size > 3
+          if !mt.nil? || !mt == {}
+            Kobanzame::Metrics.new(config, params).publish(metbox) unless metbox.empty?
+          end
+          break
+        end
+        #
+        if !metrics.nil?
+          result << metrics
+          metbox << metrics
+        end
+        logger.debug metrics if config[:debug]
+        #
+        if i % 10 == 0
+          logger.info "Collected #{result.length} metrics..."
+          if !mt.nil? || !mt == {}
+            Kobanzame::Metrics.new(config, params).publish(metbox)
+            metbox = []
+          end
+        end
+        i += 1
+        #
+        sleep container_info['_check_interval']
+      end
+
+      repo = Kobanzame::Report.new(result, params)
+      Kobanzame::Outputs.new(config, params).publish(repo)
+      exit 0
+    end
+
+    def stop
+      @stop = true
+    end
+  end
+end

metadata

@@ -0,0 +1,160 @@
+--- !ruby/object:Gem::Specification
+name: kobanzame
+version: !ruby/object:Gem::Version
+  version: 0.0.8
+platform: ruby
+authors:
+- inokappa
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2020-06-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.9'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.9'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.3.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.3.3
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-cloudwatchlogs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-cloudwatch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: serverengine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: kobanzame collects resources for ecs task.
+email:
+- inokara@gmail.com
+executables:
+- kobanzame
+extensions: []
+extra_rdoc_files: []
+files:
+- ".circleci/config.yml"
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/kobanzame
+- kobanzame.gemspec
+- kobanzame.sample.json
+- lib/kobanzame.rb
+- lib/kobanzame/cli.rb
+- lib/kobanzame/config.rb
+- lib/kobanzame/ext.rb
+- lib/kobanzame/metrics.rb
+- lib/kobanzame/metrics/cloudwatch.rb
+- lib/kobanzame/outputs.rb
+- lib/kobanzame/outputs/cloudwatch_logs.rb
+- lib/kobanzame/outputs/slack.rb
+- lib/kobanzame/outputs/stdout.rb
+- lib/kobanzame/report.rb
+- lib/kobanzame/supervisor.rb
+- lib/kobanzame/utilities.rb
+- lib/kobanzame/version.rb
+- lib/kobanzame/worker.rb
+homepage: https://github.com/inokappa/kobanzame
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: kobanzame collects resources for ecs task.
+test_files: []