kobanzame 0.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 159e7d623735c2eecf51458f18b1bbfe0c5b2f3588693ca23876710265691e85
4
+ data.tar.gz: 2546fea926c29d499b6ba7e22f2e3ee1a106c137059f307f67636129182a38d2
5
+ SHA512:
6
+ metadata.gz: d01b930e5042038c7f2809bd584e704802bdf5feb1ddfccb0306e93126ae26b71f91356d94c26a9714c2da50041d9cd687810e22b1d3f6525b0227d06ba31d1e
7
+ data.tar.gz: 91a656c17d523b5faf2f917dc8679af62e06206cee035000786999569f084e8855d087bb5f3a879e2263f85a01d6c0b5e2bedd7dec31cc54c3170fcff87f4100
@@ -0,0 +1,29 @@
1
+ version: 2
2
+
3
+ jobs:
4
+ build:
5
+ docker:
6
+ # specify the version you desire here
7
+ - image: cimg/ruby:2.6.5
8
+ working_directory: ~/repo
9
+ steps:
10
+ - checkout
11
+ - restore_cache:
12
+ keys:
13
+ - v1-dependencies-{{ checksum "Gemfile.lock" }}
14
+ # fallback to using the latest cache if no exact match is found
15
+ - v1-dependencies-
16
+ - run:
17
+ name: install dependencies
18
+ command: |
19
+ gem install bundler -v '1.17'
20
+ bundle install --jobs=4 --retry=3 --path vendor/bundle
21
+ - save_cache:
22
+ paths:
23
+ - ./vendor/bundle
24
+ key: v1-dependencies-{{ checksum "Gemfile.lock" }}
25
+ # run tests!
26
+ - run:
27
+ name: Run tests
28
+ command: |
29
+ bundle exec rake spec
@@ -0,0 +1,18 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /_yardoc/
4
+ /coverage/
5
+ /doc/
6
+ /docs/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
10
+
11
+ # rspec failure tracking
12
+ .rspec_status
13
+ *.pid
14
+ *.log
15
+ .envrc
16
+ .ruby-version
17
+ vendor/
18
+ bk/
data/.rspec ADDED
@@ -0,0 +1,3 @@
1
+ --format documentation
2
+ --color
3
+ --require spec_helper
data/Gemfile ADDED
@@ -0,0 +1,6 @@
1
+ source "https://rubygems.org"
2
+
3
+ git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
4
+
5
+ # Specify your gem's dependencies in kobanzame.gemspec
6
+ gemspec
@@ -0,0 +1,57 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ kobanzame (0.0.8)
5
+ aws-sdk-cloudwatch
6
+ aws-sdk-cloudwatchlogs
7
+ serverengine
8
+
9
+ GEM
10
+ remote: https://rubygems.org/
11
+ specs:
12
+ aws-eventstream (1.1.0)
13
+ aws-partitions (1.329.0)
14
+ aws-sdk-cloudwatch (1.39.1)
15
+ aws-sdk-core (~> 3, >= 3.99.0)
16
+ aws-sigv4 (~> 1.1)
17
+ aws-sdk-cloudwatchlogs (1.32.1)
18
+ aws-sdk-core (~> 3, >= 3.99.0)
19
+ aws-sigv4 (~> 1.1)
20
+ aws-sdk-core (3.99.2)
21
+ aws-eventstream (~> 1, >= 1.0.2)
22
+ aws-partitions (~> 1, >= 1.239.0)
23
+ aws-sigv4 (~> 1.1)
24
+ jmespath (~> 1.0)
25
+ aws-sigv4 (1.1.4)
26
+ aws-eventstream (~> 1.0, >= 1.0.2)
27
+ diff-lcs (1.3)
28
+ jmespath (1.4.0)
29
+ rake (13.0.1)
30
+ rspec (3.9.0)
31
+ rspec-core (~> 3.9.0)
32
+ rspec-expectations (~> 3.9.0)
33
+ rspec-mocks (~> 3.9.0)
34
+ rspec-core (3.9.2)
35
+ rspec-support (~> 3.9.3)
36
+ rspec-expectations (3.9.2)
37
+ diff-lcs (>= 1.2.0, < 2.0)
38
+ rspec-support (~> 3.9.0)
39
+ rspec-mocks (3.9.1)
40
+ diff-lcs (>= 1.2.0, < 2.0)
41
+ rspec-support (~> 3.9.0)
42
+ rspec-support (3.9.3)
43
+ serverengine (2.2.1)
44
+ sigdump (~> 0.2.2)
45
+ sigdump (0.2.4)
46
+
47
+ PLATFORMS
48
+ ruby
49
+
50
+ DEPENDENCIES
51
+ bundler (>= 1.9, < 3.0)
52
+ kobanzame!
53
+ rake (>= 12.3.3)
54
+ rspec (~> 3.0)
55
+
56
+ BUNDLED WITH
57
+ 1.17.2
@@ -0,0 +1,64 @@
1
+ # Kobanzame [![CircleCI](https://circleci.com/gh/inokappa/kobanzame.svg?style=svg)](https://circleci.com/gh/inokappa/kobanzame)
2
+
3
+ ![](https://raw.githubusercontent.com/inokappa/kobanzame/master/docs/images/fish_kobanzame.png)
4
+
5
+ ## About
6
+
7
+ Resource monitoring tool for ECS Task. It sticks to a Docker container like a `Kobanzame (black shark)` and gets information about the container.
8
+
9
+ ## Installation
10
+
11
+ Add this line to your application's Gemfile:
12
+
13
+ ```ruby
14
+ gem 'kobanzame'
15
+ ```
16
+
17
+ And then execute:
18
+
19
+ $ bundle
20
+
21
+ Or install it yourself as:
22
+
23
+ $ gem install kobanzame
24
+
25
+ ## Usage
26
+
27
+ Write kobanzame.json.
28
+
29
+ ```json
30
+ {
31
+ "container": {
32
+ "name": "batch-worker",
33
+ "check_interval": 1,
34
+ "report_format": "text"
35
+ },
36
+ "metrics": {
37
+ "name": "cloudwatch",
38
+ "namespace": "Custom/Kobanzame"
39
+ },
40
+ "outputs": [
41
+ {
42
+ "name": "cloudwatch_logs",
43
+ "log_group_name": "kobanzame-sample",
44
+ "log_stream_prefix": "kobanzame"
45
+ }
46
+ ]
47
+ }
48
+ ```
49
+
50
+ Starting kobanzame.
51
+
52
+ ```sh
53
+ $ kobanzame --config=kobanzame.json
54
+ ```
55
+
56
+ ## Development
57
+
58
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
59
+
60
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
61
+
62
+ ## Contributing
63
+
64
+ Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/kobanzame.
@@ -0,0 +1,6 @@
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task :default => :spec
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "kobanzame"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start(__FILE__)
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,4 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'kobanzame'
4
+ require_relative '../lib/kobanzame/cli'
@@ -0,0 +1,29 @@
1
+
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'kobanzame/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = 'kobanzame'
8
+ spec.version = Kobanzame::VERSION
9
+ spec.authors = ['inokappa']
10
+ spec.email = ['inokara@gmail.com']
11
+ spec.summary = %q{kobanzame collects resources for ecs task.}
12
+ spec.description = %q{kobanzame collects resources for ecs task.}
13
+ spec.homepage = 'https://github.com/inokappa/kobanzame'
14
+
15
+ spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
16
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|example|docs)/}) }
17
+ end
18
+ spec.bindir = 'exe'
19
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
20
+ spec.require_paths = ['lib']
21
+
22
+ spec.add_development_dependency 'bundler', '>= 1.9', '< 3.0'
23
+ spec.add_development_dependency 'rake', '>= 12.3.3'
24
+ spec.add_development_dependency 'rspec', '~> 3.0'
25
+
26
+ spec.add_runtime_dependency 'aws-sdk-cloudwatchlogs'
27
+ spec.add_runtime_dependency 'aws-sdk-cloudwatch'
28
+ spec.add_runtime_dependency 'serverengine'
29
+ end
@@ -0,0 +1,26 @@
1
+ {
2
+ "container": {
3
+ "name": "batch-worker",
4
+ "check_interval": 1,
5
+ "report_format": "json"
6
+ },
7
+ "metrics": {
8
+ "name": "cloudwatch",
9
+ "namespace": "Custom/Kobanzame"
10
+ },
11
+ "outputs": [
12
+ {
13
+ "name": "cloudwatch_logs",
14
+ "log_group_name": "kobanzame-sample",
15
+ "log_stream_prefix": "kobanzame",
16
+ "log_stream_name": "batch-worker-resource-statistics"
17
+ },
18
+ {
19
+ "name": "slack",
20
+ "webhook_url": "",
21
+ "title": "Kobanzame Sample Output",
22
+ "user_name": "kobanzame-sample",
23
+ "icon_emoji": ":shark:"
24
+ }
25
+ ]
26
+ }
@@ -0,0 +1,16 @@
1
+ require 'net/http'
2
+ require 'serverengine'
3
+ require 'kobanzame/config'
4
+ require 'kobanzame/ext'
5
+ require 'kobanzame/metrics'
6
+ require 'kobanzame/outputs'
7
+ require 'kobanzame/report'
8
+ require 'kobanzame/supervisor'
9
+ require 'kobanzame/utilities'
10
+ require 'kobanzame/version'
11
+ require 'kobanzame/worker'
12
+
13
+ module Kobanzame
14
+ class Error < StandardError; end
15
+ # Your code goes here...
16
+ end
@@ -0,0 +1,44 @@
1
+ require 'optparse'
2
+ require 'kobanzame'
3
+
4
+ op = OptionParser.new
5
+ op.version = Kobanzame::VERSION
6
+
7
+ opts = {
8
+ config: 'kobanzame.json',
9
+ pid_file: 'kobanzame.pid',
10
+ log_file: STDOUT,
11
+ daemonize: false,
12
+ debug: false
13
+ }
14
+
15
+ op.on('-c', '--config PATH', "config file path (default: #{opts[:config]})") {|v|
16
+ opts[:config] = v
17
+ }
18
+
19
+ op.on('-d', '--daemonize', "enable daemonize (default: #{opts[:daemonize]})") {|v|
20
+ opts[:daemonize] = v
21
+ }
22
+
23
+ op.on('-p', '--pid-file PATH', "pid file path (default: #{opts[:pid_file]})") {|v|
24
+ opts[:pid_file] = v
25
+ }
26
+
27
+ op.on('-l', '--log-file PATH', "log file path (default: STDOUT)") {|v|
28
+ opts[:log_file] = v
29
+ }
30
+
31
+ op.on('-D', '--debug', "start with debug mode (default: #{opts[:debug]})") {|v|
32
+ opts[:debug] = v
33
+ }
34
+
35
+ begin
36
+ op.parse(ARGV)
37
+ rescue OptionParser::InvalidOption => ex
38
+ puts op.to_s
39
+ puts "error: #{ex.message}"
40
+ exit 1
41
+ end
42
+
43
+ k = Kobanzame::Supervisor.new(opts)
44
+ k.start
@@ -0,0 +1,40 @@
1
+ module Kobanzame
2
+ class Config
3
+ def self.load_config(opts)
4
+ cfg = read_json(opts[:config])
5
+ se_config = {
6
+ debug: opts[:debug],
7
+ daemonize: opts[:daemonize],
8
+ pid_path: opts[:pid_file],
9
+ log: opts[:log_file],
10
+ log_level: 'debug',
11
+ kobanzame_config: cfg
12
+ }
13
+ se_config
14
+ end
15
+
16
+ def self.read_json(path)
17
+ config_hash = ''
18
+ File.open(path) do |file|
19
+ config_hash = JSON.load(file)
20
+ end
21
+ config_hash
22
+ end
23
+
24
+ def self.select_conf(config, key)
25
+ konf = config[:kobanzame_config]
26
+ konf[key]
27
+ end
28
+
29
+ def self.generate_task_params(container_info)
30
+ params = {}
31
+ params['task_id'] = (0...32).map{ (65 + rand(26)).chr }.join
32
+ if container_info.dig('Labels', 'com.amazonaws.ecs.task-arn')
33
+ params['task_id'] = container_info.dig('Labels', 'com.amazonaws.ecs.task-arn').split('/')[-1]
34
+ end
35
+ params['docker_id'] = container_info.dig('DockerId')
36
+ params['docker_name'] = container_info.dig('DockerName')
37
+ params
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,11 @@
1
+ class Array
2
+ def average
3
+ (self.inject(:+) / self.length).round(3)
4
+ end
5
+ end
6
+
7
+ class String
8
+ def camelize
9
+ self.split('_').map(&:capitalize).join
10
+ end
11
+ end
@@ -0,0 +1,38 @@
1
+ module Kobanzame
2
+ class Metrics
3
+ def initialize(conf, params)
4
+ @metrics_conf = Kobanzame::Config.select_conf(conf, 'metrics')
5
+ @params = params
6
+ @conf = conf
7
+ end
8
+
9
+ def logger
10
+ @logger ||= ServerEngine::DaemonLogger.new(@conf[:log] || STDOUT, @conf)
11
+ end
12
+
13
+ def publish(result)
14
+ load_output_modules.each do |mod|
15
+ begin
16
+ require "kobanzame/metrics/#{mod}"
17
+ class_name = "Kobanzame::Metric::#{mod.camelize}"
18
+ Object.const_get(class_name).new(@metrics_conf, @params).publish(result)
19
+ logger.info("Sent information to #{mod}")
20
+ rescue LoadError => ex
21
+ logger.warn("Could not load #{mod} module.")
22
+ rescue => ex
23
+ logger.warn("Could not send information to #{mod}. Reason: #{ex.message}")
24
+ end
25
+ end
26
+ end
27
+
28
+ private
29
+
30
+ def load_output_modules
31
+ modules = []
32
+ Dir.glob(File.dirname(__FILE__) + '/metrics/*').each do |r|
33
+ modules << File.basename(r, '.rb') unless r.include?('stub')
34
+ end
35
+ modules.sort
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,70 @@
1
+ require 'time'
2
+ require 'aws-sdk-cloudwatch'
3
+
4
+ module Kobanzame
5
+ module Metric
6
+ class Cloudwatch
7
+ def initialize(metric_conf, params)
8
+ @namespace = metric_conf['namespace']
9
+ @task_id = params['task_id']
10
+ end
11
+
12
+ def cw
13
+ @cw ||= Aws::CloudWatch::Client.new
14
+ end
15
+
16
+ def publish(result)
17
+ data = write_metric_data(result)
18
+ begin
19
+ cw.put_metric_data(data)
20
+ rescue Aws::CloudWatch::Errors::ServiceError
21
+ raise $!.message
22
+ end
23
+ end
24
+
25
+ private
26
+
27
+ # Input Data: [[timestamp, CPU Usage, Memory Used], [1591710550364, 0.0, 0.621], [1591710553452, 0.0, 0.621]]
28
+ def write_metric_data(datas)
29
+ metric = {}
30
+ metric_datas = []
31
+ metric[:namespace] = @namespace
32
+ datas.each do |data|
33
+ metric_data = [
34
+ {
35
+ metric_name: 'CPU Usage',
36
+ dimensions: dimentions,
37
+ timestamp: timestamp(data[0]),
38
+ value: data[1],
39
+ unit: 'Percent'
40
+ },
41
+ {
42
+ metric_name: 'Memory Used',
43
+ dimensions: dimentions,
44
+ timestamp: timestamp(data[0]),
45
+ value: data[2],
46
+ unit: 'Megabits'
47
+ }
48
+ ]
49
+ metric_datas << metric_data
50
+ end
51
+ metric[:metric_data] = metric_datas.flatten
52
+ metric
53
+ end
54
+
55
+ def dimentions
56
+ dims = [
57
+ {
58
+ name: 'TaskId',
59
+ value: @task_id,
60
+ }
61
+ ]
62
+ dims
63
+ end
64
+
65
+ def timestamp(t)
66
+ DateTime.strptime((t.to_i/1000).to_s, '%s').iso8601
67
+ end
68
+ end
69
+ end
70
+ end
@@ -0,0 +1,44 @@
1
+ module Kobanzame
2
+ class Outputs
3
+ def initialize(conf, params)
4
+ @outputs_conf = Kobanzame::Config.select_conf(conf, 'outputs')
5
+ @container_conf = Kobanzame::Config.select_conf(conf, 'container')
6
+ @params = params
7
+ @conf = conf
8
+ end
9
+
10
+ def logger
11
+ @logger ||= ServerEngine::DaemonLogger.new(@conf[:log] || STDOUT, @conf)
12
+ end
13
+
14
+ def publish(result)
15
+ load_output_modules.each do |mod|
16
+ output_conf = @outputs_conf.select { |c| c['name'] == mod }.last
17
+ if output_conf.nil?
18
+ logger.warn("Skip output to #{mod}")
19
+ next
20
+ end
21
+ begin
22
+ require "kobanzame/outputs/#{mod}"
23
+ class_name = "Kobanzame::Output::#{mod.camelize}"
24
+ Object.const_get(class_name).new(output_conf, @container_conf, @params).publish(result)
25
+ logger.info("Sent information to #{mod}")
26
+ rescue LoadError
27
+ logger.warn("Could not load #{mod} module.")
28
+ rescue => ex
29
+ logger.warn("Could not send information to #{mod}. Reason: #{ex.message}")
30
+ end
31
+ end
32
+ end
33
+
34
+ private
35
+
36
+ def load_output_modules
37
+ modules = []
38
+ Dir.glob(File.dirname(__FILE__) + '/outputs/*').each do |r|
39
+ modules << File.basename(r, '.rb') unless r.include?('stub')
40
+ end
41
+ modules.sort
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,72 @@
1
+ require 'aws-sdk-cloudwatchlogs'
2
+
3
+ module Kobanzame
4
+ module Output
5
+ class CloudwatchLogs
6
+ def initialize(output_conf, container_conf, params)
7
+ @log_group_name = output_conf['log_group_name']
8
+ @log_stream_prefix = output_conf['log_stream_prefix']
9
+ @log_stream_name = output_conf['log_stream_name']
10
+ @container_name = container_conf['name']
11
+ @format = container_conf['report_format']
12
+ @task_id = params['task_id']
13
+ end
14
+
15
+ def cwlog
16
+ @cwlog ||= Aws::CloudWatchLogs::Client.new
17
+ end
18
+
19
+ def publish(result)
20
+ event = {
21
+ log_group_name: @log_group_name,
22
+ log_stream_name: log_stream_name,
23
+ log_events: [{
24
+ timestamp: (Time.now.utc.to_f.round(3) * 1000).to_i,
25
+ message: result.send(@format)
26
+ }]
27
+ }
28
+ event['sequence_token'] = upload_sequence_token if upload_sequence_token != ''
29
+ begin
30
+ cwlog.put_log_events(event)
31
+ rescue Aws::CloudWatchLogs::Errors::ServiceError
32
+ raise $!.message
33
+ end
34
+ end
35
+
36
+ private
37
+
38
+ def log_stream_name
39
+ return @task_id if @log_stream_prefix.nil? && @log_stream_name.nil?
40
+ return @log_stream_name if @log_stream_prefix.nil?
41
+ @log_stream_prefix + '/' + @container_name + '/' + @task_id
42
+ end
43
+
44
+ def upload_sequence_token
45
+ return '' if log_streams.empty?
46
+ log_streams[0]['upload_sequence_token']
47
+ end
48
+
49
+ def log_streams
50
+ begin
51
+ res = cwlog.describe_log_streams({
52
+ log_group_name: @log_group_name,
53
+ log_stream_name_prefix: log_stream_name
54
+ })
55
+ rescue Aws::CloudWatchLogs::Errors::ServiceError
56
+ raise $!.message
57
+ end
58
+
59
+ begin
60
+ cwlog.create_log_stream({
61
+ log_group_name: @log_group_name,
62
+ log_stream_name: log_stream_name
63
+ })
64
+ return []
65
+ rescue Aws::CloudWatchLogs::Errors::ServiceError
66
+ raise $!.message
67
+ end if res['log_streams'].empty?
68
+ res['log_streams']
69
+ end
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,74 @@
1
+ require 'json'
2
+ require 'net/http'
3
+
4
+ module Kobanzame
5
+ module Output
6
+ class Slack
7
+ def initialize(output_conf, container_conf, params)
8
+ @format = container_conf['report_format']
9
+ @output_conf = output_conf
10
+ end
11
+
12
+ def publish(result)
13
+ attachements = build_attachements(result.send(@format))
14
+ params = build_params(attachements)
15
+ send(params)
16
+ end
17
+
18
+ private
19
+
20
+ def build_attachements(result)
21
+ result = build_pretty_json(result) if result.start_with?('{')
22
+ value = <<-"VALUE"
23
+ ```
24
+ #{result}
25
+ ```
26
+ VALUE
27
+ attachments = [
28
+ {
29
+ 'fallback': '',
30
+ 'title': @output_conf['title'],
31
+ 'fields': [
32
+ {
33
+ 'title': 'Output',
34
+ 'value': value,
35
+ 'short': false
36
+ },
37
+ ]
38
+ }
39
+ ]
40
+ attachments
41
+ end
42
+
43
+ def build_params(attachments)
44
+ params = {
45
+ username: @output_conf['user_name'],
46
+ icon_emoji: @output_conf['icon_emoji'] ,
47
+ text: '',
48
+ attachments: attachments
49
+ }
50
+ params
51
+ end
52
+
53
+ def build_pretty_json(json_string)
54
+ hash = JSON.parse(json_string)
55
+ JSON.pretty_generate(hash)
56
+ end
57
+
58
+ def send(build_params)
59
+ uri = URI.parse(ENV['SLACK_WEBHOOK_URL'])
60
+ http = Net::HTTP.new(uri.host, uri.port)
61
+ http.use_ssl = true
62
+ begin
63
+ http.start do
64
+ request = Net::HTTP::Post.new(uri.path)
65
+ request.set_form_data(payload: build_params.to_json)
66
+ http.request(request)
67
+ end
68
+ rescue => e
69
+ raise $!.message
70
+ end
71
+ end
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,13 @@
1
+ module Kobanzame
2
+ module Output
3
+ class Stdout < Outputs
4
+ def initialize(output_conf, container_conf, params)
5
+ @format = container_conf['report_format']
6
+ end
7
+
8
+ def publish(result)
9
+ $stdout.puts result.send(@format)
10
+ end
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,67 @@
1
+ module Kobanzame
2
+ class Report
3
+ def initialize(result, params)
4
+ @result = result
5
+ @params = params
6
+ end
7
+
8
+ def duration
9
+ times = @result.map { |a| a[0] }
10
+ return '' if times.nil?
11
+ duration = {}
12
+ duration['duration'] = (times[-1] - times[0])
13
+ duration['unit'] = 'ms'
14
+ duration
15
+ end
16
+
17
+ def cpu_usage
18
+ cpus = @result.map { |a| a[1] }
19
+ return '' if cpus.nil?
20
+ usage = {}
21
+ usage['average'] = cpus.average
22
+ usage['max'] = cpus.max
23
+ usage['unit'] = '%'
24
+ usage
25
+ end
26
+
27
+ def memory_used
28
+ memories = @result.map { |a| a[2] }
29
+ return '' if memories.nil?
30
+ usage = {}
31
+ usage['average'] = memories.average
32
+ usage['max'] = memories.max
33
+ usage['unit'] = 'MiB'
34
+ usage
35
+ end
36
+
37
+ def text
38
+ m = "Memory Used: #{memory_used['average']}MiB(ave)/#{memory_used['max']}MiB(max)"
39
+ c = "CPU Usage: #{cpu_usage['average']}%(ave)/#{cpu_usage['max']}%(max)"
40
+ task_id = @params['task_id']
41
+
42
+ "REPORT TaskID: #{task_id} Duration: #{duration['duration']}ms, #{m}, #{c}"
43
+ end
44
+
45
+ def json
46
+ report = {}
47
+
48
+ report['duration'] = duration
49
+ report['memory_used'] = memory_used
50
+ report['cpu_usage'] = cpu_usage
51
+
52
+ res = {}
53
+ res['report'] = report
54
+ res['task_id'] = @params['task_id']
55
+ res['docker_name'] = @params['docker_name']
56
+ res.to_json
57
+ end
58
+
59
+ def csv
60
+ m = "#{memory_used['average']},#{memory_used['max']}"
61
+ c = "#{cpu_usage['average']},#{cpu_usage['max']}"
62
+ task_id = @params['task_id']
63
+
64
+ "#{task_id},#{duration['duration']},#{m},#{c}"
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,14 @@
1
+ module Kobanzame
2
+ class Supervisor
3
+ def initialize(opts)
4
+ @opts = opts
5
+ end
6
+
7
+ def start
8
+ se = ServerEngine.create(nil,
9
+ Kobanzame::Worker,
10
+ Kobanzame::Config.load_config(@opts))
11
+ se.run
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,13 @@
1
+ module Kobanzame
2
+ class Utilities
3
+ def self.request(path)
4
+ url = URI.parse(ENV['ECS_CONTAINER_METADATA_URI'] + '/' + path)
5
+ req = Net::HTTP::Get.new(url.path)
6
+ res = Net::HTTP.start(url.host, url.port) {|http|
7
+ http.request(req)
8
+ }
9
+ return nil unless res.code == '200'
10
+ JSON.parse(res.body)
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,3 @@
1
+ module Kobanzame
2
+ VERSION = '0.0.8'
3
+ end
@@ -0,0 +1,113 @@
1
+ module Kobanzame
2
+ module Worker
3
+ def calc_cpu_percent(current, previous, interval = 1000000000)
4
+ # to Percent (x.x %)
5
+ # refer to: https://github.com/mackerelio/mackerel-container-agent/blob/ce7cb283a3cdbc690127c6f67c44bdce7b140619/platform/ecs/metric.go#L112-L115
6
+ ((current['cpu_usage']['total_usage'].to_f - previous['cpu_usage']['total_usage'].to_f) / interval.to_f * 100).round(3)
7
+ end
8
+
9
+ def calc_memory_size(memory_stat)
10
+ # to Megabyte (x.xx MiB)
11
+ # refer to: https://github.com/mackerelio/mackerel-container-agent/blob/ce7cb283a3cdbc690127c6f67c44bdce7b140619/platform/ecs/metric.go#L117-L119
12
+ ((memory_stat['usage'] - memory_stat['stats']['cache']).to_f / (1024.0 * 1024.0)).round(3)
13
+ end
14
+
15
+ def describe_target_container
16
+ # return Container Info
17
+ res = Kobanzame::Utilities.request('task')
18
+ return nil if res.nil?
19
+ res
20
+ end
21
+
22
+ def describe_target_container_metrics(container_id)
23
+ stat = nil
24
+ res = Kobanzame::Utilities.request('task/stats')
25
+ stat = res.dig(container_id) unless res.nil?
26
+
27
+ return nil if stat.nil? || stat.dig('memory_stats').empty?
28
+ memory_usage = calc_memory_size(stat['memory_stats'])
29
+ cpu_usage = calc_cpu_percent(stat['cpu_stats'], stat['precpu_stats'])
30
+
31
+ # Return format: ['Datetime', 'Used CPU unit', 'Used Memory size']
32
+ [Time.now.strftime('%s%L').to_i, cpu_usage, memory_usage]
33
+ end
34
+
35
+ def load_container_config
36
+ Kobanzame::Config.select_conf(config, 'container')
37
+ end
38
+
39
+ def load_metrics_config
40
+ Kobanzame::Config.select_conf(config, 'metrics')
41
+ end
42
+
43
+ def describe_container_info
44
+ cf = load_container_config
45
+ logger.info 'Waiting for starting target container.'
46
+ i = 1
47
+ container_info = ''
48
+ loop do
49
+ containers = describe_target_container
50
+ logger.warn 'Could not get all container information.' if containers.nil?
51
+ container_info = containers['Containers'].find { |c| c['Name'] == cf['name'] }
52
+ break if !container_info.nil? && container_info['DockerId'] != ''
53
+ # waiting for 120s
54
+ i += 1
55
+ if i > 5 then
56
+ logger.fatal 'Could not get the information of the batch container.'
57
+ exit 1
58
+ end
59
+ sleep cf['check_interval'].to_i
60
+ end
61
+ container_info['_check_interval'] = cf['check_interval'].to_i
62
+ container_info
63
+ end
64
+
65
+ def run
66
+ container_info = describe_container_info
67
+ mt = load_metrics_config
68
+ params = Kobanzame::Config.generate_task_params(container_info)
69
+ logger.info 'Collecting target container metrics.'
70
+
71
+ i = 1
72
+ result = []
73
+ nilbox = []
74
+ metbox = []
75
+ until @stop
76
+ metrics = describe_target_container_metrics(container_info.dig('DockerId'))
77
+ # metrics が 3 回取れなかったら, ループを抜ける
78
+ nilbox << nil if metrics.nil?
79
+ if nilbox.size > 3
80
+ if !mt.nil? || !mt == {}
81
+ Kobanzame::Metrics.new(config, params).publish(metbox) unless metbox.empty?
82
+ end
83
+ break
84
+ end
85
+ #
86
+ if !metrics.nil?
87
+ result << metrics
88
+ metbox << metrics
89
+ end
90
+ logger.debug metrics if config[:debug]
91
+ #
92
+ if i % 10 == 0
93
+ logger.info "Collected #{result.length} metrics..."
94
+ if !mt.nil? || !mt == {}
95
+ Kobanzame::Metrics.new(config, params).publish(metbox)
96
+ metbox = []
97
+ end
98
+ end
99
+ i += 1
100
+ #
101
+ sleep container_info['_check_interval']
102
+ end
103
+
104
+ repo = Kobanzame::Report.new(result, params)
105
+ Kobanzame::Outputs.new(config, params).publish(repo)
106
+ exit 0
107
+ end
108
+
109
+ def stop
110
+ @stop = true
111
+ end
112
+ end
113
+ end
metadata ADDED
@@ -0,0 +1,160 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: kobanzame
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.8
5
+ platform: ruby
6
+ authors:
7
+ - inokappa
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2020-06-14 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '1.9'
20
+ - - "<"
21
+ - !ruby/object:Gem::Version
22
+ version: '3.0'
23
+ type: :development
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ version: '1.9'
30
+ - - "<"
31
+ - !ruby/object:Gem::Version
32
+ version: '3.0'
33
+ - !ruby/object:Gem::Dependency
34
+ name: rake
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - ">="
38
+ - !ruby/object:Gem::Version
39
+ version: 12.3.3
40
+ type: :development
41
+ prerelease: false
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - ">="
45
+ - !ruby/object:Gem::Version
46
+ version: 12.3.3
47
+ - !ruby/object:Gem::Dependency
48
+ name: rspec
49
+ requirement: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - "~>"
52
+ - !ruby/object:Gem::Version
53
+ version: '3.0'
54
+ type: :development
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - "~>"
59
+ - !ruby/object:Gem::Version
60
+ version: '3.0'
61
+ - !ruby/object:Gem::Dependency
62
+ name: aws-sdk-cloudwatchlogs
63
+ requirement: !ruby/object:Gem::Requirement
64
+ requirements:
65
+ - - ">="
66
+ - !ruby/object:Gem::Version
67
+ version: '0'
68
+ type: :runtime
69
+ prerelease: false
70
+ version_requirements: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - ">="
73
+ - !ruby/object:Gem::Version
74
+ version: '0'
75
+ - !ruby/object:Gem::Dependency
76
+ name: aws-sdk-cloudwatch
77
+ requirement: !ruby/object:Gem::Requirement
78
+ requirements:
79
+ - - ">="
80
+ - !ruby/object:Gem::Version
81
+ version: '0'
82
+ type: :runtime
83
+ prerelease: false
84
+ version_requirements: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - ">="
87
+ - !ruby/object:Gem::Version
88
+ version: '0'
89
+ - !ruby/object:Gem::Dependency
90
+ name: serverengine
91
+ requirement: !ruby/object:Gem::Requirement
92
+ requirements:
93
+ - - ">="
94
+ - !ruby/object:Gem::Version
95
+ version: '0'
96
+ type: :runtime
97
+ prerelease: false
98
+ version_requirements: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - ">="
101
+ - !ruby/object:Gem::Version
102
+ version: '0'
103
+ description: kobanzame collects resources for ecs task.
104
+ email:
105
+ - inokara@gmail.com
106
+ executables:
107
+ - kobanzame
108
+ extensions: []
109
+ extra_rdoc_files: []
110
+ files:
111
+ - ".circleci/config.yml"
112
+ - ".gitignore"
113
+ - ".rspec"
114
+ - Gemfile
115
+ - Gemfile.lock
116
+ - README.md
117
+ - Rakefile
118
+ - bin/console
119
+ - bin/setup
120
+ - exe/kobanzame
121
+ - kobanzame.gemspec
122
+ - kobanzame.sample.json
123
+ - lib/kobanzame.rb
124
+ - lib/kobanzame/cli.rb
125
+ - lib/kobanzame/config.rb
126
+ - lib/kobanzame/ext.rb
127
+ - lib/kobanzame/metrics.rb
128
+ - lib/kobanzame/metrics/cloudwatch.rb
129
+ - lib/kobanzame/outputs.rb
130
+ - lib/kobanzame/outputs/cloudwatch_logs.rb
131
+ - lib/kobanzame/outputs/slack.rb
132
+ - lib/kobanzame/outputs/stdout.rb
133
+ - lib/kobanzame/report.rb
134
+ - lib/kobanzame/supervisor.rb
135
+ - lib/kobanzame/utilities.rb
136
+ - lib/kobanzame/version.rb
137
+ - lib/kobanzame/worker.rb
138
+ homepage: https://github.com/inokappa/kobanzame
139
+ licenses: []
140
+ metadata: {}
141
+ post_install_message:
142
+ rdoc_options: []
143
+ require_paths:
144
+ - lib
145
+ required_ruby_version: !ruby/object:Gem::Requirement
146
+ requirements:
147
+ - - ">="
148
+ - !ruby/object:Gem::Version
149
+ version: '0'
150
+ required_rubygems_version: !ruby/object:Gem::Requirement
151
+ requirements:
152
+ - - ">="
153
+ - !ruby/object:Gem::Version
154
+ version: '0'
155
+ requirements: []
156
+ rubygems_version: 3.0.3
157
+ signing_key:
158
+ specification_version: 4
159
+ summary: kobanzame collects resources for ecs task.
160
+ test_files: []