kobanzame 0.0.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.circleci/config.yml +29 -0
- data/.gitignore +18 -0
- data/.rspec +3 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +57 -0
- data/README.md +64 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/exe/kobanzame +4 -0
- data/kobanzame.gemspec +29 -0
- data/kobanzame.sample.json +26 -0
- data/lib/kobanzame.rb +16 -0
- data/lib/kobanzame/cli.rb +44 -0
- data/lib/kobanzame/config.rb +40 -0
- data/lib/kobanzame/ext.rb +11 -0
- data/lib/kobanzame/metrics.rb +38 -0
- data/lib/kobanzame/metrics/cloudwatch.rb +70 -0
- data/lib/kobanzame/outputs.rb +44 -0
- data/lib/kobanzame/outputs/cloudwatch_logs.rb +72 -0
- data/lib/kobanzame/outputs/slack.rb +74 -0
- data/lib/kobanzame/outputs/stdout.rb +13 -0
- data/lib/kobanzame/report.rb +67 -0
- data/lib/kobanzame/supervisor.rb +14 -0
- data/lib/kobanzame/utilities.rb +13 -0
- data/lib/kobanzame/version.rb +3 -0
- data/lib/kobanzame/worker.rb +113 -0
- metadata +160 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 159e7d623735c2eecf51458f18b1bbfe0c5b2f3588693ca23876710265691e85
|
4
|
+
data.tar.gz: 2546fea926c29d499b6ba7e22f2e3ee1a106c137059f307f67636129182a38d2
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: d01b930e5042038c7f2809bd584e704802bdf5feb1ddfccb0306e93126ae26b71f91356d94c26a9714c2da50041d9cd687810e22b1d3f6525b0227d06ba31d1e
|
7
|
+
data.tar.gz: 91a656c17d523b5faf2f917dc8679af62e06206cee035000786999569f084e8855d087bb5f3a879e2263f85a01d6c0b5e2bedd7dec31cc54c3170fcff87f4100
|
@@ -0,0 +1,29 @@
|
|
1
|
+
version: 2
|
2
|
+
|
3
|
+
jobs:
|
4
|
+
build:
|
5
|
+
docker:
|
6
|
+
# specify the version you desire here
|
7
|
+
- image: cimg/ruby:2.6.5
|
8
|
+
working_directory: ~/repo
|
9
|
+
steps:
|
10
|
+
- checkout
|
11
|
+
- restore_cache:
|
12
|
+
keys:
|
13
|
+
- v1-dependencies-{{ checksum "Gemfile.lock" }}
|
14
|
+
# fallback to using the latest cache if no exact match is found
|
15
|
+
- v1-dependencies-
|
16
|
+
- run:
|
17
|
+
name: install dependencies
|
18
|
+
command: |
|
19
|
+
gem install bundler -v '1.17'
|
20
|
+
bundle install --jobs=4 --retry=3 --path vendor/bundle
|
21
|
+
- save_cache:
|
22
|
+
paths:
|
23
|
+
- ./vendor/bundle
|
24
|
+
key: v1-dependencies-{{ checksum "Gemfile.lock" }}
|
25
|
+
# run tests!
|
26
|
+
- run:
|
27
|
+
name: Run tests
|
28
|
+
command: |
|
29
|
+
bundle exec rake spec
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,57 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
kobanzame (0.0.8)
|
5
|
+
aws-sdk-cloudwatch
|
6
|
+
aws-sdk-cloudwatchlogs
|
7
|
+
serverengine
|
8
|
+
|
9
|
+
GEM
|
10
|
+
remote: https://rubygems.org/
|
11
|
+
specs:
|
12
|
+
aws-eventstream (1.1.0)
|
13
|
+
aws-partitions (1.329.0)
|
14
|
+
aws-sdk-cloudwatch (1.39.1)
|
15
|
+
aws-sdk-core (~> 3, >= 3.99.0)
|
16
|
+
aws-sigv4 (~> 1.1)
|
17
|
+
aws-sdk-cloudwatchlogs (1.32.1)
|
18
|
+
aws-sdk-core (~> 3, >= 3.99.0)
|
19
|
+
aws-sigv4 (~> 1.1)
|
20
|
+
aws-sdk-core (3.99.2)
|
21
|
+
aws-eventstream (~> 1, >= 1.0.2)
|
22
|
+
aws-partitions (~> 1, >= 1.239.0)
|
23
|
+
aws-sigv4 (~> 1.1)
|
24
|
+
jmespath (~> 1.0)
|
25
|
+
aws-sigv4 (1.1.4)
|
26
|
+
aws-eventstream (~> 1.0, >= 1.0.2)
|
27
|
+
diff-lcs (1.3)
|
28
|
+
jmespath (1.4.0)
|
29
|
+
rake (13.0.1)
|
30
|
+
rspec (3.9.0)
|
31
|
+
rspec-core (~> 3.9.0)
|
32
|
+
rspec-expectations (~> 3.9.0)
|
33
|
+
rspec-mocks (~> 3.9.0)
|
34
|
+
rspec-core (3.9.2)
|
35
|
+
rspec-support (~> 3.9.3)
|
36
|
+
rspec-expectations (3.9.2)
|
37
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
38
|
+
rspec-support (~> 3.9.0)
|
39
|
+
rspec-mocks (3.9.1)
|
40
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
41
|
+
rspec-support (~> 3.9.0)
|
42
|
+
rspec-support (3.9.3)
|
43
|
+
serverengine (2.2.1)
|
44
|
+
sigdump (~> 0.2.2)
|
45
|
+
sigdump (0.2.4)
|
46
|
+
|
47
|
+
PLATFORMS
|
48
|
+
ruby
|
49
|
+
|
50
|
+
DEPENDENCIES
|
51
|
+
bundler (>= 1.9, < 3.0)
|
52
|
+
kobanzame!
|
53
|
+
rake (>= 12.3.3)
|
54
|
+
rspec (~> 3.0)
|
55
|
+
|
56
|
+
BUNDLED WITH
|
57
|
+
1.17.2
|
data/README.md
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
# Kobanzame [![CircleCI](https://circleci.com/gh/inokappa/kobanzame.svg?style=svg)](https://circleci.com/gh/inokappa/kobanzame)
|
2
|
+
|
3
|
+
![](https://raw.githubusercontent.com/inokappa/kobanzame/master/docs/images/fish_kobanzame.png)
|
4
|
+
|
5
|
+
## About
|
6
|
+
|
7
|
+
Resource monitoring tool for ECS Task. It sticks to a Docker container like a `Kobanzame (black shark)` and gets information about the container.
|
8
|
+
|
9
|
+
## Installation
|
10
|
+
|
11
|
+
Add this line to your application's Gemfile:
|
12
|
+
|
13
|
+
```ruby
|
14
|
+
gem 'kobanzame'
|
15
|
+
```
|
16
|
+
|
17
|
+
And then execute:
|
18
|
+
|
19
|
+
$ bundle
|
20
|
+
|
21
|
+
Or install it yourself as:
|
22
|
+
|
23
|
+
$ gem install kobanzame
|
24
|
+
|
25
|
+
## Usage
|
26
|
+
|
27
|
+
Write kobanzame.json.
|
28
|
+
|
29
|
+
```json
|
30
|
+
{
|
31
|
+
"container": {
|
32
|
+
"name": "batch-worker",
|
33
|
+
"check_interval": 1,
|
34
|
+
"report_format": "text"
|
35
|
+
},
|
36
|
+
"metrics": {
|
37
|
+
"name": "cloudwatch",
|
38
|
+
"namespace": "Custom/Kobanzame"
|
39
|
+
},
|
40
|
+
"outputs": [
|
41
|
+
{
|
42
|
+
"name": "cloudwatch_logs",
|
43
|
+
"log_group_name": "kobanzame-sample",
|
44
|
+
"log_stream_prefix": "kobanzame"
|
45
|
+
}
|
46
|
+
]
|
47
|
+
}
|
48
|
+
```
|
49
|
+
|
50
|
+
Starting kobanzame.
|
51
|
+
|
52
|
+
```sh
|
53
|
+
$ kobanzame --config=kobanzame.json
|
54
|
+
```
|
55
|
+
|
56
|
+
## Development
|
57
|
+
|
58
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
59
|
+
|
60
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
61
|
+
|
62
|
+
## Contributing
|
63
|
+
|
64
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/kobanzame.
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "kobanzame"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
data/exe/kobanzame
ADDED
data/kobanzame.gemspec
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'kobanzame/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = 'kobanzame'
|
8
|
+
spec.version = Kobanzame::VERSION
|
9
|
+
spec.authors = ['inokappa']
|
10
|
+
spec.email = ['inokara@gmail.com']
|
11
|
+
spec.summary = %q{kobanzame collects resources for ecs task.}
|
12
|
+
spec.description = %q{kobanzame collects resources for ecs task.}
|
13
|
+
spec.homepage = 'https://github.com/inokappa/kobanzame'
|
14
|
+
|
15
|
+
spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
|
16
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|example|docs)/}) }
|
17
|
+
end
|
18
|
+
spec.bindir = 'exe'
|
19
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
20
|
+
spec.require_paths = ['lib']
|
21
|
+
|
22
|
+
spec.add_development_dependency 'bundler', '>= 1.9', '< 3.0'
|
23
|
+
spec.add_development_dependency 'rake', '>= 12.3.3'
|
24
|
+
spec.add_development_dependency 'rspec', '~> 3.0'
|
25
|
+
|
26
|
+
spec.add_runtime_dependency 'aws-sdk-cloudwatchlogs'
|
27
|
+
spec.add_runtime_dependency 'aws-sdk-cloudwatch'
|
28
|
+
spec.add_runtime_dependency 'serverengine'
|
29
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
{
|
2
|
+
"container": {
|
3
|
+
"name": "batch-worker",
|
4
|
+
"check_interval": 1,
|
5
|
+
"report_format": "json"
|
6
|
+
},
|
7
|
+
"metrics": {
|
8
|
+
"name": "cloudwatch",
|
9
|
+
"namespace": "Custom/Kobanzame"
|
10
|
+
},
|
11
|
+
"outputs": [
|
12
|
+
{
|
13
|
+
"name": "cloudwatch_logs",
|
14
|
+
"log_group_name": "kobanzame-sample",
|
15
|
+
"log_stream_prefix": "kobanzame",
|
16
|
+
"log_stream_name": "batch-worker-resource-statistics"
|
17
|
+
},
|
18
|
+
{
|
19
|
+
"name": "slack",
|
20
|
+
"webhook_url": "",
|
21
|
+
"title": "Kobanzame Sample Output",
|
22
|
+
"user_name": "kobanzame-sample",
|
23
|
+
"icon_emoji": ":shark:"
|
24
|
+
}
|
25
|
+
]
|
26
|
+
}
|
data/lib/kobanzame.rb
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'serverengine'
|
3
|
+
require 'kobanzame/config'
|
4
|
+
require 'kobanzame/ext'
|
5
|
+
require 'kobanzame/metrics'
|
6
|
+
require 'kobanzame/outputs'
|
7
|
+
require 'kobanzame/report'
|
8
|
+
require 'kobanzame/supervisor'
|
9
|
+
require 'kobanzame/utilities'
|
10
|
+
require 'kobanzame/version'
|
11
|
+
require 'kobanzame/worker'
|
12
|
+
|
13
|
+
module Kobanzame
|
14
|
+
class Error < StandardError; end
|
15
|
+
# Your code goes here...
|
16
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
require 'kobanzame'
|
3
|
+
|
4
|
+
op = OptionParser.new
|
5
|
+
op.version = Kobanzame::VERSION
|
6
|
+
|
7
|
+
opts = {
|
8
|
+
config: 'kobanzame.json',
|
9
|
+
pid_file: 'kobanzame.pid',
|
10
|
+
log_file: STDOUT,
|
11
|
+
daemonize: false,
|
12
|
+
debug: false
|
13
|
+
}
|
14
|
+
|
15
|
+
op.on('-c', '--config PATH', "config file path (default: #{opts[:config]})") {|v|
|
16
|
+
opts[:config] = v
|
17
|
+
}
|
18
|
+
|
19
|
+
op.on('-d', '--daemonize', "enable daemonize (default: #{opts[:daemonize]})") {|v|
|
20
|
+
opts[:daemonize] = v
|
21
|
+
}
|
22
|
+
|
23
|
+
op.on('-p', '--pid-file PATH', "pid file path (default: #{opts[:pid_file]})") {|v|
|
24
|
+
opts[:pid_file] = v
|
25
|
+
}
|
26
|
+
|
27
|
+
op.on('-l', '--log-file PATH', "log file path (default: STDOUT)") {|v|
|
28
|
+
opts[:log_file] = v
|
29
|
+
}
|
30
|
+
|
31
|
+
op.on('-D', '--debug', "start with debug mode (default: #{opts[:debug]})") {|v|
|
32
|
+
opts[:debug] = v
|
33
|
+
}
|
34
|
+
|
35
|
+
begin
|
36
|
+
op.parse(ARGV)
|
37
|
+
rescue OptionParser::InvalidOption => ex
|
38
|
+
puts op.to_s
|
39
|
+
puts "error: #{ex.message}"
|
40
|
+
exit 1
|
41
|
+
end
|
42
|
+
|
43
|
+
k = Kobanzame::Supervisor.new(opts)
|
44
|
+
k.start
|
@@ -0,0 +1,40 @@
|
|
1
|
+
module Kobanzame
|
2
|
+
class Config
|
3
|
+
def self.load_config(opts)
|
4
|
+
cfg = read_json(opts[:config])
|
5
|
+
se_config = {
|
6
|
+
debug: opts[:debug],
|
7
|
+
daemonize: opts[:daemonize],
|
8
|
+
pid_path: opts[:pid_file],
|
9
|
+
log: opts[:log_file],
|
10
|
+
log_level: 'debug',
|
11
|
+
kobanzame_config: cfg
|
12
|
+
}
|
13
|
+
se_config
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.read_json(path)
|
17
|
+
config_hash = ''
|
18
|
+
File.open(path) do |file|
|
19
|
+
config_hash = JSON.load(file)
|
20
|
+
end
|
21
|
+
config_hash
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.select_conf(config, key)
|
25
|
+
konf = config[:kobanzame_config]
|
26
|
+
konf[key]
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.generate_task_params(container_info)
|
30
|
+
params = {}
|
31
|
+
params['task_id'] = (0...32).map{ (65 + rand(26)).chr }.join
|
32
|
+
if container_info.dig('Labels', 'com.amazonaws.ecs.task-arn')
|
33
|
+
params['task_id'] = container_info.dig('Labels', 'com.amazonaws.ecs.task-arn').split('/')[-1]
|
34
|
+
end
|
35
|
+
params['docker_id'] = container_info.dig('DockerId')
|
36
|
+
params['docker_name'] = container_info.dig('DockerName')
|
37
|
+
params
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
module Kobanzame
|
2
|
+
class Metrics
|
3
|
+
def initialize(conf, params)
|
4
|
+
@metrics_conf = Kobanzame::Config.select_conf(conf, 'metrics')
|
5
|
+
@params = params
|
6
|
+
@conf = conf
|
7
|
+
end
|
8
|
+
|
9
|
+
def logger
|
10
|
+
@logger ||= ServerEngine::DaemonLogger.new(@conf[:log] || STDOUT, @conf)
|
11
|
+
end
|
12
|
+
|
13
|
+
def publish(result)
|
14
|
+
load_output_modules.each do |mod|
|
15
|
+
begin
|
16
|
+
require "kobanzame/metrics/#{mod}"
|
17
|
+
class_name = "Kobanzame::Metric::#{mod.camelize}"
|
18
|
+
Object.const_get(class_name).new(@metrics_conf, @params).publish(result)
|
19
|
+
logger.info("Sent information to #{mod}")
|
20
|
+
rescue LoadError => ex
|
21
|
+
logger.warn("Could not load #{mod} module.")
|
22
|
+
rescue => ex
|
23
|
+
logger.warn("Could not send information to #{mod}. Reason: #{ex.message}")
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def load_output_modules
|
31
|
+
modules = []
|
32
|
+
Dir.glob(File.dirname(__FILE__) + '/metrics/*').each do |r|
|
33
|
+
modules << File.basename(r, '.rb') unless r.include?('stub')
|
34
|
+
end
|
35
|
+
modules.sort
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
require 'time'
|
2
|
+
require 'aws-sdk-cloudwatch'
|
3
|
+
|
4
|
+
module Kobanzame
|
5
|
+
module Metric
|
6
|
+
class Cloudwatch
|
7
|
+
def initialize(metric_conf, params)
|
8
|
+
@namespace = metric_conf['namespace']
|
9
|
+
@task_id = params['task_id']
|
10
|
+
end
|
11
|
+
|
12
|
+
def cw
|
13
|
+
@cw ||= Aws::CloudWatch::Client.new
|
14
|
+
end
|
15
|
+
|
16
|
+
def publish(result)
|
17
|
+
data = write_metric_data(result)
|
18
|
+
begin
|
19
|
+
cw.put_metric_data(data)
|
20
|
+
rescue Aws::CloudWatch::Errors::ServiceError
|
21
|
+
raise $!.message
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
# Input Data: [[timestamp, CPU Usage, Memory Used], [1591710550364, 0.0, 0.621], [1591710553452, 0.0, 0.621]]
|
28
|
+
def write_metric_data(datas)
|
29
|
+
metric = {}
|
30
|
+
metric_datas = []
|
31
|
+
metric[:namespace] = @namespace
|
32
|
+
datas.each do |data|
|
33
|
+
metric_data = [
|
34
|
+
{
|
35
|
+
metric_name: 'CPU Usage',
|
36
|
+
dimensions: dimentions,
|
37
|
+
timestamp: timestamp(data[0]),
|
38
|
+
value: data[1],
|
39
|
+
unit: 'Percent'
|
40
|
+
},
|
41
|
+
{
|
42
|
+
metric_name: 'Memory Used',
|
43
|
+
dimensions: dimentions,
|
44
|
+
timestamp: timestamp(data[0]),
|
45
|
+
value: data[2],
|
46
|
+
unit: 'Megabits'
|
47
|
+
}
|
48
|
+
]
|
49
|
+
metric_datas << metric_data
|
50
|
+
end
|
51
|
+
metric[:metric_data] = metric_datas.flatten
|
52
|
+
metric
|
53
|
+
end
|
54
|
+
|
55
|
+
def dimentions
|
56
|
+
dims = [
|
57
|
+
{
|
58
|
+
name: 'TaskId',
|
59
|
+
value: @task_id,
|
60
|
+
}
|
61
|
+
]
|
62
|
+
dims
|
63
|
+
end
|
64
|
+
|
65
|
+
def timestamp(t)
|
66
|
+
DateTime.strptime((t.to_i/1000).to_s, '%s').iso8601
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
module Kobanzame
|
2
|
+
class Outputs
|
3
|
+
def initialize(conf, params)
|
4
|
+
@outputs_conf = Kobanzame::Config.select_conf(conf, 'outputs')
|
5
|
+
@container_conf = Kobanzame::Config.select_conf(conf, 'container')
|
6
|
+
@params = params
|
7
|
+
@conf = conf
|
8
|
+
end
|
9
|
+
|
10
|
+
def logger
|
11
|
+
@logger ||= ServerEngine::DaemonLogger.new(@conf[:log] || STDOUT, @conf)
|
12
|
+
end
|
13
|
+
|
14
|
+
def publish(result)
|
15
|
+
load_output_modules.each do |mod|
|
16
|
+
output_conf = @outputs_conf.select { |c| c['name'] == mod }.last
|
17
|
+
if output_conf.nil?
|
18
|
+
logger.warn("Skip output to #{mod}")
|
19
|
+
next
|
20
|
+
end
|
21
|
+
begin
|
22
|
+
require "kobanzame/outputs/#{mod}"
|
23
|
+
class_name = "Kobanzame::Output::#{mod.camelize}"
|
24
|
+
Object.const_get(class_name).new(output_conf, @container_conf, @params).publish(result)
|
25
|
+
logger.info("Sent information to #{mod}")
|
26
|
+
rescue LoadError
|
27
|
+
logger.warn("Could not load #{mod} module.")
|
28
|
+
rescue => ex
|
29
|
+
logger.warn("Could not send information to #{mod}. Reason: #{ex.message}")
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
private
|
35
|
+
|
36
|
+
def load_output_modules
|
37
|
+
modules = []
|
38
|
+
Dir.glob(File.dirname(__FILE__) + '/outputs/*').each do |r|
|
39
|
+
modules << File.basename(r, '.rb') unless r.include?('stub')
|
40
|
+
end
|
41
|
+
modules.sort
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
require 'aws-sdk-cloudwatchlogs'
|
2
|
+
|
3
|
+
module Kobanzame
|
4
|
+
module Output
|
5
|
+
class CloudwatchLogs
|
6
|
+
def initialize(output_conf, container_conf, params)
|
7
|
+
@log_group_name = output_conf['log_group_name']
|
8
|
+
@log_stream_prefix = output_conf['log_stream_prefix']
|
9
|
+
@log_stream_name = output_conf['log_stream_name']
|
10
|
+
@container_name = container_conf['name']
|
11
|
+
@format = container_conf['report_format']
|
12
|
+
@task_id = params['task_id']
|
13
|
+
end
|
14
|
+
|
15
|
+
def cwlog
|
16
|
+
@cwlog ||= Aws::CloudWatchLogs::Client.new
|
17
|
+
end
|
18
|
+
|
19
|
+
def publish(result)
|
20
|
+
event = {
|
21
|
+
log_group_name: @log_group_name,
|
22
|
+
log_stream_name: log_stream_name,
|
23
|
+
log_events: [{
|
24
|
+
timestamp: (Time.now.utc.to_f.round(3) * 1000).to_i,
|
25
|
+
message: result.send(@format)
|
26
|
+
}]
|
27
|
+
}
|
28
|
+
event['sequence_token'] = upload_sequence_token if upload_sequence_token != ''
|
29
|
+
begin
|
30
|
+
cwlog.put_log_events(event)
|
31
|
+
rescue Aws::CloudWatchLogs::Errors::ServiceError
|
32
|
+
raise $!.message
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def log_stream_name
|
39
|
+
return @task_id if @log_stream_prefix.nil? && @log_stream_name.nil?
|
40
|
+
return @log_stream_name if @log_stream_prefix.nil?
|
41
|
+
@log_stream_prefix + '/' + @container_name + '/' + @task_id
|
42
|
+
end
|
43
|
+
|
44
|
+
def upload_sequence_token
|
45
|
+
return '' if log_streams.empty?
|
46
|
+
log_streams[0]['upload_sequence_token']
|
47
|
+
end
|
48
|
+
|
49
|
+
def log_streams
|
50
|
+
begin
|
51
|
+
res = cwlog.describe_log_streams({
|
52
|
+
log_group_name: @log_group_name,
|
53
|
+
log_stream_name_prefix: log_stream_name
|
54
|
+
})
|
55
|
+
rescue Aws::CloudWatchLogs::Errors::ServiceError
|
56
|
+
raise $!.message
|
57
|
+
end
|
58
|
+
|
59
|
+
begin
|
60
|
+
cwlog.create_log_stream({
|
61
|
+
log_group_name: @log_group_name,
|
62
|
+
log_stream_name: log_stream_name
|
63
|
+
})
|
64
|
+
return []
|
65
|
+
rescue Aws::CloudWatchLogs::Errors::ServiceError
|
66
|
+
raise $!.message
|
67
|
+
end if res['log_streams'].empty?
|
68
|
+
res['log_streams']
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'net/http'
|
3
|
+
|
4
|
+
module Kobanzame
|
5
|
+
module Output
|
6
|
+
class Slack
|
7
|
+
def initialize(output_conf, container_conf, params)
|
8
|
+
@format = container_conf['report_format']
|
9
|
+
@output_conf = output_conf
|
10
|
+
end
|
11
|
+
|
12
|
+
def publish(result)
|
13
|
+
attachements = build_attachements(result.send(@format))
|
14
|
+
params = build_params(attachements)
|
15
|
+
send(params)
|
16
|
+
end
|
17
|
+
|
18
|
+
private
|
19
|
+
|
20
|
+
def build_attachements(result)
|
21
|
+
result = build_pretty_json(result) if result.start_with?('{')
|
22
|
+
value = <<-"VALUE"
|
23
|
+
```
|
24
|
+
#{result}
|
25
|
+
```
|
26
|
+
VALUE
|
27
|
+
attachments = [
|
28
|
+
{
|
29
|
+
'fallback': '',
|
30
|
+
'title': @output_conf['title'],
|
31
|
+
'fields': [
|
32
|
+
{
|
33
|
+
'title': 'Output',
|
34
|
+
'value': value,
|
35
|
+
'short': false
|
36
|
+
},
|
37
|
+
]
|
38
|
+
}
|
39
|
+
]
|
40
|
+
attachments
|
41
|
+
end
|
42
|
+
|
43
|
+
def build_params(attachments)
|
44
|
+
params = {
|
45
|
+
username: @output_conf['user_name'],
|
46
|
+
icon_emoji: @output_conf['icon_emoji'] ,
|
47
|
+
text: '',
|
48
|
+
attachments: attachments
|
49
|
+
}
|
50
|
+
params
|
51
|
+
end
|
52
|
+
|
53
|
+
def build_pretty_json(json_string)
|
54
|
+
hash = JSON.parse(json_string)
|
55
|
+
JSON.pretty_generate(hash)
|
56
|
+
end
|
57
|
+
|
58
|
+
def send(build_params)
|
59
|
+
uri = URI.parse(ENV['SLACK_WEBHOOK_URL'])
|
60
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
61
|
+
http.use_ssl = true
|
62
|
+
begin
|
63
|
+
http.start do
|
64
|
+
request = Net::HTTP::Post.new(uri.path)
|
65
|
+
request.set_form_data(payload: build_params.to_json)
|
66
|
+
http.request(request)
|
67
|
+
end
|
68
|
+
rescue => e
|
69
|
+
raise $!.message
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
module Kobanzame
|
2
|
+
class Report
|
3
|
+
def initialize(result, params)
|
4
|
+
@result = result
|
5
|
+
@params = params
|
6
|
+
end
|
7
|
+
|
8
|
+
def duration
|
9
|
+
times = @result.map { |a| a[0] }
|
10
|
+
return '' if times.nil?
|
11
|
+
duration = {}
|
12
|
+
duration['duration'] = (times[-1] - times[0])
|
13
|
+
duration['unit'] = 'ms'
|
14
|
+
duration
|
15
|
+
end
|
16
|
+
|
17
|
+
def cpu_usage
|
18
|
+
cpus = @result.map { |a| a[1] }
|
19
|
+
return '' if cpus.nil?
|
20
|
+
usage = {}
|
21
|
+
usage['average'] = cpus.average
|
22
|
+
usage['max'] = cpus.max
|
23
|
+
usage['unit'] = '%'
|
24
|
+
usage
|
25
|
+
end
|
26
|
+
|
27
|
+
def memory_used
|
28
|
+
memories = @result.map { |a| a[2] }
|
29
|
+
return '' if memories.nil?
|
30
|
+
usage = {}
|
31
|
+
usage['average'] = memories.average
|
32
|
+
usage['max'] = memories.max
|
33
|
+
usage['unit'] = 'MiB'
|
34
|
+
usage
|
35
|
+
end
|
36
|
+
|
37
|
+
def text
|
38
|
+
m = "Memory Used: #{memory_used['average']}MiB(ave)/#{memory_used['max']}MiB(max)"
|
39
|
+
c = "CPU Usage: #{cpu_usage['average']}%(ave)/#{cpu_usage['max']}%(max)"
|
40
|
+
task_id = @params['task_id']
|
41
|
+
|
42
|
+
"REPORT TaskID: #{task_id} Duration: #{duration['duration']}ms, #{m}, #{c}"
|
43
|
+
end
|
44
|
+
|
45
|
+
def json
|
46
|
+
report = {}
|
47
|
+
|
48
|
+
report['duration'] = duration
|
49
|
+
report['memory_used'] = memory_used
|
50
|
+
report['cpu_usage'] = cpu_usage
|
51
|
+
|
52
|
+
res = {}
|
53
|
+
res['report'] = report
|
54
|
+
res['task_id'] = @params['task_id']
|
55
|
+
res['docker_name'] = @params['docker_name']
|
56
|
+
res.to_json
|
57
|
+
end
|
58
|
+
|
59
|
+
def csv
|
60
|
+
m = "#{memory_used['average']},#{memory_used['max']}"
|
61
|
+
c = "#{cpu_usage['average']},#{cpu_usage['max']}"
|
62
|
+
task_id = @params['task_id']
|
63
|
+
|
64
|
+
"#{task_id},#{duration['duration']},#{m},#{c}"
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module Kobanzame
|
2
|
+
class Utilities
|
3
|
+
def self.request(path)
|
4
|
+
url = URI.parse(ENV['ECS_CONTAINER_METADATA_URI'] + '/' + path)
|
5
|
+
req = Net::HTTP::Get.new(url.path)
|
6
|
+
res = Net::HTTP.start(url.host, url.port) {|http|
|
7
|
+
http.request(req)
|
8
|
+
}
|
9
|
+
return nil unless res.code == '200'
|
10
|
+
JSON.parse(res.body)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,113 @@
|
|
1
|
+
module Kobanzame
|
2
|
+
module Worker
|
3
|
+
def calc_cpu_percent(current, previous, interval = 1000000000)
|
4
|
+
# to Percent (x.x %)
|
5
|
+
# refer to: https://github.com/mackerelio/mackerel-container-agent/blob/ce7cb283a3cdbc690127c6f67c44bdce7b140619/platform/ecs/metric.go#L112-L115
|
6
|
+
((current['cpu_usage']['total_usage'].to_f - previous['cpu_usage']['total_usage'].to_f) / interval.to_f * 100).round(3)
|
7
|
+
end
|
8
|
+
|
9
|
+
def calc_memory_size(memory_stat)
|
10
|
+
# to Megabyte (x.xx MiB)
|
11
|
+
# refer to: https://github.com/mackerelio/mackerel-container-agent/blob/ce7cb283a3cdbc690127c6f67c44bdce7b140619/platform/ecs/metric.go#L117-L119
|
12
|
+
((memory_stat['usage'] - memory_stat['stats']['cache']).to_f / (1024.0 * 1024.0)).round(3)
|
13
|
+
end
|
14
|
+
|
15
|
+
def describe_target_container
|
16
|
+
# return Container Info
|
17
|
+
res = Kobanzame::Utilities.request('task')
|
18
|
+
return nil if res.nil?
|
19
|
+
res
|
20
|
+
end
|
21
|
+
|
22
|
+
def describe_target_container_metrics(container_id)
|
23
|
+
stat = nil
|
24
|
+
res = Kobanzame::Utilities.request('task/stats')
|
25
|
+
stat = res.dig(container_id) unless res.nil?
|
26
|
+
|
27
|
+
return nil if stat.nil? || stat.dig('memory_stats').empty?
|
28
|
+
memory_usage = calc_memory_size(stat['memory_stats'])
|
29
|
+
cpu_usage = calc_cpu_percent(stat['cpu_stats'], stat['precpu_stats'])
|
30
|
+
|
31
|
+
# Return format: ['Datetime', 'Used CPU unit', 'Used Memory size']
|
32
|
+
[Time.now.strftime('%s%L').to_i, cpu_usage, memory_usage]
|
33
|
+
end
|
34
|
+
|
35
|
+
def load_container_config
|
36
|
+
Kobanzame::Config.select_conf(config, 'container')
|
37
|
+
end
|
38
|
+
|
39
|
+
def load_metrics_config
|
40
|
+
Kobanzame::Config.select_conf(config, 'metrics')
|
41
|
+
end
|
42
|
+
|
43
|
+
def describe_container_info
|
44
|
+
cf = load_container_config
|
45
|
+
logger.info 'Waiting for starting target container.'
|
46
|
+
i = 1
|
47
|
+
container_info = ''
|
48
|
+
loop do
|
49
|
+
containers = describe_target_container
|
50
|
+
logger.warn 'Could not get all container information.' if containers.nil?
|
51
|
+
container_info = containers['Containers'].find { |c| c['Name'] == cf['name'] }
|
52
|
+
break if !container_info.nil? && container_info['DockerId'] != ''
|
53
|
+
# waiting for 120s
|
54
|
+
i += 1
|
55
|
+
if i > 5 then
|
56
|
+
logger.fatal 'Could not get the information of the batch container.'
|
57
|
+
exit 1
|
58
|
+
end
|
59
|
+
sleep cf['check_interval'].to_i
|
60
|
+
end
|
61
|
+
container_info['_check_interval'] = cf['check_interval'].to_i
|
62
|
+
container_info
|
63
|
+
end
|
64
|
+
|
65
|
+
def run
|
66
|
+
container_info = describe_container_info
|
67
|
+
mt = load_metrics_config
|
68
|
+
params = Kobanzame::Config.generate_task_params(container_info)
|
69
|
+
logger.info 'Collecting target container metrics.'
|
70
|
+
|
71
|
+
i = 1
|
72
|
+
result = []
|
73
|
+
nilbox = []
|
74
|
+
metbox = []
|
75
|
+
until @stop
|
76
|
+
metrics = describe_target_container_metrics(container_info.dig('DockerId'))
|
77
|
+
# metrics が 3 回取れなかったら, ループを抜ける
|
78
|
+
nilbox << nil if metrics.nil?
|
79
|
+
if nilbox.size > 3
|
80
|
+
if !mt.nil? || !mt == {}
|
81
|
+
Kobanzame::Metrics.new(config, params).publish(metbox) unless metbox.empty?
|
82
|
+
end
|
83
|
+
break
|
84
|
+
end
|
85
|
+
#
|
86
|
+
if !metrics.nil?
|
87
|
+
result << metrics
|
88
|
+
metbox << metrics
|
89
|
+
end
|
90
|
+
logger.debug metrics if config[:debug]
|
91
|
+
#
|
92
|
+
if i % 10 == 0
|
93
|
+
logger.info "Collected #{result.length} metrics..."
|
94
|
+
if !mt.nil? || !mt == {}
|
95
|
+
Kobanzame::Metrics.new(config, params).publish(metbox)
|
96
|
+
metbox = []
|
97
|
+
end
|
98
|
+
end
|
99
|
+
i += 1
|
100
|
+
#
|
101
|
+
sleep container_info['_check_interval']
|
102
|
+
end
|
103
|
+
|
104
|
+
repo = Kobanzame::Report.new(result, params)
|
105
|
+
Kobanzame::Outputs.new(config, params).publish(repo)
|
106
|
+
exit 0
|
107
|
+
end
|
108
|
+
|
109
|
+
def stop
|
110
|
+
@stop = true
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
metadata
ADDED
@@ -0,0 +1,160 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: kobanzame
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.8
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- inokappa
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2020-06-14 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.9'
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '3.0'
|
23
|
+
type: :development
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '1.9'
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '3.0'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: rake
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - ">="
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: 12.3.3
|
40
|
+
type: :development
|
41
|
+
prerelease: false
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - ">="
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: 12.3.3
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: rspec
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - "~>"
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '3.0'
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - "~>"
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: '3.0'
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: aws-sdk-cloudwatchlogs
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - ">="
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: '0'
|
68
|
+
type: :runtime
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - ">="
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '0'
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: aws-sdk-cloudwatch
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - ">="
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '0'
|
82
|
+
type: :runtime
|
83
|
+
prerelease: false
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '0'
|
89
|
+
- !ruby/object:Gem::Dependency
|
90
|
+
name: serverengine
|
91
|
+
requirement: !ruby/object:Gem::Requirement
|
92
|
+
requirements:
|
93
|
+
- - ">="
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
version: '0'
|
96
|
+
type: :runtime
|
97
|
+
prerelease: false
|
98
|
+
version_requirements: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - ">="
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '0'
|
103
|
+
description: kobanzame collects resources for ecs task.
|
104
|
+
email:
|
105
|
+
- inokara@gmail.com
|
106
|
+
executables:
|
107
|
+
- kobanzame
|
108
|
+
extensions: []
|
109
|
+
extra_rdoc_files: []
|
110
|
+
files:
|
111
|
+
- ".circleci/config.yml"
|
112
|
+
- ".gitignore"
|
113
|
+
- ".rspec"
|
114
|
+
- Gemfile
|
115
|
+
- Gemfile.lock
|
116
|
+
- README.md
|
117
|
+
- Rakefile
|
118
|
+
- bin/console
|
119
|
+
- bin/setup
|
120
|
+
- exe/kobanzame
|
121
|
+
- kobanzame.gemspec
|
122
|
+
- kobanzame.sample.json
|
123
|
+
- lib/kobanzame.rb
|
124
|
+
- lib/kobanzame/cli.rb
|
125
|
+
- lib/kobanzame/config.rb
|
126
|
+
- lib/kobanzame/ext.rb
|
127
|
+
- lib/kobanzame/metrics.rb
|
128
|
+
- lib/kobanzame/metrics/cloudwatch.rb
|
129
|
+
- lib/kobanzame/outputs.rb
|
130
|
+
- lib/kobanzame/outputs/cloudwatch_logs.rb
|
131
|
+
- lib/kobanzame/outputs/slack.rb
|
132
|
+
- lib/kobanzame/outputs/stdout.rb
|
133
|
+
- lib/kobanzame/report.rb
|
134
|
+
- lib/kobanzame/supervisor.rb
|
135
|
+
- lib/kobanzame/utilities.rb
|
136
|
+
- lib/kobanzame/version.rb
|
137
|
+
- lib/kobanzame/worker.rb
|
138
|
+
homepage: https://github.com/inokappa/kobanzame
|
139
|
+
licenses: []
|
140
|
+
metadata: {}
|
141
|
+
post_install_message:
|
142
|
+
rdoc_options: []
|
143
|
+
require_paths:
|
144
|
+
- lib
|
145
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
146
|
+
requirements:
|
147
|
+
- - ">="
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: '0'
|
150
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
151
|
+
requirements:
|
152
|
+
- - ">="
|
153
|
+
- !ruby/object:Gem::Version
|
154
|
+
version: '0'
|
155
|
+
requirements: []
|
156
|
+
rubygems_version: 3.0.3
|
157
|
+
signing_key:
|
158
|
+
specification_version: 4
|
159
|
+
summary: kobanzame collects resources for ecs task.
|
160
|
+
test_files: []
|