kobanzame 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.circleci/config.yml +29 -0
- data/.gitignore +18 -0
- data/.rspec +3 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +57 -0
- data/README.md +64 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/exe/kobanzame +4 -0
- data/kobanzame.gemspec +29 -0
- data/kobanzame.sample.json +26 -0
- data/lib/kobanzame.rb +16 -0
- data/lib/kobanzame/cli.rb +44 -0
- data/lib/kobanzame/config.rb +40 -0
- data/lib/kobanzame/ext.rb +11 -0
- data/lib/kobanzame/metrics.rb +38 -0
- data/lib/kobanzame/metrics/cloudwatch.rb +70 -0
- data/lib/kobanzame/outputs.rb +44 -0
- data/lib/kobanzame/outputs/cloudwatch_logs.rb +72 -0
- data/lib/kobanzame/outputs/slack.rb +74 -0
- data/lib/kobanzame/outputs/stdout.rb +13 -0
- data/lib/kobanzame/report.rb +67 -0
- data/lib/kobanzame/supervisor.rb +14 -0
- data/lib/kobanzame/utilities.rb +13 -0
- data/lib/kobanzame/version.rb +3 -0
- data/lib/kobanzame/worker.rb +113 -0
- metadata +160 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 159e7d623735c2eecf51458f18b1bbfe0c5b2f3588693ca23876710265691e85
|
4
|
+
data.tar.gz: 2546fea926c29d499b6ba7e22f2e3ee1a106c137059f307f67636129182a38d2
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: d01b930e5042038c7f2809bd584e704802bdf5feb1ddfccb0306e93126ae26b71f91356d94c26a9714c2da50041d9cd687810e22b1d3f6525b0227d06ba31d1e
|
7
|
+
data.tar.gz: 91a656c17d523b5faf2f917dc8679af62e06206cee035000786999569f084e8855d087bb5f3a879e2263f85a01d6c0b5e2bedd7dec31cc54c3170fcff87f4100
|
@@ -0,0 +1,29 @@
|
|
1
|
+
version: 2
|
2
|
+
|
3
|
+
jobs:
|
4
|
+
build:
|
5
|
+
docker:
|
6
|
+
# specify the version you desire here
|
7
|
+
- image: cimg/ruby:2.6.5
|
8
|
+
working_directory: ~/repo
|
9
|
+
steps:
|
10
|
+
- checkout
|
11
|
+
- restore_cache:
|
12
|
+
keys:
|
13
|
+
- v1-dependencies-{{ checksum "Gemfile.lock" }}
|
14
|
+
# fallback to using the latest cache if no exact match is found
|
15
|
+
- v1-dependencies-
|
16
|
+
- run:
|
17
|
+
name: install dependencies
|
18
|
+
command: |
|
19
|
+
gem install bundler -v '1.17'
|
20
|
+
bundle install --jobs=4 --retry=3 --path vendor/bundle
|
21
|
+
- save_cache:
|
22
|
+
paths:
|
23
|
+
- ./vendor/bundle
|
24
|
+
key: v1-dependencies-{{ checksum "Gemfile.lock" }}
|
25
|
+
# run tests!
|
26
|
+
- run:
|
27
|
+
name: Run tests
|
28
|
+
command: |
|
29
|
+
bundle exec rake spec
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,57 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
kobanzame (0.0.8)
|
5
|
+
aws-sdk-cloudwatch
|
6
|
+
aws-sdk-cloudwatchlogs
|
7
|
+
serverengine
|
8
|
+
|
9
|
+
GEM
|
10
|
+
remote: https://rubygems.org/
|
11
|
+
specs:
|
12
|
+
aws-eventstream (1.1.0)
|
13
|
+
aws-partitions (1.329.0)
|
14
|
+
aws-sdk-cloudwatch (1.39.1)
|
15
|
+
aws-sdk-core (~> 3, >= 3.99.0)
|
16
|
+
aws-sigv4 (~> 1.1)
|
17
|
+
aws-sdk-cloudwatchlogs (1.32.1)
|
18
|
+
aws-sdk-core (~> 3, >= 3.99.0)
|
19
|
+
aws-sigv4 (~> 1.1)
|
20
|
+
aws-sdk-core (3.99.2)
|
21
|
+
aws-eventstream (~> 1, >= 1.0.2)
|
22
|
+
aws-partitions (~> 1, >= 1.239.0)
|
23
|
+
aws-sigv4 (~> 1.1)
|
24
|
+
jmespath (~> 1.0)
|
25
|
+
aws-sigv4 (1.1.4)
|
26
|
+
aws-eventstream (~> 1.0, >= 1.0.2)
|
27
|
+
diff-lcs (1.3)
|
28
|
+
jmespath (1.4.0)
|
29
|
+
rake (13.0.1)
|
30
|
+
rspec (3.9.0)
|
31
|
+
rspec-core (~> 3.9.0)
|
32
|
+
rspec-expectations (~> 3.9.0)
|
33
|
+
rspec-mocks (~> 3.9.0)
|
34
|
+
rspec-core (3.9.2)
|
35
|
+
rspec-support (~> 3.9.3)
|
36
|
+
rspec-expectations (3.9.2)
|
37
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
38
|
+
rspec-support (~> 3.9.0)
|
39
|
+
rspec-mocks (3.9.1)
|
40
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
41
|
+
rspec-support (~> 3.9.0)
|
42
|
+
rspec-support (3.9.3)
|
43
|
+
serverengine (2.2.1)
|
44
|
+
sigdump (~> 0.2.2)
|
45
|
+
sigdump (0.2.4)
|
46
|
+
|
47
|
+
PLATFORMS
|
48
|
+
ruby
|
49
|
+
|
50
|
+
DEPENDENCIES
|
51
|
+
bundler (>= 1.9, < 3.0)
|
52
|
+
kobanzame!
|
53
|
+
rake (>= 12.3.3)
|
54
|
+
rspec (~> 3.0)
|
55
|
+
|
56
|
+
BUNDLED WITH
|
57
|
+
1.17.2
|
data/README.md
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
# Kobanzame [](https://circleci.com/gh/inokappa/kobanzame)
|
2
|
+
|
3
|
+

|
4
|
+
|
5
|
+
## About
|
6
|
+
|
7
|
+
Resource monitoring tool for ECS Task. It sticks to a Docker container like a `Kobanzame (black shark)` and gets information about the container.
|
8
|
+
|
9
|
+
## Installation
|
10
|
+
|
11
|
+
Add this line to your application's Gemfile:
|
12
|
+
|
13
|
+
```ruby
|
14
|
+
gem 'kobanzame'
|
15
|
+
```
|
16
|
+
|
17
|
+
And then execute:
|
18
|
+
|
19
|
+
$ bundle
|
20
|
+
|
21
|
+
Or install it yourself as:
|
22
|
+
|
23
|
+
$ gem install kobanzame
|
24
|
+
|
25
|
+
## Usage
|
26
|
+
|
27
|
+
Write kobanzame.json.
|
28
|
+
|
29
|
+
```json
|
30
|
+
{
|
31
|
+
"container": {
|
32
|
+
"name": "batch-worker",
|
33
|
+
"check_interval": 1,
|
34
|
+
"report_format": "text"
|
35
|
+
},
|
36
|
+
"metrics": {
|
37
|
+
"name": "cloudwatch",
|
38
|
+
"namespace": "Custom/Kobanzame"
|
39
|
+
},
|
40
|
+
"outputs": [
|
41
|
+
{
|
42
|
+
"name": "cloudwatch_logs",
|
43
|
+
"log_group_name": "kobanzame-sample",
|
44
|
+
"log_stream_prefix": "kobanzame"
|
45
|
+
}
|
46
|
+
]
|
47
|
+
}
|
48
|
+
```
|
49
|
+
|
50
|
+
Starting kobanzame.
|
51
|
+
|
52
|
+
```sh
|
53
|
+
$ kobanzame --config=kobanzame.json
|
54
|
+
```
|
55
|
+
|
56
|
+
## Development
|
57
|
+
|
58
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
59
|
+
|
60
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
61
|
+
|
62
|
+
## Contributing
|
63
|
+
|
64
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/kobanzame.
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "kobanzame"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
data/exe/kobanzame
ADDED
data/kobanzame.gemspec
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'kobanzame/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = 'kobanzame'
|
8
|
+
spec.version = Kobanzame::VERSION
|
9
|
+
spec.authors = ['inokappa']
|
10
|
+
spec.email = ['inokara@gmail.com']
|
11
|
+
spec.summary = %q{kobanzame collects resources for ecs task.}
|
12
|
+
spec.description = %q{kobanzame collects resources for ecs task.}
|
13
|
+
spec.homepage = 'https://github.com/inokappa/kobanzame'
|
14
|
+
|
15
|
+
spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
|
16
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|example|docs)/}) }
|
17
|
+
end
|
18
|
+
spec.bindir = 'exe'
|
19
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
20
|
+
spec.require_paths = ['lib']
|
21
|
+
|
22
|
+
spec.add_development_dependency 'bundler', '>= 1.9', '< 3.0'
|
23
|
+
spec.add_development_dependency 'rake', '>= 12.3.3'
|
24
|
+
spec.add_development_dependency 'rspec', '~> 3.0'
|
25
|
+
|
26
|
+
spec.add_runtime_dependency 'aws-sdk-cloudwatchlogs'
|
27
|
+
spec.add_runtime_dependency 'aws-sdk-cloudwatch'
|
28
|
+
spec.add_runtime_dependency 'serverengine'
|
29
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
{
|
2
|
+
"container": {
|
3
|
+
"name": "batch-worker",
|
4
|
+
"check_interval": 1,
|
5
|
+
"report_format": "json"
|
6
|
+
},
|
7
|
+
"metrics": {
|
8
|
+
"name": "cloudwatch",
|
9
|
+
"namespace": "Custom/Kobanzame"
|
10
|
+
},
|
11
|
+
"outputs": [
|
12
|
+
{
|
13
|
+
"name": "cloudwatch_logs",
|
14
|
+
"log_group_name": "kobanzame-sample",
|
15
|
+
"log_stream_prefix": "kobanzame",
|
16
|
+
"log_stream_name": "batch-worker-resource-statistics"
|
17
|
+
},
|
18
|
+
{
|
19
|
+
"name": "slack",
|
20
|
+
"webhook_url": "",
|
21
|
+
"title": "Kobanzame Sample Output",
|
22
|
+
"user_name": "kobanzame-sample",
|
23
|
+
"icon_emoji": ":shark:"
|
24
|
+
}
|
25
|
+
]
|
26
|
+
}
|
data/lib/kobanzame.rb
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'serverengine'
|
3
|
+
require 'kobanzame/config'
|
4
|
+
require 'kobanzame/ext'
|
5
|
+
require 'kobanzame/metrics'
|
6
|
+
require 'kobanzame/outputs'
|
7
|
+
require 'kobanzame/report'
|
8
|
+
require 'kobanzame/supervisor'
|
9
|
+
require 'kobanzame/utilities'
|
10
|
+
require 'kobanzame/version'
|
11
|
+
require 'kobanzame/worker'
|
12
|
+
|
13
|
+
module Kobanzame
|
14
|
+
class Error < StandardError; end
|
15
|
+
# Your code goes here...
|
16
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
require 'kobanzame'
|
3
|
+
|
4
|
+
op = OptionParser.new
|
5
|
+
op.version = Kobanzame::VERSION
|
6
|
+
|
7
|
+
opts = {
|
8
|
+
config: 'kobanzame.json',
|
9
|
+
pid_file: 'kobanzame.pid',
|
10
|
+
log_file: STDOUT,
|
11
|
+
daemonize: false,
|
12
|
+
debug: false
|
13
|
+
}
|
14
|
+
|
15
|
+
op.on('-c', '--config PATH', "config file path (default: #{opts[:config]})") {|v|
|
16
|
+
opts[:config] = v
|
17
|
+
}
|
18
|
+
|
19
|
+
op.on('-d', '--daemonize', "enable daemonize (default: #{opts[:daemonize]})") {|v|
|
20
|
+
opts[:daemonize] = v
|
21
|
+
}
|
22
|
+
|
23
|
+
op.on('-p', '--pid-file PATH', "pid file path (default: #{opts[:pid_file]})") {|v|
|
24
|
+
opts[:pid_file] = v
|
25
|
+
}
|
26
|
+
|
27
|
+
op.on('-l', '--log-file PATH', "log file path (default: STDOUT)") {|v|
|
28
|
+
opts[:log_file] = v
|
29
|
+
}
|
30
|
+
|
31
|
+
op.on('-D', '--debug', "start with debug mode (default: #{opts[:debug]})") {|v|
|
32
|
+
opts[:debug] = v
|
33
|
+
}
|
34
|
+
|
35
|
+
begin
|
36
|
+
op.parse(ARGV)
|
37
|
+
rescue OptionParser::InvalidOption => ex
|
38
|
+
puts op.to_s
|
39
|
+
puts "error: #{ex.message}"
|
40
|
+
exit 1
|
41
|
+
end
|
42
|
+
|
43
|
+
k = Kobanzame::Supervisor.new(opts)
|
44
|
+
k.start
|
@@ -0,0 +1,40 @@
|
|
1
|
+
module Kobanzame
|
2
|
+
class Config
|
3
|
+
def self.load_config(opts)
|
4
|
+
cfg = read_json(opts[:config])
|
5
|
+
se_config = {
|
6
|
+
debug: opts[:debug],
|
7
|
+
daemonize: opts[:daemonize],
|
8
|
+
pid_path: opts[:pid_file],
|
9
|
+
log: opts[:log_file],
|
10
|
+
log_level: 'debug',
|
11
|
+
kobanzame_config: cfg
|
12
|
+
}
|
13
|
+
se_config
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.read_json(path)
|
17
|
+
config_hash = ''
|
18
|
+
File.open(path) do |file|
|
19
|
+
config_hash = JSON.load(file)
|
20
|
+
end
|
21
|
+
config_hash
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.select_conf(config, key)
|
25
|
+
konf = config[:kobanzame_config]
|
26
|
+
konf[key]
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.generate_task_params(container_info)
|
30
|
+
params = {}
|
31
|
+
params['task_id'] = (0...32).map{ (65 + rand(26)).chr }.join
|
32
|
+
if container_info.dig('Labels', 'com.amazonaws.ecs.task-arn')
|
33
|
+
params['task_id'] = container_info.dig('Labels', 'com.amazonaws.ecs.task-arn').split('/')[-1]
|
34
|
+
end
|
35
|
+
params['docker_id'] = container_info.dig('DockerId')
|
36
|
+
params['docker_name'] = container_info.dig('DockerName')
|
37
|
+
params
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
module Kobanzame
|
2
|
+
class Metrics
|
3
|
+
def initialize(conf, params)
|
4
|
+
@metrics_conf = Kobanzame::Config.select_conf(conf, 'metrics')
|
5
|
+
@params = params
|
6
|
+
@conf = conf
|
7
|
+
end
|
8
|
+
|
9
|
+
def logger
|
10
|
+
@logger ||= ServerEngine::DaemonLogger.new(@conf[:log] || STDOUT, @conf)
|
11
|
+
end
|
12
|
+
|
13
|
+
def publish(result)
|
14
|
+
load_output_modules.each do |mod|
|
15
|
+
begin
|
16
|
+
require "kobanzame/metrics/#{mod}"
|
17
|
+
class_name = "Kobanzame::Metric::#{mod.camelize}"
|
18
|
+
Object.const_get(class_name).new(@metrics_conf, @params).publish(result)
|
19
|
+
logger.info("Sent information to #{mod}")
|
20
|
+
rescue LoadError => ex
|
21
|
+
logger.warn("Could not load #{mod} module.")
|
22
|
+
rescue => ex
|
23
|
+
logger.warn("Could not send information to #{mod}. Reason: #{ex.message}")
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def load_output_modules
|
31
|
+
modules = []
|
32
|
+
Dir.glob(File.dirname(__FILE__) + '/metrics/*').each do |r|
|
33
|
+
modules << File.basename(r, '.rb') unless r.include?('stub')
|
34
|
+
end
|
35
|
+
modules.sort
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
require 'time'
|
2
|
+
require 'aws-sdk-cloudwatch'
|
3
|
+
|
4
|
+
module Kobanzame
|
5
|
+
module Metric
|
6
|
+
class Cloudwatch
|
7
|
+
def initialize(metric_conf, params)
|
8
|
+
@namespace = metric_conf['namespace']
|
9
|
+
@task_id = params['task_id']
|
10
|
+
end
|
11
|
+
|
12
|
+
def cw
|
13
|
+
@cw ||= Aws::CloudWatch::Client.new
|
14
|
+
end
|
15
|
+
|
16
|
+
def publish(result)
|
17
|
+
data = write_metric_data(result)
|
18
|
+
begin
|
19
|
+
cw.put_metric_data(data)
|
20
|
+
rescue Aws::CloudWatch::Errors::ServiceError
|
21
|
+
raise $!.message
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
# Input Data: [[timestamp, CPU Usage, Memory Used], [1591710550364, 0.0, 0.621], [1591710553452, 0.0, 0.621]]
|
28
|
+
def write_metric_data(datas)
|
29
|
+
metric = {}
|
30
|
+
metric_datas = []
|
31
|
+
metric[:namespace] = @namespace
|
32
|
+
datas.each do |data|
|
33
|
+
metric_data = [
|
34
|
+
{
|
35
|
+
metric_name: 'CPU Usage',
|
36
|
+
dimensions: dimentions,
|
37
|
+
timestamp: timestamp(data[0]),
|
38
|
+
value: data[1],
|
39
|
+
unit: 'Percent'
|
40
|
+
},
|
41
|
+
{
|
42
|
+
metric_name: 'Memory Used',
|
43
|
+
dimensions: dimentions,
|
44
|
+
timestamp: timestamp(data[0]),
|
45
|
+
value: data[2],
|
46
|
+
unit: 'Megabits'
|
47
|
+
}
|
48
|
+
]
|
49
|
+
metric_datas << metric_data
|
50
|
+
end
|
51
|
+
metric[:metric_data] = metric_datas.flatten
|
52
|
+
metric
|
53
|
+
end
|
54
|
+
|
55
|
+
def dimentions
|
56
|
+
dims = [
|
57
|
+
{
|
58
|
+
name: 'TaskId',
|
59
|
+
value: @task_id,
|
60
|
+
}
|
61
|
+
]
|
62
|
+
dims
|
63
|
+
end
|
64
|
+
|
65
|
+
def timestamp(t)
|
66
|
+
DateTime.strptime((t.to_i/1000).to_s, '%s').iso8601
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
module Kobanzame
|
2
|
+
class Outputs
|
3
|
+
def initialize(conf, params)
|
4
|
+
@outputs_conf = Kobanzame::Config.select_conf(conf, 'outputs')
|
5
|
+
@container_conf = Kobanzame::Config.select_conf(conf, 'container')
|
6
|
+
@params = params
|
7
|
+
@conf = conf
|
8
|
+
end
|
9
|
+
|
10
|
+
def logger
|
11
|
+
@logger ||= ServerEngine::DaemonLogger.new(@conf[:log] || STDOUT, @conf)
|
12
|
+
end
|
13
|
+
|
14
|
+
def publish(result)
|
15
|
+
load_output_modules.each do |mod|
|
16
|
+
output_conf = @outputs_conf.select { |c| c['name'] == mod }.last
|
17
|
+
if output_conf.nil?
|
18
|
+
logger.warn("Skip output to #{mod}")
|
19
|
+
next
|
20
|
+
end
|
21
|
+
begin
|
22
|
+
require "kobanzame/outputs/#{mod}"
|
23
|
+
class_name = "Kobanzame::Output::#{mod.camelize}"
|
24
|
+
Object.const_get(class_name).new(output_conf, @container_conf, @params).publish(result)
|
25
|
+
logger.info("Sent information to #{mod}")
|
26
|
+
rescue LoadError
|
27
|
+
logger.warn("Could not load #{mod} module.")
|
28
|
+
rescue => ex
|
29
|
+
logger.warn("Could not send information to #{mod}. Reason: #{ex.message}")
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
private
|
35
|
+
|
36
|
+
def load_output_modules
|
37
|
+
modules = []
|
38
|
+
Dir.glob(File.dirname(__FILE__) + '/outputs/*').each do |r|
|
39
|
+
modules << File.basename(r, '.rb') unless r.include?('stub')
|
40
|
+
end
|
41
|
+
modules.sort
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
require 'aws-sdk-cloudwatchlogs'
|
2
|
+
|
3
|
+
module Kobanzame
|
4
|
+
module Output
|
5
|
+
class CloudwatchLogs
|
6
|
+
def initialize(output_conf, container_conf, params)
|
7
|
+
@log_group_name = output_conf['log_group_name']
|
8
|
+
@log_stream_prefix = output_conf['log_stream_prefix']
|
9
|
+
@log_stream_name = output_conf['log_stream_name']
|
10
|
+
@container_name = container_conf['name']
|
11
|
+
@format = container_conf['report_format']
|
12
|
+
@task_id = params['task_id']
|
13
|
+
end
|
14
|
+
|
15
|
+
def cwlog
|
16
|
+
@cwlog ||= Aws::CloudWatchLogs::Client.new
|
17
|
+
end
|
18
|
+
|
19
|
+
def publish(result)
|
20
|
+
event = {
|
21
|
+
log_group_name: @log_group_name,
|
22
|
+
log_stream_name: log_stream_name,
|
23
|
+
log_events: [{
|
24
|
+
timestamp: (Time.now.utc.to_f.round(3) * 1000).to_i,
|
25
|
+
message: result.send(@format)
|
26
|
+
}]
|
27
|
+
}
|
28
|
+
event['sequence_token'] = upload_sequence_token if upload_sequence_token != ''
|
29
|
+
begin
|
30
|
+
cwlog.put_log_events(event)
|
31
|
+
rescue Aws::CloudWatchLogs::Errors::ServiceError
|
32
|
+
raise $!.message
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def log_stream_name
|
39
|
+
return @task_id if @log_stream_prefix.nil? && @log_stream_name.nil?
|
40
|
+
return @log_stream_name if @log_stream_prefix.nil?
|
41
|
+
@log_stream_prefix + '/' + @container_name + '/' + @task_id
|
42
|
+
end
|
43
|
+
|
44
|
+
def upload_sequence_token
|
45
|
+
return '' if log_streams.empty?
|
46
|
+
log_streams[0]['upload_sequence_token']
|
47
|
+
end
|
48
|
+
|
49
|
+
def log_streams
|
50
|
+
begin
|
51
|
+
res = cwlog.describe_log_streams({
|
52
|
+
log_group_name: @log_group_name,
|
53
|
+
log_stream_name_prefix: log_stream_name
|
54
|
+
})
|
55
|
+
rescue Aws::CloudWatchLogs::Errors::ServiceError
|
56
|
+
raise $!.message
|
57
|
+
end
|
58
|
+
|
59
|
+
begin
|
60
|
+
cwlog.create_log_stream({
|
61
|
+
log_group_name: @log_group_name,
|
62
|
+
log_stream_name: log_stream_name
|
63
|
+
})
|
64
|
+
return []
|
65
|
+
rescue Aws::CloudWatchLogs::Errors::ServiceError
|
66
|
+
raise $!.message
|
67
|
+
end if res['log_streams'].empty?
|
68
|
+
res['log_streams']
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'net/http'
|
3
|
+
|
4
|
+
module Kobanzame
|
5
|
+
module Output
|
6
|
+
class Slack
|
7
|
+
def initialize(output_conf, container_conf, params)
|
8
|
+
@format = container_conf['report_format']
|
9
|
+
@output_conf = output_conf
|
10
|
+
end
|
11
|
+
|
12
|
+
def publish(result)
|
13
|
+
attachements = build_attachements(result.send(@format))
|
14
|
+
params = build_params(attachements)
|
15
|
+
send(params)
|
16
|
+
end
|
17
|
+
|
18
|
+
private
|
19
|
+
|
20
|
+
def build_attachements(result)
|
21
|
+
result = build_pretty_json(result) if result.start_with?('{')
|
22
|
+
value = <<-"VALUE"
|
23
|
+
```
|
24
|
+
#{result}
|
25
|
+
```
|
26
|
+
VALUE
|
27
|
+
attachments = [
|
28
|
+
{
|
29
|
+
'fallback': '',
|
30
|
+
'title': @output_conf['title'],
|
31
|
+
'fields': [
|
32
|
+
{
|
33
|
+
'title': 'Output',
|
34
|
+
'value': value,
|
35
|
+
'short': false
|
36
|
+
},
|
37
|
+
]
|
38
|
+
}
|
39
|
+
]
|
40
|
+
attachments
|
41
|
+
end
|
42
|
+
|
43
|
+
def build_params(attachments)
|
44
|
+
params = {
|
45
|
+
username: @output_conf['user_name'],
|
46
|
+
icon_emoji: @output_conf['icon_emoji'] ,
|
47
|
+
text: '',
|
48
|
+
attachments: attachments
|
49
|
+
}
|
50
|
+
params
|
51
|
+
end
|
52
|
+
|
53
|
+
def build_pretty_json(json_string)
|
54
|
+
hash = JSON.parse(json_string)
|
55
|
+
JSON.pretty_generate(hash)
|
56
|
+
end
|
57
|
+
|
58
|
+
def send(build_params)
|
59
|
+
uri = URI.parse(ENV['SLACK_WEBHOOK_URL'])
|
60
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
61
|
+
http.use_ssl = true
|
62
|
+
begin
|
63
|
+
http.start do
|
64
|
+
request = Net::HTTP::Post.new(uri.path)
|
65
|
+
request.set_form_data(payload: build_params.to_json)
|
66
|
+
http.request(request)
|
67
|
+
end
|
68
|
+
rescue => e
|
69
|
+
raise $!.message
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
module Kobanzame
|
2
|
+
class Report
|
3
|
+
def initialize(result, params)
|
4
|
+
@result = result
|
5
|
+
@params = params
|
6
|
+
end
|
7
|
+
|
8
|
+
def duration
|
9
|
+
times = @result.map { |a| a[0] }
|
10
|
+
return '' if times.nil?
|
11
|
+
duration = {}
|
12
|
+
duration['duration'] = (times[-1] - times[0])
|
13
|
+
duration['unit'] = 'ms'
|
14
|
+
duration
|
15
|
+
end
|
16
|
+
|
17
|
+
def cpu_usage
|
18
|
+
cpus = @result.map { |a| a[1] }
|
19
|
+
return '' if cpus.nil?
|
20
|
+
usage = {}
|
21
|
+
usage['average'] = cpus.average
|
22
|
+
usage['max'] = cpus.max
|
23
|
+
usage['unit'] = '%'
|
24
|
+
usage
|
25
|
+
end
|
26
|
+
|
27
|
+
def memory_used
|
28
|
+
memories = @result.map { |a| a[2] }
|
29
|
+
return '' if memories.nil?
|
30
|
+
usage = {}
|
31
|
+
usage['average'] = memories.average
|
32
|
+
usage['max'] = memories.max
|
33
|
+
usage['unit'] = 'MiB'
|
34
|
+
usage
|
35
|
+
end
|
36
|
+
|
37
|
+
def text
|
38
|
+
m = "Memory Used: #{memory_used['average']}MiB(ave)/#{memory_used['max']}MiB(max)"
|
39
|
+
c = "CPU Usage: #{cpu_usage['average']}%(ave)/#{cpu_usage['max']}%(max)"
|
40
|
+
task_id = @params['task_id']
|
41
|
+
|
42
|
+
"REPORT TaskID: #{task_id} Duration: #{duration['duration']}ms, #{m}, #{c}"
|
43
|
+
end
|
44
|
+
|
45
|
+
def json
|
46
|
+
report = {}
|
47
|
+
|
48
|
+
report['duration'] = duration
|
49
|
+
report['memory_used'] = memory_used
|
50
|
+
report['cpu_usage'] = cpu_usage
|
51
|
+
|
52
|
+
res = {}
|
53
|
+
res['report'] = report
|
54
|
+
res['task_id'] = @params['task_id']
|
55
|
+
res['docker_name'] = @params['docker_name']
|
56
|
+
res.to_json
|
57
|
+
end
|
58
|
+
|
59
|
+
def csv
|
60
|
+
m = "#{memory_used['average']},#{memory_used['max']}"
|
61
|
+
c = "#{cpu_usage['average']},#{cpu_usage['max']}"
|
62
|
+
task_id = @params['task_id']
|
63
|
+
|
64
|
+
"#{task_id},#{duration['duration']},#{m},#{c}"
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module Kobanzame
|
2
|
+
class Utilities
|
3
|
+
def self.request(path)
|
4
|
+
url = URI.parse(ENV['ECS_CONTAINER_METADATA_URI'] + '/' + path)
|
5
|
+
req = Net::HTTP::Get.new(url.path)
|
6
|
+
res = Net::HTTP.start(url.host, url.port) {|http|
|
7
|
+
http.request(req)
|
8
|
+
}
|
9
|
+
return nil unless res.code == '200'
|
10
|
+
JSON.parse(res.body)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,113 @@
|
|
1
|
+
module Kobanzame
|
2
|
+
module Worker
|
3
|
+
def calc_cpu_percent(current, previous, interval = 1000000000)
|
4
|
+
# to Percent (x.x %)
|
5
|
+
# refer to: https://github.com/mackerelio/mackerel-container-agent/blob/ce7cb283a3cdbc690127c6f67c44bdce7b140619/platform/ecs/metric.go#L112-L115
|
6
|
+
((current['cpu_usage']['total_usage'].to_f - previous['cpu_usage']['total_usage'].to_f) / interval.to_f * 100).round(3)
|
7
|
+
end
|
8
|
+
|
9
|
+
def calc_memory_size(memory_stat)
|
10
|
+
# to Megabyte (x.xx MiB)
|
11
|
+
# refer to: https://github.com/mackerelio/mackerel-container-agent/blob/ce7cb283a3cdbc690127c6f67c44bdce7b140619/platform/ecs/metric.go#L117-L119
|
12
|
+
((memory_stat['usage'] - memory_stat['stats']['cache']).to_f / (1024.0 * 1024.0)).round(3)
|
13
|
+
end
|
14
|
+
|
15
|
+
def describe_target_container
|
16
|
+
# return Container Info
|
17
|
+
res = Kobanzame::Utilities.request('task')
|
18
|
+
return nil if res.nil?
|
19
|
+
res
|
20
|
+
end
|
21
|
+
|
22
|
+
def describe_target_container_metrics(container_id)
|
23
|
+
stat = nil
|
24
|
+
res = Kobanzame::Utilities.request('task/stats')
|
25
|
+
stat = res.dig(container_id) unless res.nil?
|
26
|
+
|
27
|
+
return nil if stat.nil? || stat.dig('memory_stats').empty?
|
28
|
+
memory_usage = calc_memory_size(stat['memory_stats'])
|
29
|
+
cpu_usage = calc_cpu_percent(stat['cpu_stats'], stat['precpu_stats'])
|
30
|
+
|
31
|
+
# Return format: ['Datetime', 'Used CPU unit', 'Used Memory size']
|
32
|
+
[Time.now.strftime('%s%L').to_i, cpu_usage, memory_usage]
|
33
|
+
end
|
34
|
+
|
35
|
+
def load_container_config
|
36
|
+
Kobanzame::Config.select_conf(config, 'container')
|
37
|
+
end
|
38
|
+
|
39
|
+
def load_metrics_config
|
40
|
+
Kobanzame::Config.select_conf(config, 'metrics')
|
41
|
+
end
|
42
|
+
|
43
|
+
def describe_container_info
|
44
|
+
cf = load_container_config
|
45
|
+
logger.info 'Waiting for starting target container.'
|
46
|
+
i = 1
|
47
|
+
container_info = ''
|
48
|
+
loop do
|
49
|
+
containers = describe_target_container
|
50
|
+
logger.warn 'Could not get all container information.' if containers.nil?
|
51
|
+
container_info = containers['Containers'].find { |c| c['Name'] == cf['name'] }
|
52
|
+
break if !container_info.nil? && container_info['DockerId'] != ''
|
53
|
+
# waiting for 120s
|
54
|
+
i += 1
|
55
|
+
if i > 5 then
|
56
|
+
logger.fatal 'Could not get the information of the batch container.'
|
57
|
+
exit 1
|
58
|
+
end
|
59
|
+
sleep cf['check_interval'].to_i
|
60
|
+
end
|
61
|
+
container_info['_check_interval'] = cf['check_interval'].to_i
|
62
|
+
container_info
|
63
|
+
end
|
64
|
+
|
65
|
+
def run
|
66
|
+
container_info = describe_container_info
|
67
|
+
mt = load_metrics_config
|
68
|
+
params = Kobanzame::Config.generate_task_params(container_info)
|
69
|
+
logger.info 'Collecting target container metrics.'
|
70
|
+
|
71
|
+
i = 1
|
72
|
+
result = []
|
73
|
+
nilbox = []
|
74
|
+
metbox = []
|
75
|
+
until @stop
|
76
|
+
metrics = describe_target_container_metrics(container_info.dig('DockerId'))
|
77
|
+
# metrics が 3 回取れなかったら, ループを抜ける
|
78
|
+
nilbox << nil if metrics.nil?
|
79
|
+
if nilbox.size > 3
|
80
|
+
if !mt.nil? || !mt == {}
|
81
|
+
Kobanzame::Metrics.new(config, params).publish(metbox) unless metbox.empty?
|
82
|
+
end
|
83
|
+
break
|
84
|
+
end
|
85
|
+
#
|
86
|
+
if !metrics.nil?
|
87
|
+
result << metrics
|
88
|
+
metbox << metrics
|
89
|
+
end
|
90
|
+
logger.debug metrics if config[:debug]
|
91
|
+
#
|
92
|
+
if i % 10 == 0
|
93
|
+
logger.info "Collected #{result.length} metrics..."
|
94
|
+
if !mt.nil? || !mt == {}
|
95
|
+
Kobanzame::Metrics.new(config, params).publish(metbox)
|
96
|
+
metbox = []
|
97
|
+
end
|
98
|
+
end
|
99
|
+
i += 1
|
100
|
+
#
|
101
|
+
sleep container_info['_check_interval']
|
102
|
+
end
|
103
|
+
|
104
|
+
repo = Kobanzame::Report.new(result, params)
|
105
|
+
Kobanzame::Outputs.new(config, params).publish(repo)
|
106
|
+
exit 0
|
107
|
+
end
|
108
|
+
|
109
|
+
def stop
|
110
|
+
@stop = true
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
metadata
ADDED
@@ -0,0 +1,160 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: kobanzame
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.8
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- inokappa
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2020-06-14 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.9'
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '3.0'
|
23
|
+
type: :development
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '1.9'
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '3.0'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: rake
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - ">="
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: 12.3.3
|
40
|
+
type: :development
|
41
|
+
prerelease: false
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - ">="
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: 12.3.3
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: rspec
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - "~>"
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '3.0'
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - "~>"
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: '3.0'
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: aws-sdk-cloudwatchlogs
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - ">="
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: '0'
|
68
|
+
type: :runtime
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - ">="
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '0'
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: aws-sdk-cloudwatch
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - ">="
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '0'
|
82
|
+
type: :runtime
|
83
|
+
prerelease: false
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '0'
|
89
|
+
- !ruby/object:Gem::Dependency
|
90
|
+
name: serverengine
|
91
|
+
requirement: !ruby/object:Gem::Requirement
|
92
|
+
requirements:
|
93
|
+
- - ">="
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
version: '0'
|
96
|
+
type: :runtime
|
97
|
+
prerelease: false
|
98
|
+
version_requirements: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - ">="
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '0'
|
103
|
+
description: kobanzame collects resources for ecs task.
|
104
|
+
email:
|
105
|
+
- inokara@gmail.com
|
106
|
+
executables:
|
107
|
+
- kobanzame
|
108
|
+
extensions: []
|
109
|
+
extra_rdoc_files: []
|
110
|
+
files:
|
111
|
+
- ".circleci/config.yml"
|
112
|
+
- ".gitignore"
|
113
|
+
- ".rspec"
|
114
|
+
- Gemfile
|
115
|
+
- Gemfile.lock
|
116
|
+
- README.md
|
117
|
+
- Rakefile
|
118
|
+
- bin/console
|
119
|
+
- bin/setup
|
120
|
+
- exe/kobanzame
|
121
|
+
- kobanzame.gemspec
|
122
|
+
- kobanzame.sample.json
|
123
|
+
- lib/kobanzame.rb
|
124
|
+
- lib/kobanzame/cli.rb
|
125
|
+
- lib/kobanzame/config.rb
|
126
|
+
- lib/kobanzame/ext.rb
|
127
|
+
- lib/kobanzame/metrics.rb
|
128
|
+
- lib/kobanzame/metrics/cloudwatch.rb
|
129
|
+
- lib/kobanzame/outputs.rb
|
130
|
+
- lib/kobanzame/outputs/cloudwatch_logs.rb
|
131
|
+
- lib/kobanzame/outputs/slack.rb
|
132
|
+
- lib/kobanzame/outputs/stdout.rb
|
133
|
+
- lib/kobanzame/report.rb
|
134
|
+
- lib/kobanzame/supervisor.rb
|
135
|
+
- lib/kobanzame/utilities.rb
|
136
|
+
- lib/kobanzame/version.rb
|
137
|
+
- lib/kobanzame/worker.rb
|
138
|
+
homepage: https://github.com/inokappa/kobanzame
|
139
|
+
licenses: []
|
140
|
+
metadata: {}
|
141
|
+
post_install_message:
|
142
|
+
rdoc_options: []
|
143
|
+
require_paths:
|
144
|
+
- lib
|
145
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
146
|
+
requirements:
|
147
|
+
- - ">="
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: '0'
|
150
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
151
|
+
requirements:
|
152
|
+
- - ">="
|
153
|
+
- !ruby/object:Gem::Version
|
154
|
+
version: '0'
|
155
|
+
requirements: []
|
156
|
+
rubygems_version: 3.0.3
|
157
|
+
signing_key:
|
158
|
+
specification_version: 4
|
159
|
+
summary: kobanzame collects resources for ecs task.
|
160
|
+
test_files: []
|