fluent-plugin-docker-stats 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2190acce2c335545f820f349e2d54821554e6ee9c0adc3a89f41c49515176f70
+  data.tar.gz: 82bea8766f3244fcfc455e0a21c468f575633eaf1b1f4eed5b8739e9be37c2c4
+SHA512:
+  metadata.gz: b386ed72c8160e716b6c2b5579be825993555488045469099d17d2c4bb052db2736b81e7523922b3dbac32c1858f0fc8ac43f43dae4355736d16c86f63c7b5d5
+  data.tar.gz: 90b739fe4c659014cb4eb68d8d8b7cc65ea30c9afa73ddaa03149eb7583eef2174f0658f20d2a8c23045b182217f43bea621e2ca5e7284d0f732e951e1cf8c7a

data/CHANGELOG

@@ -0,0 +1,3 @@
+## v1.0.0 2023/12/15
+
+- Initial release

data/Gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gem 'docker-api', '~> 1.34.2'
+
+gemspec

data/README.md

@@ -0,0 +1,45 @@
+# Fluentd Docker Stats Input Plugin
+
+This is a [Fluentd](http://www.fluentd.org) plugin to collect Docker stats periodically.
+Tested with Fluentd v1.16.2-1.1 and `docker-api` 1.34.2.
+
+## How it works
+
+The script connects to the Docker host using the `docker-api` SDK and periodically queries container statistics. 
+It provides insights into the resource usage, health, and other relevant information of the running Docker containers.
+
+
+## Installing
+
+Make sure you have a Ruby environment. Then initialize this repo:
+
+    gem install bundle
+
+Then:
+
+    bundle install
+
+## Example config
+
+```
+<source>
+  @type docker_stats
+  stats_interval 60s
+  tag docker.stats
+</source>
+```
+
+## Parameters
+
+* **stats_interval**: how often to poll Docker containers for stats. The default is every minute.
+* **tag**: The tag for the input source. The default value is "docker
+* **container_ids**: A list of container IDs for reading stats. The default value is empty, which will fetch stats for all containers.
+
+## Example output
+
+```
+2023-12-15 09:49:12.530109174 +0000 docker.stats: {"container_id":"e59768d2f1cc8448cc1e609324cdddf70e8e26557bfe307a0887d11ec2132af3","container_name":"mysql","created_time":"2023-07-26T01:39:21.286462238Z","status":"running","is_runn
+ing":true,"is_restarting":false,"is_paused":false,"is_oom_killed":false,"started_time":"2023-12-15T00:56:28.701735927Z","finished_time":"2023-12-15T00:56:24.669503934Z","mem_usage":264130560,"mem_limit":33356099584,"mem_max_usage":266227712,"cpu_system_usage":346646650000000,"cpu_total_usage":16206423900,"cpu_percent":0.004675199918995323,"networks":[{"network_name":"eth0","rx":2110,"tx":0}]}
+```
+
+

data/Rakefile

@@ -0,0 +1,10 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << "test" << "lib"
+  test.pattern = 'test/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-docker-stats.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-docker-stats"
+  spec.version       = "1.0.0"
+  spec.authors       = ["James Ma"]
+  spec.email         = ["jamesmawm@gmail.com"]
+  spec.summary       = %q{Fluentd plugin to collect Docker container stats}
+  spec.homepage      = "https://github.com/jamesmawm/fluent-plugin-docker-stats"
+  spec.license       = "Apache License, Version 2.0"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "fakefs"
+  spec.add_development_dependency "test-unit", "~> 3.1"
+  spec.add_development_dependency "minitest", "~> 5.8"
+  spec.add_runtime_dependency "fluentd"
+  spec.add_runtime_dependency "docker-api"
+end

data/lib/fluent/plugin/in_docker_stats.rb

@@ -0,0 +1,135 @@
+require 'fluent/plugin/input'
+require 'docker'
+require 'uri'
+
+module Fluent::Plugin
+
+  class DockerStatsInput < Input
+    Fluent::Plugin.register_input('docker_stats', self)
+
+    config_param :stats_interval, :string, :default => "60s"
+    config_param :tag, :string, :default => "docker"
+    config_param :container_ids, :array, :default => nil # mainly for testing
+
+    def initialize
+      super
+      puts "Found Docker details: #{Docker.version}"
+      puts "Using interval: #{@stats_interval}"
+      puts "Using tag: #{@tag}"
+    end
+
+    def configure(conf)
+      super
+    end
+
+    def start
+      @loop = Coolio::Loop.new
+      tw = TimerWatcher.new(@stats_interval, true, @log, &method(:get_metrics))
+      tw.attach(@loop)
+      @thread = Thread.new(&method(:run))
+    end
+
+    def run
+      @loop.run
+    rescue
+      log.error "unexpected error", :error => $!.to_s
+      log.error_backtrace
+    end
+
+    def get_metrics
+      ids = @container_ids || list_container_ids
+      ids.each do |container_id|
+        emit_container_stats(container_id)
+      end
+    end
+
+    def emit_container_stats(container_id)
+      puts "emit_container_stats: #{container_id}"
+      container = Docker::Container.get(container_id)
+
+      record = {
+        "container_id": container_id,
+        "container_name": container.info['Name'].sub(/^\//, ''),
+        "created_time": container.info["Created"]
+      }
+
+      state = container.info['State']
+      record["status"] = state['Status']
+      record["is_running"] = state['Running']
+      record["is_restarting"] = state['Restarting']
+      record["is_paused"] = state['Paused']
+      record["is_oom_killed"] = state['OOMKilled']
+      record["started_time"] = state['StartedAt']
+      record["finished_time"] = state['FinishedAt']
+
+      stats = container.stats(stream: false)
+
+      memory_stats = stats['memory_stats']
+      record["mem_usage"] = memory_stats['usage']
+      record["mem_limit"] = memory_stats['limit']
+      record["mem_max_usage"] = memory_stats['max_usage']
+
+      cpu_stats, = stats['cpu_stats']
+      cpu_usage = cpu_stats['cpu_usage']
+
+      cpu_system_usage = cpu_stats['system_cpu_usage']
+      cpu_total_usage = cpu_usage['total_usage']
+      cpu_percent = (cpu_total_usage.to_f / cpu_system_usage.to_f) * 100
+
+      record["cpu_system_usage"] = cpu_system_usage
+      record["cpu_total_usage"] = cpu_total_usage
+      record["cpu_percent"] = cpu_percent
+
+      record["networks"] = []
+      stats['networks'].each do |network_name, network_info|
+        record["networks"] << {
+          "network_name": network_name,
+          "rx": network_info['rx_bytes'],
+          "tx": network_info['tx_bytes'],
+        }
+      end
+
+      storage_stats = stats['storage_stats']
+      if stats['storage_stats'] && !stats['storage_stats'].empty?
+        record["volumes"] = []
+        volume_stats = storage_stats['volumes']
+        volume_stats.each do |volume_name, volume_info|
+          puts "Volume #{volume_name} - Used: #{volume_info['used']} bytes, Total: #{volume_info['total']} bytes"
+          record["volumes"] << {
+            "volume_name": volume_name,
+            "volume_used": volume_info['used'],
+            "volume_total": volume_info['total'],
+          }
+        end
+      end
+
+      router.emit(@tag, Fluent::Engine.now, record)
+    end
+
+    def list_container_ids
+      Docker::Container.all.map do |container|
+        container.id
+      end
+    end
+
+    def shutdown
+      @loop.stop
+      @thread.join
+    end
+
+    class TimerWatcher < Coolio::TimerWatcher
+      def initialize(interval, repeat, log, &callback)
+        @callback = callback
+        @log = log
+        super(interval, repeat)
+      end
+
+      def on_timer
+        @callback.call
+      rescue
+        @log.error $!.to_s
+        @log.error_backtrace
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,147 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-docker-stats
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- James Ma
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-12-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fakefs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.8'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: docker-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
+email:
+- jamesmawm@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG
+- Gemfile
+- README.md
+- Rakefile
+- fluent-plugin-docker-stats.gemspec
+- lib/fluent/plugin/in_docker_stats.rb
+homepage: https://github.com/jamesmawm/fluent-plugin-docker-stats
+licenses:
+- Apache License, Version 2.0
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.10
+signing_key:
+specification_version: 4
+summary: Fluentd plugin to collect Docker container stats
+test_files: []