fluent-plugin-docker-tag-resolver 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8de876a20e60190cee56842ee3bb1cb267c49302
+  data.tar.gz: cfd31975efe710f63f9985d8363c288eb92cc772
+SHA512:
+  metadata.gz: 76b56e4bb5d8675fffb471ade8384d779447969272301fe2c1312f98d63905f3b9af79f6ea19bb7306a80c4318ad942ecd24fdf82c8d1c01ad996e7d3f6da0a2
+  data.tar.gz: 25a2f782ad9f0174ea5b9a4b40d3fc4d0acc4ce3504cea1a33aa109bf831d0e4dcba13e6c11bfb22aac009d4b46f2e06d1d2da76b99d0779b3ed5e2602b5fe55

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+vendor/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in fluent-plugin-docker-tag-resolver.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,14 @@
+Copyright (c) 2015- Naoki AINOYA
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

data/README.md

@@ -0,0 +1,80 @@
+# fluent-plugin-docker-tag-resolver
+
+## Overview
+
+This plugin finds the docker container name and its image name from container-id in record tag,
+and rewrite container-id to human-readable tag name.
+
+It's focused on collecting docker container logs from `/var/lib/docker/containrs/*/*-json.log`.
+
+For example, the plugin rewrites the tags like below:
+
+Before:
+
+```
+"docker.log.var.lib.docker.containers.695e035397f1d5c6cd88225dab54afaed170b93c3ebf51e4354c4daf796e6017.695e035397f1d5c6cd88225dab54afaed170b93c3ebf51e4354c4daf796e6017-json.log"
+```
+
+After:
+
+```
+#The tags represented with docker.container.#{image_name}.#{container_name}.#{container_id}
+
+docker.container.kubernetes/fluentd-gcp:1.0.backstabbing_yonath.695e035397f1d5c6cd88225dab54afaed170b93c3ebf51e4354c4daf796e6017
+```
+
+
+## Installation
+
+Install with fluent-gem command as:
+
+```
+# for google-fluentd
+$ /opt/google-fluentd/embedded/lib/ruby/gems/2.1.0/gems/fluent-plugin-docker-tag-resolver-0.1.0# /opt/google-fluentd/embedded/bin/gem install 
+```
+
+## Configuration
+
+### Usage
+
+
+```
+<source>
+  type tail
+  format none
+  time_key time
+  path /var/lib/docker/containers/*/*-json.log
+  pos_file /var/lib/docker/containers/containers.log.pos
+  time_format %Y-%m-%dT%H:%M:%S
+  tag docker.log.*
+</source>
+
+<match docker.log.**>
+  type docker_tag_resolver
+</match>
+
+<match docker.container.**>
+  type stdout
+</match>
+
+```
+
+### Result
+
+```
+2015-03-06 08:09:34 +0000 docker.container.kubernetes/fluentd-gcp:1.0.backstabbing_yonath.695e035397f1d5c6cd88225dab54afaed170b93c3ebf51e4354c4daf796e6017: {"message":"{\"log\":\"\\u001b(B\\u001b[m  349 root      20   0   91208  23124   6888 S   0.0  1.1   0:00.21 google-fluentd                                                                  \\u001b(B\\u001b[m\\u001b[39;49m\\u001b[K\\r\\n\",\"stream\":\"stdout\",\"time\":\"2015-03-06T08:09:34.980863575Z\"}"}
+2015-03-06 08:09:34 +0000 docker.container.kubernetes/fluentd-gcp:1.0.backstabbing_yonath.695e035397f1d5c6cd88225dab54afaed170b93c3ebf51e4354c4daf796e6017: {"message":"{\"log\":\"\\u001b(B\\u001b[m  355 root      20   0    4388    680    604 S   0.0  0.0   0:00.01 tail                                                                            \\u001b(B\\u001b[m\\u001b[39;49m\\u001b[K\\r\\n\",\"stream\":\"stdout\",\"time\":\"2015-03-06T08:09:34.980863575Z\"}"}
+2015-03-06 08:09:34 +0000 docker.container.kubernetes/fluentd-gcp:1.0.backstabbing_yonath.695e035397f1d5c6cd88225dab54afaed170b93c3ebf51e4354c4daf796e6017: {"message":"{\"log\":\"\\u001b(B\\u001b[m\\u001b[1m  356 root      20   0   19872   2516   2176 R   0.0  0.1   0:00.01 top                                                                             \\u001b(B\\u001b[m\\u001b[39;49m\\u001b[K\\r\\n\",\"stream\":\"stdout\",\"time\":\"2015-03-06T08:09:34.980863575Z\"}"}
+2015-03-06 08:09:37 +0000 docker.container.kubernetes/fluentd-gcp:1.0.backstabbing_yonath.695e035397f1d5c6cd88225dab54afaed170b93c3ebf51e4354c4daf796e6017: {"message":"{\"log\":\"\\u001b[J\\u001b[?1l\\u001b\\u003e\\u001b[31;1H\\r\\n\",\"stream\":\"stdout\",\"time\":\"2015-03-06T08:09:37.507569483Z\"}"}
+```
+
+## Reference
+
+[fluent/fluent-plugin-rewrite-tag-filter](https://github.com/fluent/fluent-plugin-rewrite-tag-filter)
+
+[Collecting Docker Log Files with Fluentd and sending to GCP.](https://github.com/GoogleCloudPlatform/kubernetes/tree/master/contrib/logging/fluentd-gcp-image)
+
+## License
+
+Apache License, Version 2.0
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test
+

data/fluent-plugin-docker-tag-resolver.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "fluent-plugin-docker-tag-resolver"
+  s.version     = "0.1.0"
+  s.license     = "Apache 2.0"
+  s.authors     = ["Naoki AINOYA"]
+  s.email       = ["ainonic@gmail.com"]
+  s.homepage    = "https://github.com/ainoya/fluent-plugin-docker-tag-resolver"
+  s.summary     = %q{Fluentd Output filter plugin.}
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "rake"
+  s.add_development_dependency "pry"
+  s.add_runtime_dependency "fluentd"
+  s.add_runtime_dependency "string-scrub"
+  s.add_runtime_dependency "docker-api"
+end

data/lib/fluent/plugin/out_docker_tag_resolver.rb

@@ -0,0 +1,59 @@
+class Fluent::DockerTagResolverOutput < Fluent::Output
+
+  # Define `log` method for v0.10.42 or earlier
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
+
+  Fluent::Plugin.register_output('docker_tag_resolver', self)
+
+  def initialize
+    super
+    require  'docker'
+
+    @containers = Docker::Container.all
+
+    @find_containers = Proc.new do |id|
+      container = @containers.select{|c| c.id == id}.first
+
+      if container.nil?
+        @containers = Docker::Container.all 
+        @containers.select{|c| c.id == id}.first
+      else
+        container
+      end
+    end
+  end
+
+  def emit(tag, es, chain)
+    es.each do |time,record|
+      rewrited_tag = rewrite_tag(tag, record)
+      next if rewrited_tag.nil? || tag == rewrited_tag
+      Fluent::Engine.emit(rewrited_tag, time, record)
+    end
+
+    chain.next
+  end
+
+  def rewrite_tag(tag, record)
+    container_id, _ , _ = tag.split('.').last(3)
+
+    container = @find_containers.call(container_id)
+
+    return tag unless container
+
+    image_name = container.info['Image']
+    container_name = container.info['Names'].first
+
+    return tag if image_name.nil? or container_name.nil?
+
+    container_name.sub!(/^\//, '')
+    container_name.tr!('.','_')
+
+    rewrited_tag = "docker.container.%s.%s.%s" % [image_name, container_name, container_id]
+    return rewrited_tag
+  end
+
+
+end
+

data/test/helper.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/out_docker_tag_resolver'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_out_docker_tag_resolver.rb

@@ -0,0 +1,12 @@
+require 'helper'
+
+class DockerTagResolverOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf=CONFIG,tag='test')
+    Fluent::Test::OutputTestDriver.new(Fluent::DockerTagResolver, tag).configure(conf)
+  end
+
+end

metadata

@@ -0,0 +1,125 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-docker-tag-resolver
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Naoki AINOYA
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: string-scrub
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: docker-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- ainonic@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-docker-tag-resolver.gemspec
+- lib/fluent/plugin/out_docker_tag_resolver.rb
+- test/helper.rb
+- test/plugin/test_out_docker_tag_resolver.rb
+homepage: https://github.com/ainoya/fluent-plugin-docker-tag-resolver
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Fluentd Output filter plugin.
+test_files:
+- test/helper.rb
+- test/plugin/test_out_docker_tag_resolver.rb