fluent-plugin-file-sprintf 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b5a4f17b7e16b09210c8e077f49a1a0e3b2b2944
+  data.tar.gz: 8722818ad5766d16dd4cb05fa9446e916fa28d57
+SHA512:
+  metadata.gz: 9873101ad62256ed8adbf48f21bc71de2e897359830e741d51359dc525ff9be5a14d044b0d09c4e9d25468ccd89db531e6ef91a5ae266a0af7012bef53a553c6
+  data.tar.gz: 50155220f312fa1f8de018a7ff43bb579195fb77b10b43c83c564e55264f32060155b865703fc9f30eedaa8c095507c794adf7df0cc1aed92d7c753a38544af7

data/.gitignore

@@ -0,0 +1,6 @@
+Gemfile.lock
+nbproject/
+.project
+*.gem
+pkg
+test/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-incremental.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 toyama0919
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,123 @@
+# fluent-plugin-file-sprintf
+
+sprintf output file plugin for Fluentd.
+
+## Installation
+
+### td-agent(Linux)
+
+    /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-file-sprintf
+
+### td-agent(Mac)
+
+    sudo /usr/local/Cellar/td-agent/1.1.XX/bin/fluent-gem install fluent-plugin-file-sprintf
+
+### fluentd only
+
+    gem install fluent-plugin-file-sprintf
+
+
+## parameter
+
+param    |   value|exsample
+--------|------|------
+compress|file compress gzip(default:true)|true
+buffer_path|buffer file path(require)|/tmp/buffer/test.ltsv.*.log
+path|output file path(require)|/tmp/test.ltsv
+format|sprintf format(require)|%s
+key_names|key names comma separator(require)|ltsv
+time_format|time value output format(default:%Y-%m-%d %H:%M:%S)|%Y-%m-%d %H:%M:%S
+file_size_limit|log rotate file size limit byte(default 8388608)|31457280
+
+## key_names reserved words
+
+param    |   value|
+--------|------|
+time|output time string
+tag|output tag string
+ltsv|output ltsv string
+msgpack|output msgpack string
+json|output json string
+
+
+## Configuration Exsample(json format)
+
+	<match apache.json>
+		type file_sprintf
+		compress true
+		buffer_path /tmp/buffer/apache.json.*.log
+		path /tmp/apache.json
+		format %s
+		key_names json
+		file_size_limit 31457280
+	</match>
+
+
+## Configuration Exsample(labeled tsv format)
+
+	<match apache.ltsv>
+		type file_sprintf
+		compress true
+		buffer_path /tmp/buffer/apache.ltsv.*.log
+		path /tmp/apache.ltsv
+		format %s
+		key_names ltsv
+		file_size_limit 31457280
+	</match>
+
+## Configuration Exsample(message pack format)
+
+	<match apache.ltsv>
+		type file_sprintf
+		compress true
+		buffer_path /tmp/buffer/apache.msgpack.*.log
+		path /tmp/apache.msgpack
+		format %s
+		key_names msgpack
+		file_size_limit 31457280
+	</match>
+
+## Configuration Exsample(custom json format)
+
+	<match apache.myjson>
+		type file_sprintf
+		compress true
+		buffer_path /tmp/buffer/apache.json.*.log
+		path /tmp/apache.json
+		format {"method":"%s","agent":"%s","referer":"%s","path":"%s","host":"%s","time":"%s","tag":"%s"}
+		key_names method,agent,referer,path,host,time,tag
+		file_size_limit 31457280
+	</match>
+
+## Configuration Exsample(tsv format)
+
+	<match apache.tsv>
+		type file_sprintf
+		compress true
+		buffer_path /tmp/buffer/apache.tsv.*.log
+		path /tmp/apache.tsv
+		format %s\t%s\t%s\t%s\t%s\t%s\t%s
+		key_names method,agent,referer,path,host,time,tag
+		file_size_limit 31457280
+	</match>
+
+## Configuration Exsample(elasticsearch bulk import format. multiline format)
+
+	<store>
+		type file_sprintf
+		compress false
+		buffer_path /tmp/buffer/es.json.*.log
+		path /tmp/es.json
+		format { "index" : { "_index" : "test_index", "_type" : "test_type" } }\n%s
+		key_names json
+		file_size_limit 31457280
+	</store>
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,17 @@
+require "bundler"
+require "bundler/gem_tasks"
+Bundler::GemHelper.install_tasks
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/plugin/*.rb']
+  test.verbose = true
+end
+
+task :coverage do |t|
+  ENV['COVERAGE'] = '1'
+  Rake::Task["test"].invoke
+end
+
+task :default => [:build]

data/fluent-plugin-file-sprintf.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-file-sprintf"
+  spec.version       = "0.0.1"
+  spec.authors       = ["Hiroshi Toyama"]
+  spec.email         = ["toyama0919@gmail.com"]
+  spec.description   = %q{sprintf output file plugin for Fluentd.}
+  spec.summary       = spec.description
+  spec.homepage      = "https://github.com/toyama0919/fluent-plugin-file-sprintf/"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "ltsv"
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

data/lib/fluent/plugin/out_file_sprintf.rb

@@ -0,0 +1,82 @@
+# -*- encoding : utf-8 -*-
+module Fluent
+  class FileSprintfOutput < Fluent::TimeSlicedOutput
+    Fluent::Plugin.register_output('file_sprintf', self)
+
+    config_param :path, :string
+    config_param :file_size_limit, :integer ,:default => 8388608
+    config_param :format, :string
+    config_param :compress, :bool, :default => true
+    config_param :key_names, :string
+    config_param :time_format, :string, :default => "%Y-%m-%d %H:%M:%S"
+
+    def initialize
+      super
+      require 'zlib'
+      require 'ltsv'
+    end
+
+    def start
+      super
+    end
+
+    def shutdown
+      super
+    end
+
+    def configure(conf)
+      super
+      @key_names = @key_names.split(',').map{|key|
+        key = key.strip
+        result = ""
+        if key == 'time'
+          result = "Time.at(time).strftime('#{time_format}')"
+        elsif key == 'tag'
+          result = 'tag'
+        elsif key == 'ltsv'
+          result = 'LTSV.dump(record)'
+        elsif key == 'json'
+          result = 'record.to_json'
+        elsif key == 'msgpack'
+          result = 'record.to_msgpack'
+        else
+          result = "record['" + key + "']"
+        end
+        result
+      }
+      @key_names = @key_names.join(',')
+      @eval_string = "%Q{#{@format}} % [#{@key_names}]"
+      $log.info "format => #{@eval_string}"
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def write(chunk)
+      filepath = @path
+      file_size = 0
+      File.open(filepath,'a') { |file|
+        chunk.msgpack_each do |tag, time, record|
+          result = eval(@eval_string)
+          file.puts result
+        end
+        file_size = file.size
+      }
+
+      if file_size > @file_size_limit
+        if @compress
+          Zlib::GzipWriter.open(@path + "." + "#{Fluent::Engine.now}" + ".gz") do |gz|
+            gz.mtime = File.mtime(filepath)
+            gz.orig_name = filepath
+            gz.write IO.binread(filepath)
+          end
+        else
+          FileUtils.cp filepath, @path + "." + "#{Fluent::Engine.now}"
+        end
+        FileUtils.remove_file(filepath, force = true)
+      end
+    end
+
+  end
+end

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-file-sprintf
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Hiroshi Toyama
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ltsv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: sprintf output file plugin for Fluentd.
+email:
+- toyama0919@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-file-sprintf.gemspec
+- lib/fluent/plugin/out_file_sprintf.rb
+homepage: https://github.com/toyama0919/fluent-plugin-file-sprintf/
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.0
+signing_key: 
+specification_version: 4
+summary: sprintf output file plugin for Fluentd.
+test_files: []