fluent-parser-elasticsearch 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
data/.gitignore ADDED
@@ -0,0 +1,21 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ coverage
6
+ InstalledFiles
7
+ lib/bundler/man
8
+ pkg
9
+ rdoc
10
+ spec/reports
11
+ test/tmp
12
+ test/version_tmp
13
+ tmp
14
+ *.swp
15
+ vendor/
16
+ Gemfile.lock
17
+
18
+ # YARD artifacts
19
+ .yardoc
20
+ _yardoc
21
+ doc/
data/.travis.yml ADDED
@@ -0,0 +1,7 @@
1
+ language: ruby
2
+
3
+ rvm:
4
+ - 2.1.1
5
+ - 2.1.0
6
+ - 2.0.0
7
+ - 1.9.3
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-parser-elasticsearch.gemspec
4
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,14 @@
1
+ Copyright (c) 2014- Kentaro Yoshida
2
+
3
+ Licensed under the Apache License, Version 2.0 (the "License");
4
+ you may not use this file except in compliance with the License.
5
+ You may obtain a copy of the License at
6
+
7
+ http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ Unless required by applicable law or agreed to in writing, software
10
+ distributed under the License is distributed on an "AS IS" BASIS,
11
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ See the License for the specific language governing permissions and
13
+ limitations under the License.
14
+
data/README.md ADDED
@@ -0,0 +1,53 @@
1
+ # Fluentd Parser for elasticsearch
2
+
3
+ [Fluentd](http://fluentd.org/) Parser Plugin to add choice for `format` option for elasticsearch logs with `in_tail` plugin.
4
+
5
+ ## Installation
6
+
7
+ install with `gem` or `fluent-gem` command as:
8
+
9
+ ```bash
10
+ # for fluentd
11
+ $ gem install fluent-parser-elasticsearch
12
+
13
+ # for td-agent
14
+ $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-parser-elasticsearch
15
+ ```
16
+
17
+ ## Usage
18
+
19
+ After installing this plugin, it has got ready to use this 3rd party format like below.
20
+
21
+ ```xml
22
+ <source>
23
+ type tail
24
+ tag elasticsearch.general_log
25
+ format elasticsearch
26
+ path /var/log/elasticsearch/elasticsearch.log
27
+ pos_file /var/log/td-agent/elasticsearch.log.pos
28
+ </source>
29
+
30
+ <match elasticsearch.*>
31
+ type stdout
32
+ </match>
33
+ ```
34
+
35
+ ## TODO
36
+
37
+ Pull requests are very welcome!!
38
+
39
+ ## Contributing
40
+
41
+ 1. Fork it
42
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
43
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
44
+ 4. Push to the branch (`git push origin my-new-feature`)
45
+ 5. Create new Pull Request
46
+
47
+ ## Copyright
48
+
49
+ Copyright (c) 2014- Kentaro Yoshida (@yoshi_ken)
50
+
51
+ ## License
52
+
53
+ Apache License, Version 2.0
data/Rakefile ADDED
@@ -0,0 +1,9 @@
1
+ require "bundler/gem_tasks"
2
+ require "rake/testtask"
3
+ Rake::TestTask.new(:test) do |test|
4
+ test.libs << 'lib' << 'test'
5
+ test.pattern = 'test/**/test_*.rb'
6
+ test.verbose = true
7
+ end
8
+
9
+ task :default => :test
@@ -0,0 +1,22 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = "fluent-parser-elasticsearch"
7
+ spec.version = "0.0.1"
8
+ spec.authors = ["Kentaro Yoshida"]
9
+ spec.email = ["y.ken.studio@gmail.com"]
10
+ spec.summary = %q{Fluentd Parser Plugin to add choice for `format` option for elasticsearch logs with in_tail plugin.}
11
+ spec.homepage = "https://github.com/y-ken/fluent-parser-elasticsearch"
12
+ spec.license = "Apache License, Version 2.0"
13
+
14
+ spec.files = `git ls-files`.split($/)
15
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
16
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
17
+ spec.require_paths = ["lib"]
18
+
19
+ spec.add_development_dependency "bundler"
20
+ spec.add_development_dependency "rake"
21
+ spec.add_runtime_dependency("fluentd", [">= 0.10.46"])
22
+ end
@@ -0,0 +1,13 @@
1
+ module Fluent
2
+ class TextParser
3
+
4
+ parser_elasticsearch = Proc.new {
5
+ RegexpParser.new(
6
+ /^\[(?<time>[^ ]* [^ ]*)\]\[(?<log_level>[^ ]*) *?\]\[(?<log_type>[^ ]*) *\] \[(?<node_name>[^ ]*) *\] (?<message>.+)/,
7
+ {'time_format' => "%Y-%m-%d %H:%M:%S,%L"}
8
+ )
9
+ }
10
+ TextParser.register_template('elasticsearch', parser_elasticsearch)
11
+
12
+ end
13
+ end
data/test/helper.rb ADDED
@@ -0,0 +1,28 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+ begin
4
+ Bundler.setup(:default, :development)
5
+ rescue Bundler::BundlerError => e
6
+ $stderr.puts e.message
7
+ $stderr.puts "Run `bundle install` to install missing gems"
8
+ exit e.status_code
9
+ end
10
+ require 'test/unit'
11
+
12
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
13
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
14
+ require 'fluent/test'
15
+ unless ENV.has_key?('VERBOSE')
16
+ nulllogger = Object.new
17
+ nulllogger.instance_eval {|obj|
18
+ def method_missing(method, *args)
19
+ # pass
20
+ end
21
+ }
22
+ $log = nulllogger
23
+ end
24
+
25
+ require 'fluent/plugin/parser_elasticsearch'
26
+
27
+ class Test::Unit::TestCase
28
+ end
@@ -0,0 +1,35 @@
1
+ require 'helper'
2
+
3
+ module ParserTest
4
+ include Fluent
5
+
6
+ def str2time(str_time, format = nil)
7
+ if format
8
+ Time.strptime(str_time, format).to_i
9
+ else
10
+ Time.parse(str_time).to_i
11
+ end
12
+ end
13
+ end
14
+
15
+ class ElasticsearchParserTest < Test::Unit::TestCase
16
+ include ParserTest
17
+
18
+ def setup
19
+ TextParser.new
20
+ @parser = TextParser::TEMPLATE_REGISTRY.lookup('elasticsearch').call
21
+ end
22
+
23
+ def test_call
24
+ time, record = @parser.call('[2014-03-18 18:27:34,897][INFO ][http ] [es01] bound_address {inet[/0:0:0:0:0:0:0:0:9200]}, publish_address {inet[/10.0.0.185:9200]}')
25
+
26
+ assert_equal(str2time('2014-03-18 18:27:34,897', '%Y-%m-%d %H:%M:%S,%L'), time)
27
+ assert_equal({
28
+ 'log_level' => 'INFO',
29
+ 'log_type' => 'http',
30
+ 'node_name' => 'es01',
31
+ 'message' => 'bound_address {inet[/0:0:0:0:0:0:0:0:9200]}, publish_address {inet[/10.0.0.185:9200]}'
32
+ }, record)
33
+ end
34
+ end
35
+
metadata ADDED
@@ -0,0 +1,107 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-parser-elasticsearch
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Kentaro Yoshida
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2014-04-22 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: bundler
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ! '>='
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
22
+ type: :development
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ! '>='
28
+ - !ruby/object:Gem::Version
29
+ version: '0'
30
+ - !ruby/object:Gem::Dependency
31
+ name: rake
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ! '>='
36
+ - !ruby/object:Gem::Version
37
+ version: '0'
38
+ type: :development
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ! '>='
44
+ - !ruby/object:Gem::Version
45
+ version: '0'
46
+ - !ruby/object:Gem::Dependency
47
+ name: fluentd
48
+ requirement: !ruby/object:Gem::Requirement
49
+ none: false
50
+ requirements:
51
+ - - ! '>='
52
+ - !ruby/object:Gem::Version
53
+ version: 0.10.46
54
+ type: :runtime
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
58
+ requirements:
59
+ - - ! '>='
60
+ - !ruby/object:Gem::Version
61
+ version: 0.10.46
62
+ description:
63
+ email:
64
+ - y.ken.studio@gmail.com
65
+ executables: []
66
+ extensions: []
67
+ extra_rdoc_files: []
68
+ files:
69
+ - .gitignore
70
+ - .travis.yml
71
+ - Gemfile
72
+ - LICENSE
73
+ - README.md
74
+ - Rakefile
75
+ - fluent-parser-elasticsearch.gemspec
76
+ - lib/fluent/plugin/parser_elasticsearch.rb
77
+ - test/helper.rb
78
+ - test/plugin/test_parser_elasticsearch.rb
79
+ homepage: https://github.com/y-ken/fluent-parser-elasticsearch
80
+ licenses:
81
+ - Apache License, Version 2.0
82
+ post_install_message:
83
+ rdoc_options: []
84
+ require_paths:
85
+ - lib
86
+ required_ruby_version: !ruby/object:Gem::Requirement
87
+ none: false
88
+ requirements:
89
+ - - ! '>='
90
+ - !ruby/object:Gem::Version
91
+ version: '0'
92
+ required_rubygems_version: !ruby/object:Gem::Requirement
93
+ none: false
94
+ requirements:
95
+ - - ! '>='
96
+ - !ruby/object:Gem::Version
97
+ version: '0'
98
+ requirements: []
99
+ rubyforge_project:
100
+ rubygems_version: 1.8.23
101
+ signing_key:
102
+ specification_version: 3
103
+ summary: Fluentd Parser Plugin to add choice for `format` option for elasticsearch
104
+ logs with in_tail plugin.
105
+ test_files:
106
+ - test/helper.rb
107
+ - test/plugin/test_parser_elasticsearch.rb