fluent-plugin-esslowquery 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2d4433da6072fd01f6207e65b6e071efd797b9d8
+  data.tar.gz: 33aa3e328fb5a598f9020d06228d41f1ae7ff024
+SHA512:
+  metadata.gz: eb0ee2d40e16a2a588d2a53c083136da510630d0d11c4f756b28541f10ff6b65fb2024a5d68c69a315764ffd4a6694075be5389effa0ce122eaecbc523ac9dc1
+  data.tar.gz: 637e812843e305ca5e452cf43995d9e7d22af98fde4f98e514874283d526a27591aa6c2f40ea9a760b7780c3122ee0cc637c6b19fb4745a895f1b5895b8f1843

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,51 @@
+#fluent-plugin-esslowquery
+
+Fluent parser plugin for Elasticsearch slow query log file.
+
+##Installation
+
+```shell
+$ gem install fluent-plugin-esslowquery
+```
+
+##How to use
+
+Edit `/etc/td-agent/td-agent.conf` file.
+
+```conf
+<source>
+  type tail
+  path /var/log/elasticsearch/elasticsearch-{cluster-name}_index_search_slowlog.log
+  tag elasticsearch.{cluster-name}.search_slowlog_query
+  pos_file /var/run/td-agent/elasticsearch-search-slow.pos
+  format es_slow_query
+</source>
+
+<source>
+  type tail
+  path /var/log/elasticsearch/elasticsearch-{cluster-name}_index_indexing_slowlog.log
+  tag elasticsearch.{cluster-name}.indexing_slowlog_query
+  pos_file /var/run/td-agent/elasticsearch-indexing-slow.pos
+  format es_slow_query
+</source>
+```
+
+##Expected record format
+
+```json
+{
+    "extra_source": "{\"from\":0,\"size\":0}",
+    "index": "comments",
+    "node": "{cluster-name}-{node-id}",
+    "search_type": "COUNT",
+    "severity": "TRACE",
+    "shard": 4,
+    "source": "index.search.slowlog.query",
+    "source_body": "{\"query\":{\"filtered\":{\"query\":{\"match_all\":{}},\"filter\":{\"term\":{\"tags\":\"elasticsearch\"}}}}}",
+    "stats": "",
+    "took": "282.7ms",
+    "took_millis": 282,
+    "total_shards": 1,
+    "types": "document"
+}
+```

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/fluent-plugin-esslowquery.gemspec

@@ -0,0 +1,16 @@
+Gem::Specification.new do |gem|
+  gem.authors       = ["Boguslaw Mista"]
+  gem.email         = ["bodziomista@gmail.com"]
+  gem.description   = "Fluent parser plugin for Elasticsearch slow query log file."
+  gem.summary       = "Fluent parser plugin for Elasticsearch slow query log file."
+  gem.homepage      = "https://github.com/iaintshine/fluent-plugin-esslowquery"
+  gem.license       = "MIT"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "fluent-plugin-esslowquery"
+  gem.require_paths = ["lib"]
+  gem.version       = "1.0.0"
+  gem.add_dependency "fluentd", [">= 0.12.0", "< 2"]
+end

data/lib/fluent/plugin/parser_es_slow_query.rb

@@ -0,0 +1,60 @@
+module Fluent
+  class ElasticsearchSlowLogParser < Parser
+    REGEXP = /^\[(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\]\[(?<severity>[a-zA-Z]+)\]\[(?<source>\S+)\] \[(?<node>\S+)\] \[(?<index>\w+)\]\[(?<shard>\d+)\] took\[(?<took>.+)\], took_millis\[(?<took_millis>\d+)\], types\[(?<types>.+)\], stats\[(?<stats>.*)\], search_type\[(?<search_type>.*)\], total_shards\[(?<total_shards>\d+)\], source\[(?<source_body>.*)\], extra_source\[(?<extra_source>.*)\]/
+    TIME_FORMAT = "%Y-%m-%d %H:%M:%S,%N"
+
+    Plugin.register_parser("es_slow_query", self)
+
+    def initialize
+      super
+      @time_parser = TextParser::TimeParser.new(TIME_FORMAT)
+      @mutex = Mutex.new
+    end
+
+    def patterns
+      {'format' => REGEXP, 'time_format' => TIME_FORMAT}
+    end
+
+    def parse(text)
+      m = REGEXP.match(text)
+      unless m
+        if block_given?
+          yield nil, nil
+          return
+        else
+          return nil, nil
+        end
+      end
+
+      shard = m['shard'].to_i
+      took_millis = m['took_millis'].to_i
+      total_shards = m['total_shards'].to_i
+
+      time = m['time']
+      time = @mutex.synchronize { @time_parser.parse(time) }
+
+      record = {
+        'severity' => m['severity'],
+        'source' => m['source'],
+        'node' => m['node'],
+        'index' => m['index'],
+        'shard' => shard,
+        'took' => m['took'],
+        'took_millis' => took_millis,
+        'types' => m['types'],
+        'stats' => m['stats'],
+        'search_type' => m['search_type'],
+        'total_shards' => total_shards,
+        'source_body' => m['source_body'],
+        'extra_source' => m['extra_source']
+      }
+      record["time"] = m['time'] if @keep_time_key
+
+      if block_given?
+        yield time, record
+      else
+        return time, record
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-esslowquery
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Boguslaw Mista
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-09-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+description: Fluent parser plugin for Elasticsearch slow query log file.
+email:
+- bodziomista@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- fluent-plugin-esslowquery.gemspec
+- lib/fluent/plugin/parser_es_slow_query.rb
+homepage: https://github.com/iaintshine/fluent-plugin-esslowquery
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.3
+signing_key: 
+specification_version: 4
+summary: Fluent parser plugin for Elasticsearch slow query log file.
+test_files: []
+has_rdoc: