fluent-plugin-esslowquery 1.0.2 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9eddfff2ba328373e82890f795079dd0bdb1e153
-  data.tar.gz: e932413c6b989d99ba0c4a572b5ab59809968831
+  metadata.gz: c95c49ad0eda635329ee955882bf5a61d0d78616
+  data.tar.gz: e4ecd7cc218367315341925e41e25d12d37121e0
 SHA512:
-  metadata.gz: caef9092c3f652a94ecb6745270af1166c69aaae9e6f8cf9ef611783e08db6bd7d91191fcad5c12638ab2c12f2f5f29d80720547c71fda4b8d48319095ae00d8
-  data.tar.gz: 197cd681f216fb607aff63a32cda452874165850615c896f5b1aa947f03c059615cd12875e195ae763a621d29dc0017b428956c2fe70c7563e4b0f1d0cdfc3a9
+  metadata.gz: 26608ff0fef20868f43d039020df3d4101209832b0a5b9b7efa2b6e2d460458388c6d2bdc364e190e32ff7808bc668c772400dc4238c02c9d13a91ee1795c935
+  data.tar.gz: 9d922d61f8a44896debd55050b4d77f5541e81bfc8e5bde146300b626beca6d306899d1513f3fd9c09a4f5b0610d00c5b2612e7737ae9eb2860af47eab3483dc

data/AUTHORS

@@ -0,0 +1,6 @@
+# This file lists all individuals having contributed content to the repository.
+# For how it is generated, see `script/generate-authors.sh`.
+
+iaintshine <bodziomista@gmail.com>
+Joe Doss <jdoss@kennasecurity.com>
+Mirosław Nagaś <mgolden@smoczus.pl>

data/CHANGELOG.md

@@ -0,0 +1,7 @@
+## CHANGELOG
+
+### V1.1.0 (2016-02-16)
+
+**Features and Improvements**
+
+* Slow indexing parser [#2](https://github.com/iaintshine/fluent-plugin-esslowquery/pull/2)

data/README.md

@@ -26,12 +26,13 @@ Edit `/etc/td-agent/td-agent.conf` file.
   path /var/log/elasticsearch/elasticsearch-{cluster-name}_index_indexing_slowlog.log
   tag elasticsearch.{cluster-name}.indexing_slowlog_query
   pos_file /var/run/td-agent/elasticsearch-indexing-slow.pos
-  format es_slow_query
+  format es_slow_indexing
 </source>
 ```
 
 ##Expected record format
 
+### Slow Query
 ```json
 {
     "extra_source": "{\"from\":0,\"size\":0}",
@@ -49,3 +50,20 @@ Edit `/etc/td-agent/td-agent.conf` file.
     "types": "document"
 }
 ```
+### Slow Indexing
+
+```json
+{
+    "severity": "INFO ",
+    "source": "index.indexing.slowlog.index",
+    "node": "{cluster-name}-{node-id}",
+    "index": "comments",
+    "shard": 4,
+    "took": "891.4ms",
+    "took_millis": 891,
+    "type": "document",
+    "indexing_id": 120543866,
+    "routing": 2012927,
+    "source_body": "{}"
+}
+```

data/fluent-plugin-esslowquery.gemspec

@@ -1,8 +1,8 @@
 Gem::Specification.new do |gem|
   gem.authors       = ["Boguslaw Mista"]
   gem.email         = ["bodziomista@gmail.com"]
-  gem.description   = "Fluent parser plugin for Elasticsearch slow query log file."
-  gem.summary       = "Fluent parser plugin for Elasticsearch slow query log file."
+  gem.description   = "Fluent parser plugin for Elasticsearch slow query and slow indexing log files."
+  gem.summary       = "Fluent parser plugin for Elasticsearch slow query and slow indexing log files."
   gem.homepage      = "https://github.com/iaintshine/fluent-plugin-esslowquery"
   gem.license       = "MIT"
 
@@ -11,6 +11,6 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-esslowquery"
   gem.require_paths = ["lib"]
-  gem.version       = "1.0.2"
+  gem.version       = "1.1.0"
   gem.add_dependency "fluentd", [">= 0.12.0", "< 2"]
 end

data/lib/fluent/plugin/parser_es_slow_indexing.rb

@@ -0,0 +1,61 @@
+module Fluent
+  class TextParser
+    class ElasticsearchSlowIndexingLogParser < Parser
+      REGEXP = /^\[(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\]\[(?<severity>[a-zA-Z]+\s*)\]\[(?<source>\S+)\] \[(?<node>\S+)\] \[(?<index>\w+)\]\[(?<shard>\d+)\] took\[(?<took>.+)\], took_millis\[(?<took_millis>\d+)\], type\[(?<type>.+)\], id\[(?<indexing_id>.*)\], routing\[(?<routing>.*)\], source\[(?<source_body>.*)\]/
+      TIME_FORMAT = "%Y-%m-%d %H:%M:%S,%N"
+
+      Plugin.register_parser("es_slow_indexing", self)
+
+      def initialize
+        super
+        @time_parser = TextParser::TimeParser.new(TIME_FORMAT)
+        @mutex = Mutex.new
+      end
+
+      def patterns
+        {'format' => REGEXP, 'time_format' => TIME_FORMAT}
+      end
+
+      def parse(text)
+        m = REGEXP.match(text)
+        unless m
+          if block_given?
+            yield nil, nil
+            return
+          else
+            return nil, nil
+          end
+        end
+
+        shard = m['shard'].to_i
+        took_millis = m['took_millis'].to_i
+        indexing_id= m['indexing_id'].to_i
+        routing = m['routing'].to_i
+
+        time = m['time']
+        time = @mutex.synchronize { @time_parser.parse(time) }
+
+        record = {
+          'severity' => m['severity'],
+          'source' => m['source'],
+          'node' => m['node'],
+          'index' => m['index'],
+          'shard' => shard,
+          'took' => m['took'],
+          'took_millis' => took_millis,
+          'type' => m['type'],
+          'indexing_id' => indexing_id,
+          'routing' => routing,
+          'source_body' => m['source_body']
+        }
+        record["time"] = m['time'] if @keep_time_key
+
+        if block_given?
+          yield time, record
+        else
+          return time, record
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_es_slow_query.rb

@@ -1,5 +1,5 @@
 module Fluent
-  class ElasticsearchSlowLogParser < Parser
+  class ElasticsearchSlowQueryLogParser < Parser
     REGEXP = /^\[(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\]\[(?<severity>[a-zA-Z]+\s*)\]\[(?<source>\S+)\] \[(?<node>\S+)\] \[(?<index>\w+)\]\[(?<shard>\d+)\] took\[(?<took>.+)\], took_millis\[(?<took_millis>\d+)\], types\[(?<types>.*)\], stats\[(?<stats>.*)\], search_type\[(?<search_type>.*)\], total_shards\[(?<total_shards>\d+)\], source\[(?<source_body>.*)\], extra_source\[(?<extra_source>.*)\]/
     TIME_FORMAT = "%Y-%m-%d %H:%M:%S,%N"
 

data/scripts/generate-authors.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+{
+	cat <<-'EOH'
+	# This file lists all individuals having contributed content to the repository.
+	# For how it is generated, see `script/generate-authors.sh`.
+	EOH
+	echo
+	git log --format='%aN <%aE>' | LC_ALL=C.UTF-8 sort -uf
+} > AUTHORS

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-esslowquery
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.1.0
 platform: ruby
 authors:
 - Boguslaw Mista
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-10 00:00:00.000000000 Z
+date: 2016-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -30,7 +30,8 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
-description: Fluent parser plugin for Elasticsearch slow query log file.
+description: Fluent parser plugin for Elasticsearch slow query and slow indexing log
+  files.
 email:
 - bodziomista@gmail.com
 executables: []
@@ -38,11 +39,15 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- AUTHORS
+- CHANGELOG.md
 - Gemfile
 - README.md
 - Rakefile
 - fluent-plugin-esslowquery.gemspec
+- lib/fluent/plugin/parser_es_slow_indexing.rb
 - lib/fluent/plugin/parser_es_slow_query.rb
+- scripts/generate-authors.sh
 homepage: https://github.com/iaintshine/fluent-plugin-esslowquery
 licenses:
 - MIT
@@ -66,5 +71,5 @@ rubyforge_project:
 rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
-summary: Fluent parser plugin for Elasticsearch slow query log file.
+summary: Fluent parser plugin for Elasticsearch slow query and slow indexing log files.
 test_files: []