fluent-plugin-mongo-slow-query 0.0.5 → 0.1.0

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: 284e26c6dc167bed1678ea14f4efb320be3148e5
-  data.tar.gz: a50274330aa6b8fe11eb052d4fa2414cf095ba1d
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    YzJhM2VkMGUzMTA2ZTczMTg1ZjU1ZDVmNjdjYTI2MjUwOWI5OTBiOA==
+  data.tar.gz: !binary |-
+    ZjQxMDYwOWYyMzYzNWRiYjZhZjc4MjZlOTQyNTU3YWQ1YWVjNjlmZg==
 SHA512:
-  metadata.gz: fdc3f9ef96cc3e558831772744e8709d10a6e1b42df55435a2a2bf732c67f926acefbdda555372b969c3d9634508fc688ad144d8059912a142fa8a98e124f865
-  data.tar.gz: da2a2a1938f10227c0a62f2106d378ea69ced0c930a07f4eebf7cc62d77c0fb08abe73c73fa8982430f6e75561d119de834d50df8843e7962b3c5fce39985c28
+  metadata.gz: !binary |-
+    OWQ4M2FmNDFmM2RhZTQ5Y2JmYTRlZWRiMmQ1NDU2OTg3ZDBlZWRjZGI5MGI1
+    ZjViZDlmNjcwZWM0Y2NkNDk1MDAxMGM1NTIxZDEyMjI0MWY4ZTU4ZWRhNzIw
+    YjIxN2Q4NDFjMGE3ZjFiOWZmODkyYTgwOGRjMDQ0ZTM3MDg5OWY=
+  data.tar.gz: !binary |-
+    NDRmMTdiYWRhODE1ZGM4NjZmMDZiOTE0YWQwMmQwMWMwYjNiOTBiZTBkNGMx
+    MTRjZGQ5NzlmNGVhNDU4YzRmNTFkZWRjZjkyMTMxZWJhNjExZmU4MmYwZGYw
+    OGI2YjQyMmNhYzNhMDcxYjdiNzk0ZGUzZWM4NGE4ZWE4ZmUwMzU=

data/lib/fluent/plugin/in_mongo_slow_query.rb

@@ -10,22 +10,18 @@ module Fluent
         # 'conf' is a Hash that includes configuration parameters.
         # If the configuration is invalid, raise Fluent::ConfigError.
         def configure(conf)
-            #unless conf.has_key?("format")
-            #    conf["format"] = '/(?<time>.*) \[\w+\] (?<op>[^ ]+) (?<ns>[^ ]+) (?<detail>((query: (?<query>{.+}) update: {.*})|(query: (?<query>{.+})))) .* (?<ms>\d+)ms/'
-            #    $log.warn "load default format: ", conf["format"]
-            #end
-
-            # load default format that degisned for MongoDB
-            conf["format"] = '/(?<time>.*) \[\w+\] (?<op>[^ ]+) (?<ns>[^ ]+) ((?<detail>(query: (?<query>\{.+\}) update: \{.*\}))|((?<detail>(query: (?<query>\{.+\}))) planSummary: .*)|((?<detail>query: (?<query>\{.+\})))) .* (?<ms>\d+)ms/'
+            # load log format for MongoDB
+            conf["format"] = '/^(?<time>.*?)(?:\s+\w\s\w+\s*)? \[conn\d+\] (?<op>\w+) (?<ns>\S+) (?<detail>(?:(?:query: (?<query>\{.+\}) update: \{.*\})|(?:query: (?<query>\{.*\}) planSummary: \w+(?: \{.*\})?)|(?:query: (?<query>\{.*\}))|(?:command: \{.*\}))) (?<stat>.*) (?<ms>\d+)ms$/'
 
             # not set "time_format"
             # default use Ruby's DateTime.parse() to pase time
             #
-            # be compatible for v2.2, 2.4 and 2.6
+            # be compatible for v2.2, 2.4, 2.6 and 3.0
             # difference of time format
-            # 2.2: Wed Sep 17 10:00:00 [conn] ...
-            # 2.4: Wed Sep 17 10:00:00.123 [conn] ...
-            # 2.6: 2014-09-17T10:00:43.506+0800  [conn] ...
+            # 2.2: Wed Sep 17 10:00:00 [connXXX] ...
+            # 2.4: Wed Sep 17 10:00:00.123 [connXXX] ...
+            # 2.6: 2014-09-17T10:00:43.506+0800 [connXXX] ...
+            # 3.0: 2015-03-18T15:28:44.321+0800 I QUERY    [conn5462] ...
             #unless conf.has_key?("time_format")
             #    #conf["time_format"] = '%a %b %d %H:%M:%S'
             #    #conf["time_format"] = '%a %b %d %H:%M:%S.%L'
@@ -41,12 +37,58 @@ module Fluent
                     line.chomp!  # remove \n
                     time, record = parse_line(line)
                     if time && record
-                        record["query"] = get_query_prototype(record["query"])
+                        record["query"] = get_query_prototype(record["query"]) if record["query"]
                         record["ms"] = record["ms"].to_i
                         record["ts"] = time
+
+                        case record["op"]
+                        when "query"
+                        when "getmore"
+                            res = /ntoskip:(?<ntoskip>\d+)/.match(record["stat"])
+                            if res
+                                record["ntoskip"] = res["ntoskip"].to_i
+                            end
+                            res = /nscanned:(?<nscanned>\d+)/.match(record["stat"])
+                            if res
+                                record["nscanned"] = res["nscanned"].to_i
+                            end
+                            res = /nreturned:(?<nreturned>\d+)/.match(record["stat"])
+                            if res
+                                record["nreturned"] = res["nreturned"].to_i
+                            end
+                            res = /reslen:(?<reslen>\d+)/.match(record["stat"])
+                            if res
+                                record["reslen"] = res["reslen"].to_i
+                            end
+                        when "update"
+                            res = /nscanned:(?<nscanned>\d+)/.match(record["stat"])
+                            if res
+                                record["nscanned"] = res["nscanned"].to_i
+                            end
+                            res = /nMatched:(?<nmatched>\d+)/.match(record["stat"])
+                            if res # MongoDB v3.0
+                                record["nmatched"] = res["nmatched"].to_i
+                                res = /nModified:(?<nmodified>\d+)/.match(record["stat"])
+                                if res
+                                    record["nmodified"] = res["nmodified"].to_i
+                                end
+                            else # MongoDB v2.4
+                                res = /nupdated:(?<nmodified>\d+)/.match(record["stat"])
+                                if res
+                                    record["nmodified"] = res["nmodified"].to_i
+                                end
+                            end
+                        when "remove"
+                            res = /ndeleted:(?<ndeleted>\d+)/.match(record["stat"])
+                            if res
+                                record["ndeleted"] = res["ndeleted"].to_i
+                            end
+                        end
+
                         #if record.has_key?("update")
                         #    record["update"] = get_query_prototype(record["update"])
                         #end
+
                         es.add(time, record)
                     end
                 rescue
@@ -73,10 +115,11 @@ module Fluent
                     ns_array += extract_query_prototype(val, ns)
                 elsif val.class == Array
                     val.each do |item|
-                        if item.class == Hash
+                        if item.class == Hash # e.g. $eq $gt $gte $lt $lte $ne $in $nin 
                             ns_array += extract_query_prototype(item, ns)
-                        else
-                            ns_array << ns + '.' + item
+                        else # e.g. $and $or $nor
+                            ns_array << ns 
+                            break
                         end
                     end
                 else
@@ -92,7 +135,7 @@ module Fluent
                 prototype = extract_query_prototype(JSON.parse(to_json(query)))
                 return '{ ' + prototype.join(', ') + ' }'
             rescue
-                $log.error $!.to_s
+                $log.warn $!.to_s
                 return query
             end
         end

metadata

@@ -1,16 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo-slow-query
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.1.0
 platform: ruby
 authors:
-- caosiyang
+- Caosiyang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-18 00:00:00.000000000 Z
+date: 2015-03-27 00:00:00.000000000 Z
 dependencies: []
-description: fluent plugin for mongo slow query
+description: Fluent plugin for MongoDB slow operation. It's helpful to statistics
+  the slow queries and build indexes
 email: csy3228@gmail.com
 executables: []
 extensions: []
@@ -27,18 +28,19 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.3.0
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
-summary: fluent plugin for mongo slow query
+summary: Fluent plugin for MongoDB slow operation. It's helpful to statistics the
+  slow queries and build indexes
 test_files: []