fluent-plugin-groonga-query-log 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: acf3ada166906b4c1583d559239d1af2e4917fb4
-  data.tar.gz: bb12b02153446679022f26e28ad299ca50c6b4fc
+SHA256:
+  metadata.gz: 74fcfe61f76412fb02a18b924ecf9538f6792d920caa88fda7b22f7bd1d7216c
+  data.tar.gz: b81d0f452cb132cd280b79ec67b9bbea86a5531a5b47cb584dda49423ca3cdc7
 SHA512:
-  metadata.gz: d7292a34808e12ba99aa0f381fe42d77e570819f016452c738611d9f7b203d29ed951181c6a423ed561aefb956b4e3852d0a2e92b36fcd0222d0dbc87c41d5ab
-  data.tar.gz: 69d4bb292bf90fc2c8ea641aa02915bdc03b47dacbc34ba6738b03c53b11565e9380d868f86442c1e8255c5d2202429fd128be84e55fa79effce7b67568f9f90
+  metadata.gz: 5ecc7257af50cd3868ab7f3327bf9e9ad838357aa6bef2c1d3d13c7fbbe3cededa54dde3d55fdb181a5a11473b3b0adde010ac643ae73700913fc26a53c5abf1
+  data.tar.gz: d5054e6b5967f2f113f2bdb582c3759cfc5a099d9ff00adef9ce9bd8b3174ed6bc1c91262a409c7b079e5a8832441389cb1703acfb7068595caa7d98d35ba8b2

data/README.md

@@ -6,20 +6,311 @@ fluent-plugin-groonga-query-log
 
 ## Description
 
-Fluent-plugin-groonga-query-log is a Fluentd plugin to parse
+Fluent-plugin-groonga-query-log is a Fluentd filter plugin to parse
 [Groonga](http://groonga.org/)'s
 [query log](http://groonga.org/docs/reference/log.html#query-log) with
 Fluentd.
 
 You can detect slow query in real time by using this plugin.
 
+Here is a sample configuration that stores slow queries to Groonga:
+
+    <source>
+      @type tail
+      path /var/log/groonga/query.log
+      pos_file /var/log/fluentd/groonga-query-log.pos
+      tag groonga.query
+      format none
+    </source>
+
+    <filter groonga.query>
+      @type groonga_query_log
+    </filter>
+
+    <filter groonga.query>
+      @type grep
+      regexp1 slow \Atrue\z
+    </filter>
+
+    <filter groonga.query>
+      @type record_transformer
+      enable_ruby true
+      renew_record true
+      keep_keys elapsed
+      <record>
+        log ${JSON.generate(to_h)}
+      </record>
+    </filter>
+
+    <match groonga.query>
+      @type groonga
+      store_table SlowQueries
+
+      protocol http
+      host 127.0.0.1
+
+      buffer_type file
+      buffer_path /var/lib/fluentd/groonga.buffer
+      flush_interval 1
+    </match>
+
+You need to prepare your environment to use the configuration.
+
+Create the following directories:
+
+    % sudo mkdir -p /var/log/fluentd
+    % sudo mkdir -p /var/lib/fluentd
+
+User who runs Fluentd must have write permission of the
+directories. Set suitable permission to the directories:
+
+    % sudo chown -R fluentd-user:fluentd-user /var/log/fluentd
+    % sudo chown -R fluentd-user:fluentd-user /var/lib/fluentd
+
+Run Groonga that stores slow queries on `127.0.0.1`:
+
+    % groonga --protocol http -s DB_PATH
+
+Run `fluentd` with the configuration:
+
+    % fluentd --config groonga-slow-queries.conf
+
+Now, slow queries are stored `SlowQueries` table in Groonga:
+
+    % curl 'localhost:10041/d/select?table=SlowQueries&output_pretty=yes'
+    [
+      [
+        0,
+        1453454123.58033,
+        8.70227813720703e-05
+      ],
+      [
+        [
+          [
+            66
+          ],
+          [
+            [
+              "_id",
+              "UInt32"
+            ],
+            [
+              "elapsed",
+              "Float"
+            ],
+            [
+              "log",
+              "Text"
+            ]
+          ],
+          [
+            1,
+            0.265,
+            "{\"start_time\":...}"
+          ],
+          [
+            2,
+            0.303,
+            "{\"start_time\":...}"
+          ],
+          ...
+        ]
+      ]
+    ]
+
+Each query log is stored as one record. Record has the following two
+columns:
+
+  * `elapsed`: The elapsed time to execute the query.
+
+  * `log`: The query details as JSON. It includes executed command,
+    elapsed time for each condition and so on.
+
 ## Install
 
     % gem install fluent-plugin-groonga-query-log
 
 ## Usage
 
-TODO
+You can use `groonga-query-log` filter for parsing raw Groonga's query
+log text.
+
+Here is a sample raw Groonga's query log text:
+
+    2015-08-12 15:50:40.130990|0x7fb07d113da0|>/d/select?table=Entries&match_columns=name&query=xml
+    2015-08-12 15:50:40.296165|0x7fb07d113da0|:000000165177838 filter(10)
+    2015-08-12 15:50:40.296172|0x7fb07d113da0|:000000165184723 select(10)
+    2015-08-12 15:50:41.228129|0x7fb07d113da0|:000001097153433 output(10)
+    2015-08-12 15:50:41.228317|0x7fb07d113da0|<000001097334986 rc=0
+
+`groonga-query-log` filter emits the following record by parsing the
+above raw Groonga's query log text:
+
+    {
+      "start_time": "2015-08-12T06:50:40.130990Z",
+      "last_time": "2015-08-12T06:50:41.228324Z",
+      "elapsed": 1.0973349860000001,
+      "return_code": 0,
+      "slow": true,
+      "command": {
+        "raw": "/d/select?table=Entries&match_columns=name&query=xml",
+        "name": "select",
+        "parameters": [
+          {
+            "key": "table",
+            "value": "Entries"
+          },
+          {
+            "key": "match_columns",
+            "value": "name"
+          },
+          {
+            "key": "query",
+            "value": "xml"
+          }
+        ]
+      },
+      "operations": [
+        {
+          "context": "query: xml",
+          "name": "filter",
+          "relative_elapsed": 0.165177838,
+          "slow": true
+        },
+        {
+          "context": null,
+          "name": "select",
+          "relative_elapsed": 6.884999999999999e-06,
+          "slow": false
+        },
+        {
+          "context": null,
+          "name": "output",
+          "relative_elapsed": 0.93196871,
+          "slow": true
+        }
+      ]
+    }
+
+Here are parameters of this filter:
+
+  * `raw_data_column_name`: It specifies column name that stores raw
+    Groonga's query log text.
+    * Default: `message`
+
+  * `slow_operation_threshold`: It specifies threshold to treat an
+    operation is slow. If one or more operations in a query spend more
+    than the threshold, the query is slow query.
+    * Default: `0.1`
+
+  * `slow_response_threshold`: It specifies threshold to treat a
+    request is slow. If a request spends more than the threshold, the
+    query in the request is slow query.
+    * Default: `0.2`
+
+  * `flatten`: It specifies whether parsed query log is mapped to a
+    flat object or a nested object. A float object will be useful to
+    store the parsed log to non document oriented database such as
+    RDBMS.
+
+    Here is a sample record of parsed query log:
+
+        {
+          ...,
+          "command": {
+            "raw": "/d/select?table=Entries&match_columns=name&query=xml",
+            "name": "select",
+            "parameters": [
+              {
+                "key": "table",
+                "value": "Entries"
+              },
+              {
+                "key": "match_columns",
+                "value": "name"
+              },
+              {
+                "key": "query",
+                "value": "xml"
+              }
+            ]
+          },
+          ...
+        }
+
+    Here is the flatten record of the above record:
+
+        {
+          ...,
+          "command.raw": "/d/select?table=Entries&match_columns=name&query=xml",
+          "command.name": "select",
+          "command.parameters[0].key": "table",
+          "command.parameters[0].value": "Entries",
+          "command.parameters[1].key": "match_columns",
+          "command.parameters[1].value": "name",
+          "command.parameters[0].key": "query",
+          "command.parameters[0].value": "xml",
+          ...
+        }
+
+    * Default: `false` (nested object)
+
+  * `flatten_separator`: It specifies separator that is used when
+    `flatten` is `true`. If `flatten` is `true`, nested keys are
+    mapped to one flatten key. This separator is used to concatenate
+    nested keys.
+
+    `.` is used for nested object by default. For example,
+
+         {
+           "a": {
+             "b": 1
+           }
+         }
+
+    is flatten to the following:
+
+         {
+           "a.b": 1
+         }
+
+    `[...]` is used for element in an array by default. For example,
+
+         {
+           "a": [
+             1,
+             2
+           ]
+         }
+
+    is flatten to the following:
+
+         {
+           "a[0]": 1,
+           "a[1]": 2
+         }
+
+    If `"_"` is used as the separator,
+
+         {
+           "a": [
+             1,
+             2
+           ],
+           "b": {
+             "c": 3
+           }
+         }
+
+    is flatten to the following:
+
+         {
+           "a_0": 1,
+           "a_1": 2,
+           "b_c": 3
+         }
+
+    * Default: `.` for object and `[...]` for array
 
 ## Authors
 
@@ -27,7 +318,7 @@ TODO
 
 ## License
 
-LGPL 3. See doc/text/lgpl-3.txt for details.
+LGPL 3 or later. See doc/text/lgpl-3.txt for details.
 
 (Kouhei Sutou has a right to change the license including
 contributed patches.)

data/doc/text/news.md

@@ -1,5 +1,11 @@
 # News
 
+## 1.0.1: 2019-01-09
+
+### Improvements
+
+  * Stopped to use deprecated groonga-query-log API.
+
 ## 1.0.0: 2015-08-12
 
 The first release!!!

data/fluent-plugin-groonga-query-log.gemspec

@@ -1,6 +1,6 @@
 # -*- mode: ruby -*-
 #
-# Copyright (C) 2015  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2015-2018  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published by
@@ -17,7 +17,7 @@
 
 Gem::Specification.new do |spec|
   spec.name = "fluent-plugin-groonga-query-log"
-  spec.version = "1.0.0"
+  spec.version = "1.0.1"
   spec.authors = ["Kouhei Sutou"]
   spec.email = ["kou@clear-code.com"]
   spec.summary = "Fluentd plugin to parse Groonga's query log."
@@ -34,7 +34,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency("fluentd")
-  spec.add_runtime_dependency("groonga-query-log")
+  spec.add_runtime_dependency("groonga-query-log", ">= 1.2.9")
 
   spec.add_development_dependency("rake")
   spec.add_development_dependency("bundler")

data/lib/fluent/plugin/filter_groonga_query_log.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2015  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2015-2018  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published by
@@ -15,7 +15,7 @@
 
 require "time"
 
-require "groonga/query-log"
+require "groonga-query-log"
 
 module Fluent
   class GroongaQueryLogFilter < Filter
@@ -30,7 +30,11 @@ module Fluent
     def configure(conf)
       super
 
-      @parser = Groonga::QueryLog::Parser.new
+      options = {
+        :slow_operation_threshold => @slow_operation_threshold,
+        :slow_response_threshold => @slow_response_threshold,
+      }
+      @parser = GroongaQueryLog::Parser.new(options)
     end
 
     def filter_stream(tag, event_stream)

data/sample/dump.conf

@@ -1,5 +1,5 @@
 <source>
-  type tail
+  @type tail
   path "#{ENV['GROONGA_QUERY_LOG_PATH'] || '/var/log/groonga/query.log'}"
   read_from_head true
   tag groonga.query
@@ -7,9 +7,9 @@
 </source>
 
 <filter groonga.query>
-  type groonga_query_log
+  @type groonga_query_log
 </filter>
 
 <match **>
-  type stdout
+  @type stdout
 </match>

data/sample/store-slow-query.conf

@@ -0,0 +1,41 @@
+<source>
+  @type tail
+  path "#{ENV['GROONGA_QUERY_LOG_PATH'] || '/var/log/groonga/query.log'}"
+  pos_file /var/log/fluentd/groonga-query-log.pos
+  read_from_head "#{ENV['GROONGA_QUERY_LOG_READ_FROM_HEAD'] || 'false'}"
+  tag groonga.query
+  format none
+</source>
+
+<filter groonga.query>
+  @type groonga_query_log
+</filter>
+
+<filter groonga.query>
+  @type grep
+  regexp1 slow \Atrue\z
+</filter>
+
+<filter groonga.query>
+  @type record_transformer
+  enable_ruby true
+  renew_record true
+  keep_keys elapsed
+  <record>
+    log ${JSON.generate(to_h)}
+  </record>
+</filter>
+
+<match groonga.query>
+  @type groonga
+  store_table SlowQueries
+
+  protocol http
+  host 127.0.0.1
+
+  <buffer>
+    @type file
+    path /var/lib/fluentd/groonga.buffer
+    flush_interval 1
+  </buffer>
+</match>

data/sample/store.conf

@@ -1,5 +1,5 @@
 <source>
-  type tail
+  @type tail
   path "#{ENV['GROONGA_QUERY_LOG_PATH'] || '/var/log/groonga/query.log'}"
   pos_file /var/log/td-agent/groonga-query-log.pos
   read_from_head "#{ENV['GROONGA_QUERY_LOG_READ_FROM_HEAD'] || 'false'}"
@@ -8,19 +8,21 @@
 </source>
 
 <filter groonga.query>
-  type groonga_query_log
+  @type groonga_query_log
   flatten true
   flatten_separator _
 </filter>
 
 <match groonga.query>
-  type groonga
-  store_table QueyLogs
+  @type groonga
+  store_table QueryLogs
 
   protocol http
   host 127.0.0.1
 
-  buffer_type file
-  buffer_path /tmp/buffer
-  flush_interval 1
+  <buffer>
+    @type file
+    path /tmp/buffer
+    flush_interval 1
+  </buffer>
 </match>

data/test/test_filter_groonga_query_log.rb

@@ -231,5 +231,97 @@ class GroongaQueryLogFilterTest < Test::Unit::TestCase
       CONFIGURATION
       assert_equal([[@now, statistic]], event_stream.to_a)
     end
+
+    test "slow_operation_threshold" do
+      messages = [
+        "2017-07-12 15:00:00.000000|0x7fb07d113da0|>/d/select?table=Entries&match_columns=name&query=xml",
+        "2017-07-12 15:00:00.100000|0x7fb07d113da0|:000000010000000 filter(10)",
+        "2017-07-12 15:00:00.200000|0x7fb07d113da0|:000000020000000 select(10)",
+        "2017-07-12 15:00:00.300000|0x7fb07d113da0|:000000030000000 output(10)",
+        "2017-07-12 15:00:00.400000|0x7fb07d113da0|<000000040000000 rc=0",
+      ]
+      statistics = {
+        "start_time" => "2017-07-12T06:00:00.000000Z",
+        "last_time" => "2017-07-12T06:00:00.040000Z",
+        "elapsed" => 0.04,
+        "return_code" => 0,
+        "slow" => false,
+        "command" => {
+          "name"=>"select",
+          "parameters" => [
+            {"key" => :table, "value" => "Entries"},
+            {"key" => :match_columns, "value" => "name"},
+            {"key" => :query, "value" => "xml"}
+          ],
+          "raw" => "/d/select?table=Entries&match_columns=name&query=xml"},
+         "operations" => [
+          {"context" => "query: xml",
+           "name" => "filter",
+           "relative_elapsed" => 0.01,
+           "slow" => true
+          },
+          {
+            "context" => nil,
+            "name" => "select",
+            "relative_elapsed" => 0.01,
+            "slow" => true
+          },
+          {
+            "context" => nil,
+            "name" => "output",
+            "relative_elapsed" => 0.01,
+            "slow" => true
+          }
+        ],
+      }
+      event_stream = emit("slow_operation_threshold 0.01", messages)
+      assert_equal([[@now, statistics]], event_stream.to_a)
+    end
+
+    test "slow_response_threshold" do
+      messages = [
+        "2017-07-12 15:00:00.000000|0x7fb07d113da0|>/d/select?table=Entries&match_columns=name&query=xml",
+        "2017-07-12 15:00:00.100000|0x7fb07d113da0|:000000010000000 filter(10)",
+        "2017-07-12 15:00:00.200000|0x7fb07d113da0|:000000020000000 select(10)",
+        "2017-07-12 15:00:00.300000|0x7fb07d113da0|:000000030000000 output(10)",
+        "2017-07-12 15:00:00.400000|0x7fb07d113da0|<000000040000000 rc=0",
+      ]
+      statistics = {
+        "start_time" => "2017-07-12T06:00:00.000000Z",
+        "last_time" => "2017-07-12T06:00:00.040000Z",
+        "elapsed" => 0.04,
+        "return_code" => 0,
+        "slow" => true,
+        "command" => {
+          "name"=>"select",
+          "parameters" => [
+            {"key" => :table, "value" => "Entries"},
+            {"key" => :match_columns, "value" => "name"},
+            {"key" => :query, "value" => "xml"}
+          ],
+          "raw" => "/d/select?table=Entries&match_columns=name&query=xml"},
+         "operations" => [
+          {"context" => "query: xml",
+           "name" => "filter",
+           "relative_elapsed" => 0.01,
+           "slow" => false
+          },
+          {
+            "context" => nil,
+            "name" => "select",
+            "relative_elapsed" => 0.01,
+            "slow" => false
+          },
+          {
+            "context" => nil,
+            "name" => "output",
+            "relative_elapsed" => 0.01,
+            "slow" => false
+          }
+        ],
+      }
+      event_stream = emit("slow_response_threshold 0.01", messages)
+      assert_equal([[@now, statistics]], event_stream.to_a)
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-groonga-query-log
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-12 00:00:00.000000000 Z
+date: 2019-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.2.9
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.2.9
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -109,6 +109,7 @@ files:
 - fluent-plugin-groonga-query-log.gemspec
 - lib/fluent/plugin/filter_groonga_query_log.rb
 - sample/dump.conf
+- sample/store-slow-query.conf
 - sample/store.conf
 - test/run-test.rb
 - test/test_filter_groonga_query_log.rb
@@ -131,12 +132,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 3.0.2
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to parse Groonga's query log.
 test_files:
 - test/run-test.rb
 - test/test_filter_groonga_query_log.rb
-has_rdoc: