fluent-plugin-postgresql-csvlog 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3f63769c3a4c78fc3db074b812e6f75e77c46b355563b403c3de620c49935392
-  data.tar.gz: 2fd8d1acd1fac5ce24b72ed4d488fc5f80f0a8752084784cf16f419389f34499
+  metadata.gz: 5dda962fe3d98ea5f101183e982959b78c92142ca2b9a7807e51567a2bc75819
+  data.tar.gz: 3efd68ecdc6bcdb0a3341b2a5961452508ea92b10a2e5eeba6843cb29f27009a
 SHA512:
-  metadata.gz: 34fefa81b79223bf1840481f6ffd232e4d343a1eed9876f8f58893f686c8c4a1c1fbd37679c1a8960bcddef3a992733fe4b53158ee7558c10b49ac42a1793996
-  data.tar.gz: d105daf33a577b40e0035bc2a6d512390c361bb42bb9a11c2290e7489f04f813338779b45682969fa7616e8d7bb319db04dfec1eaf624586e50709c4a71eba83
+  metadata.gz: 47099891a7c455ce85e98239b3c69990b7c14b5450f2be9d643902c00ac29a6bbb7a5f34678380b1e310414308e80514f8915b0865f92ef7524221d35fc75ca8
+  data.tar.gz: 22bd0a3a4f2bf5ef903c3f90d11d0f92a5822887a07cd048eca701a4cb270c8be4f1c43f5fa2e9f3529ec26896874f9156dc0eac457c07028f41e03e3e7bf34a

data/README.md

@@ -47,11 +47,13 @@ ingest and parse PostgreSQL CSV logs:
 
 <filter postgres.postgres_csv>
   @type postgresql_slowlog
+  output_key query
 </filter>
 
 <filter postgres.postgres_csv>
   @type postgresql_redactor
-  key sql
+  input_key query
+  output_key sql
   fingerprint_key fingerprint
 </filter>
 

data/fluent-plugin-postgresql-csvlog.gemspec

@@ -2,7 +2,7 @@ $:.push File.expand_path('lib', __dir__)
 
 Gem::Specification.new do |s|
   s.name        = 'fluent-plugin-postgresql-csvlog'
-  s.version     = '0.1.0'
+  s.version     = '0.2.0'
   s.authors     = ['stanhu']
   s.email       = ['stanhu@gmail.com']
   s.homepage    = 'https://gitlab.com/gitlab-org/fluent-plugin-postgresql-csvlog'

data/lib/fluent/plugin/filter_postgresql_redactor.rb

@@ -7,22 +7,25 @@ module Fluent::Plugin
   class PostgreSQLRedactor < Filter
     Fluent::Plugin.register_filter('postgresql_redactor', self)
 
-    desc 'Field to parse for SQL queries'
-    config_param :key, :string, default: 'sql'
+    desc 'Input field to parse for SQL queries'
+    config_param :input_key, :string, default: 'query'
+
+    desc 'Output field to store SQL queries'
+    config_param :output_key, :string, default: 'sql'
 
     desc 'Name of field to store SQL query fingerprint'
     config_param :fingerprint_key, :string, default: 'fingerprint'
 
     def filter(_tag, _time, record)
-      statement = record['statement']
+      statement = record[@input_key]
 
       return record unless statement
 
       normalized = PgQuery.normalize(statement)
       record[@fingerprint_key] = PgQuery.parse(normalized).fingerprint if @fingerprint_key
 
-      record.delete('statement')
-      record[@key] = normalized
+      record.delete(@input_key)
+      record[@output_key] = normalized
       record.delete('message')
 
       record

data/lib/fluent/plugin/filter_postgresql_slowlog.rb

@@ -12,6 +12,9 @@ module Fluent
     class PostgreSQLSlowLog < Filter
       Fluent::Plugin.register_filter('postgresql_slowlog', self)
 
+      desc 'Field to output SQL queries'
+      config_param :output_key, :string, default: 'query'
+
       SLOWLOG_REGEXP = /^duration: (\d+(?:\.\d+)?) ms .*?:\s*(.*)/m.freeze
 
       def filter(_tag, _time, record)
@@ -20,7 +23,7 @@ module Fluent
         # rubocop:disable Style/PerlBackrefs
         if record['message'] =~ SLOWLOG_REGEXP
           record['duration_s'] = $1.to_f / 1000.0
-          record['statement'] = $2
+          record[@output_key] = $2
         end
         # rubocop:enable Style/PerlBackrefs
 

data/test/plugin/test_filter_marginalia.rb

@@ -22,7 +22,8 @@ class Marginalia < Test::Unit::TestCase
     inputs = [
       { 'statement' => 'SELECT * FROM projects' },
       { 'statement' => 'SELECT COUNT(*) FROM "projects" /* this is just a comment */' },
-      { 'statement' => 'SELECT COUNT(*) FROM "projects" /*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/' }
+      { 'statement' => 'SELECT COUNT(*) FROM "projects" /*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/' },
+      { 'statement' => 'SELECT COUNT(*) FROM "projects" /*application:web,correlation_id:01F1D2T1SC9DM82A4865ATG1CP,endpoint_id:POST /api/:version/groups/:id/-/packages/mavenpath/:file_name*/' }
     ]
 
     d.run(default_tag: @tag) do
@@ -40,10 +41,19 @@ class Marginalia < Test::Unit::TestCase
                    }
                  ),
                  d.filtered[2].last)
+    assert_equal(inputs[3].merge(
+                   {
+                     'application' => 'web',
+                     'correlation_id' => '01F1D2T1SC9DM82A4865ATG1CP',
+                     'endpoint_id' => 'POST /api/:version/groups/:id/-/packages/mavenpath/:file_name'
+                   }
+                 ),
+                 d.filtered[3].last)
 
     assert_equal('SELECT * FROM projects', d.filtered[0].last['statement'])
     assert_equal('SELECT COUNT(*) FROM "projects"', d.filtered[1].last['statement'])
     assert_equal('SELECT COUNT(*) FROM "projects"', d.filtered[2].last['statement'])
+    assert_equal('SELECT COUNT(*) FROM "projects"', d.filtered[3].last['statement'])
   end
 
   test 'parses prepended Marginalia comments' do
@@ -52,6 +62,7 @@ class Marginalia < Test::Unit::TestCase
     inputs = [
       { 'statement' => '/* this is just a comment */ SELECT COUNT(*) FROM "projects"' },
       { 'statement' => '/*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/ SELECT COUNT(*) FROM "projects"' },
+      { 'statement' => '/*application:web,correlation_id:01F1D2T1SC9DM82A4865ATG1CP,endpoint_id:POST /api/:version/groups/:id/-/packages/mavenpath/:file_name*/ SELECT COUNT(*) FROM "projects"' },
       { 'statement' => '/*application:sidekiq*/ SELECT COUNT(*) FROM "projects"',
         'application' => 'test-conflict' }
     ]
@@ -72,10 +83,18 @@ class Marginalia < Test::Unit::TestCase
                  d.filtered[1].last)
     assert_equal(inputs[2].merge(
                    {
-                     'statement_application' => 'sidekiq'
+                     'application' => 'web',
+                     'correlation_id' => '01F1D2T1SC9DM82A4865ATG1CP',
+                     'endpoint_id' => 'POST /api/:version/groups/:id/-/packages/mavenpath/:file_name'
                    }
                  ),
                  d.filtered[2].last)
+    assert_equal(inputs[3].merge(
+                   {
+                     'statement_application' => 'sidekiq'
+                   }
+                 ),
+                 d.filtered[3].last)
 
     assert_equal('SELECT COUNT(*) FROM "projects"', d.filtered[0].last['statement'])
     assert_equal('SELECT COUNT(*) FROM "projects"', d.filtered[1].last['statement'])

data/test/plugin/test_filter_postgresql_redactor.rb

@@ -8,11 +8,7 @@ class PostgreSQLRedactorTest < Test::Unit::TestCase
     @tag = 'test.tag'
   end
 
-  CONFIG = '
-    <filter test.tag>
-      @type postgresql_redactor
-    </filter>
-  '
+  CONFIG = ''
 
   def create_driver(conf = CONFIG)
     Fluent::Test::Driver::Filter.new(Fluent::Plugin::PostgreSQLRedactor).configure(conf)
@@ -23,7 +19,7 @@ class PostgreSQLRedactorTest < Test::Unit::TestCase
 
     inputs = [
       { 'message' => 'duration: 2357.1 ms  execute <unnamed>: SELECT * FROM projects WHERE id = 1',
-        'statement' => %(SELECT * FROM projects WHERE id = 1),
+        'query' => %(SELECT * FROM projects WHERE id = 1),
         'duration_s' => 2.3571 }
     ]
 
@@ -38,13 +34,36 @@ class PostgreSQLRedactorTest < Test::Unit::TestCase
   test 'handles parse errors' do
     d = create_driver
 
-    input = { 'statement' => 'create index something test (bla) include (bar)' }
+    input = { 'query' => 'create index something test (bla) include (bar)' }
 
     d.run(default_tag: @tag) do
       d.feed(input)
     end
 
-    assert_equal(%w[pg_query_error statement], d.filtered[0].last.keys.sort)
-    assert_equal(input['statement'], d.filtered[0].last['statement'])
+    assert_equal(%w[pg_query_error query], d.filtered[0].last.keys.sort)
+    assert_equal(input['query'], d.filtered[0].last['query'])
+  end
+
+  test 'uses configured input and output keys' do
+    d = create_driver(<<~CONF
+      input_key sql
+      output_key out_sql
+    CONF
+                     )
+
+    inputs = [
+      {
+        'message' => 'duration: 2357.1 ms  execute <unnamed>: SELECT * FROM projects WHERE id = 1',
+        'sql' => %(SELECT * FROM projects WHERE id = 1),
+        'duration_s' => 2.3571
+      }
+    ]
+
+    d.run(default_tag: @tag) do
+      inputs.each { |input| d.feed(input) }
+    end
+
+    assert_equal(%w[duration_s fingerprint out_sql], d.filtered[0].last.keys.sort)
+    assert_equal('SELECT * FROM projects WHERE id = $1', d.filtered[0].last['out_sql'])
   end
 end

data/test/plugin/test_filter_postgresql_slowlog.rb

@@ -32,14 +32,14 @@ class PostgreSQLSlowLogTest < Test::Unit::TestCase
 
     assert_equal(inputs[0].merge(
                    {
-                     'statement' => 'SELECT * FROM projects',
+                     'query' => 'SELECT * FROM projects',
                      'duration_s' => 2.3571
                    }
                  ),
                  d.filtered[0].last)
     assert_equal(inputs[1].merge(
                    {
-                     'statement' => 'SELECT COUNT(*) FROM "projects" /*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/',
+                     'query' => 'SELECT COUNT(*) FROM "projects" /*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/',
                      'duration_s' => 1.873345
                    }
                  ),
@@ -56,4 +56,28 @@ class PostgreSQLSlowLogTest < Test::Unit::TestCase
 
     assert_equal(input, d.filtered[0].last)
   end
+
+  test 'outputs slow log entries to configured output key' do
+    d = create_driver(
+      <<~CONF
+        output_key my_key
+      CONF
+    )
+
+    inputs = [
+      { 'message' => 'duration: 2357.1 ms  execute <unnamed>: SELECT * FROM projects' }
+    ]
+
+    d.run(default_tag: @tag) do
+      inputs.each { |input| d.feed(input) }
+    end
+
+    assert_equal(inputs[0].merge(
+                   {
+                     'my_key' => 'SELECT * FROM projects',
+                     'duration_s' => 2.3571
+                   }
+                 ),
+                 d.filtered[0].last)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-postgresql-csvlog
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - stanhu
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-21 00:00:00.000000000 Z
+date: 2021-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -81,7 +81,6 @@ extra_rdoc_files: []
 files:
 - ".gitlab-ci.yml"
 - Gemfile
-- Gemfile.lock
 - LICENSE
 - README.md
 - Rakefile

data/Gemfile.lock

@@ -1,52 +0,0 @@
-PATH
-  remote: .
-  specs:
-    fluent-plugin-postgresql-csvlog (0.1.0)
-      fluentd (>= 1.0, < 2)
-      pg_query (~> 2.0)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    concurrent-ruby (1.1.8)
-    cool.io (1.7.1)
-    fluentd (1.12.1)
-      bundler
-      cool.io (>= 1.4.5, < 2.0.0)
-      http_parser.rb (>= 0.5.1, < 0.7.0)
-      msgpack (>= 1.3.1, < 2.0.0)
-      serverengine (>= 2.2.2, < 3.0.0)
-      sigdump (~> 0.2.2)
-      strptime (>= 0.2.2, < 1.0.0)
-      tzinfo (>= 1.0, < 3.0)
-      tzinfo-data (~> 1.0)
-      yajl-ruby (~> 1.0)
-    google-protobuf (3.15.6-universal-darwin)
-    http_parser.rb (0.6.0)
-    msgpack (1.4.2)
-    pg_query (2.0.1)
-      google-protobuf (~> 3.15.5)
-    power_assert (2.0.0)
-    rake (13.0.3)
-    serverengine (2.2.3)
-      sigdump (~> 0.2.2)
-    sigdump (0.2.4)
-    strptime (0.2.5)
-    test-unit (3.4.0)
-      power_assert
-    tzinfo (2.0.4)
-      concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2021.1)
-      tzinfo (>= 1.0.0)
-    yajl-ruby (1.4.1)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  fluent-plugin-postgresql-csvlog!
-  rake
-  test-unit (~> 3.2)
-
-BUNDLED WITH
-   2.1.4