fluent-plugin-postgresql-csvlog 0.0.2 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 002b7169e3f3ac6493eb09dca5c2e3820a944f38c64a0b45641f3cdbd1717ddf
-  data.tar.gz: eac3646a2404665924c4fa30114b91ecce49dbadef3919d89eab458450568328
+  metadata.gz: 3f63769c3a4c78fc3db074b812e6f75e77c46b355563b403c3de620c49935392
+  data.tar.gz: 2fd8d1acd1fac5ce24b72ed4d488fc5f80f0a8752084784cf16f419389f34499
 SHA512:
-  metadata.gz: 354478f3573f0934dcee72305069aa8f25333087d1667fc4e127963b8f8f955b66a02462925e4672f27f80e43b772640a68a4760266f461704bd9486ad66a3ef
-  data.tar.gz: f6d89c2db73d337b1aa52e9838ab08f6f3bda80e7053307482fa5b284b4b6f81ae0389e2a20411fbcc383c878b5309a320946d5849d43fa466e469037b9d5102
+  metadata.gz: 34fefa81b79223bf1840481f6ffd232e4d343a1eed9876f8f58893f686c8c4a1c1fbd37679c1a8960bcddef3a992733fe4b53158ee7558c10b49ac42a1793996
+  data.tar.gz: d105daf33a577b40e0035bc2a6d512390c361bb42bb9a11c2290e7489f04f813338779b45682969fa7616e8d7bb319db04dfec1eaf624586e50709c4a71eba83

data/.gitlab-ci.yml

@@ -2,6 +2,7 @@ image: "ruby:2.7"
 
 test:
   before_script:
+    - bundle config set path vendor
     - bundle install --jobs $(nproc)
   script:
     - bundle exec rake test

data/Gemfile.lock

@@ -1,16 +1,16 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-postgresql-csvlog (0.0.2)
+    fluent-plugin-postgresql-csvlog (0.1.0)
       fluentd (>= 1.0, < 2)
-      pg_query (~> 1.3)
+      pg_query (~> 2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     concurrent-ruby (1.1.8)
-    cool.io (1.7.0)
-    fluentd (1.12.0)
+    cool.io (1.7.1)
+    fluentd (1.12.1)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
@@ -21,12 +21,14 @@ GEM
       tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
+    google-protobuf (3.15.6-universal-darwin)
     http_parser.rb (0.6.0)
     msgpack (1.4.2)
-    pg_query (1.3.0)
+    pg_query (2.0.1)
+      google-protobuf (~> 3.15.5)
     power_assert (2.0.0)
     rake (13.0.3)
-    serverengine (2.2.2)
+    serverengine (2.2.3)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     strptime (0.2.5)

data/README.md

@@ -51,11 +51,14 @@ ingest and parse PostgreSQL CSV logs:
 
 <filter postgres.postgres_csv>
   @type postgresql_redactor
+  key sql
+  fingerprint_key fingerprint
 </filter>
 
 <filter postgres.postgres_csv>
   @type marginalia
   key sql
+  strip_comment true
 </filter>
 
 # Output resulting JSON file to a directory in /tmp

data/fluent-plugin-postgresql-csvlog.gemspec

@@ -2,7 +2,7 @@ $:.push File.expand_path('lib', __dir__)
 
 Gem::Specification.new do |s|
   s.name        = 'fluent-plugin-postgresql-csvlog'
-  s.version     = '0.0.2'
+  s.version     = '0.1.0'
   s.authors     = ['stanhu']
   s.email       = ['stanhu@gmail.com']
   s.homepage    = 'https://gitlab.com/gitlab-org/fluent-plugin-postgresql-csvlog'
@@ -15,7 +15,7 @@ Gem::Specification.new do |s|
   s.require_paths = ['lib']
 
   s.add_dependency 'fluentd', ['>= 1.0', '< 2']
-  s.add_dependency 'pg_query', '~> 1.3'
+  s.add_dependency 'pg_query', '~> 2.0'
 
   s.add_development_dependency 'rake'
   s.add_development_dependency 'test-unit', '~> 3.2'

data/lib/fluent/plugin/filter_marginalia.rb

@@ -16,9 +16,11 @@ module Fluent
       desc 'Field to parse for Marginalia comments (key1:value1,key2:value2)'
       config_param :key, :string, default: 'sql'
 
-      MARGINALIA_PREPENDED_REGEXP = %r{^(?<comment>/\*.*\*/).*}m.freeze
-      MARGINALIA_APPENDED_REGEXP = %r{.*(?<comment>/\*.*\*/)$}m.freeze
-      MARGINALIA_KEY_VALUE_REGEXP = /^(?<key>.*):?(?<value>.*)$/.freeze
+      desc 'Whether to strip the comment from the record specified by key'
+      config_param :strip_comment, :bool, default: true
+
+      MARGINALIA_PREPENDED_REGEXP = %r{^(?<comment>/\*.*\*/)(?<sql>.*)}m.freeze
+      MARGINALIA_APPENDED_REGEXP = %r{(?<sql>.*)(?<comment>/\*.*\*/)$}m.freeze
 
       def filter(_tag, _time, record)
         parse_comments(record)
@@ -38,15 +40,9 @@ module Fluent
         return unless comment_match
 
         entries = extract_entries(comment_match['comment'])
+        parse_entries(entries, record)
 
-        entries.each do |component|
-          data = component.split(':', 2)
-
-          break unless data.length == 2
-
-          stored_key = store_key(record, data[0])
-          record[stored_key] = data[1]
-        end
+        record[@key] = comment_match['sql'].strip if @strip_comment
       end
 
       def match_marginalia_comment(sql)
@@ -73,7 +69,21 @@ module Fluent
         comment.gsub!(%r{\*/$}, '')
       end
 
+      def parse_entries(entries, record)
+        entries.each do |component|
+          data = component.split(':', 2)
+
+          break unless data.length == 2
+
+          stored_key = store_key(record, data[0])
+          record[stored_key] = data[1]
+        end
+      end
+
       def store_key(record, component_key)
+        # In case there is a conflict with the Marginalia key
+        # (e.g. `correlation_id`), we use the base key
+        # (`sql_correlation_id`) instead.
         if record.key?(component_key)
           "#{@key}_#{component_key}"
         else

data/lib/fluent/plugin/filter_postgresql_redactor.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'fluent/plugin/filter'
 require 'pg_query'
 
@@ -5,9 +7,11 @@ module Fluent::Plugin
   class PostgreSQLRedactor < Filter
     Fluent::Plugin.register_filter('postgresql_redactor', self)
 
-    def configure(conf)
-      super
-    end
+    desc 'Field to parse for SQL queries'
+    config_param :key, :string, default: 'sql'
+
+    desc 'Name of field to store SQL query fingerprint'
+    config_param :fingerprint_key, :string, default: 'fingerprint'
 
     def filter(_tag, _time, record)
       statement = record['statement']
@@ -15,14 +19,14 @@ module Fluent::Plugin
       return record unless statement
 
       normalized = PgQuery.normalize(statement)
+      record[@fingerprint_key] = PgQuery.parse(normalized).fingerprint if @fingerprint_key
+
       record.delete('statement')
-      record['sql'] = normalized
+      record[@key] = normalized
       record.delete('message')
 
       record
     rescue PgQuery::ParseError
-      # pg_query currently only supports PostgresQL 10:
-      # https://github.com/lfittl/pg_query/issues/184
       record['pg_query_error'] = true
       record
     end

data/test/plugin/test_filter_marginalia.rb

@@ -29,8 +29,8 @@ class Marginalia < Test::Unit::TestCase
       inputs.each { |input| d.feed(input) }
     end
 
-    assert_equal(inputs[0].merge, d.filtered[0].last)
-    assert_equal(inputs[1].merge, d.filtered[1].last)
+    assert_equal(inputs[0], d.filtered[0].last)
+    assert_equal(inputs[1], d.filtered[1].last)
     assert_equal(inputs[2].merge(
                    {
                      'application' => 'sidekiq',
@@ -40,6 +40,10 @@ class Marginalia < Test::Unit::TestCase
                    }
                  ),
                  d.filtered[2].last)
+
+    assert_equal('SELECT * FROM projects', d.filtered[0].last['statement'])
+    assert_equal('SELECT COUNT(*) FROM "projects"', d.filtered[1].last['statement'])
+    assert_equal('SELECT COUNT(*) FROM "projects"', d.filtered[2].last['statement'])
   end
 
   test 'parses prepended Marginalia comments' do
@@ -56,7 +60,7 @@ class Marginalia < Test::Unit::TestCase
       inputs.each { |input| d.feed(input) }
     end
 
-    assert_equal(inputs[0].merge, d.filtered[0].last)
+    assert_equal(inputs[0], d.filtered[0].last)
     assert_equal(inputs[1].merge(
                    {
                      'application' => 'sidekiq',
@@ -72,5 +76,33 @@ class Marginalia < Test::Unit::TestCase
                    }
                  ),
                  d.filtered[2].last)
+
+    assert_equal('SELECT COUNT(*) FROM "projects"', d.filtered[0].last['statement'])
+    assert_equal('SELECT COUNT(*) FROM "projects"', d.filtered[1].last['statement'])
+    assert_equal('SELECT COUNT(*) FROM "projects"', d.filtered[2].last['statement'])
+  end
+
+  test 'parses Marginalia comments with strip_comment disabled' do
+    d = create_driver(
+      <<~CONF
+        strip_comment false
+        key sql
+      CONF
+    )
+
+    sql = %(SELECT COUNT(*) FROM "projects" /*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/)
+    appended_sql = %(SELECT COUNT(*) FROM "projects" /*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/')
+
+    inputs = [
+      { 'sql' => sql },
+      { 'sql' => appended_sql }
+    ]
+
+    d.run(default_tag: @tag) do
+      inputs.each { |input| d.feed(input) }
+    end
+
+    assert_equal(sql, d.filtered[0].last['sql'])
+    assert_equal(appended_sql, d.filtered[1].last['sql'])
   end
 end

data/test/plugin/test_filter_postgresql_redactor.rb

@@ -31,14 +31,14 @@ class PostgreSQLRedactorTest < Test::Unit::TestCase
       inputs.each { |input| d.feed(input) }
     end
 
-    assert_equal(%w[duration_s sql], d.filtered[0].last.keys.sort)
+    assert_equal(%w[duration_s fingerprint sql], d.filtered[0].last.keys.sort)
     assert_equal('SELECT * FROM projects WHERE id = $1', d.filtered[0].last['sql'])
   end
 
   test 'handles parse errors' do
     d = create_driver
 
-    input = { 'statement' => 'create index concurrently foo on test (bla) include (bar)' }
+    input = { 'statement' => 'create index something test (bla) include (bar)' }
 
     d.run(default_tag: @tag) do
       d.feed(input)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-postgresql-csvlog
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors:
 - stanhu
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-20 00:00:00.000000000 Z
+date: 2021-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -36,14 +36,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement