fluent-plugin-postgresql-csvlog 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9fd4d916e4d8cc9ebb39d62e847de1c2dca1ce0175d738ca55430a962b3283ab
-  data.tar.gz: b3e6baacf5bd683c39eca587c55ea0326ffe18a55d3b4f8b23ba435642641b8c
+  metadata.gz: 002b7169e3f3ac6493eb09dca5c2e3820a944f38c64a0b45641f3cdbd1717ddf
+  data.tar.gz: eac3646a2404665924c4fa30114b91ecce49dbadef3919d89eab458450568328
 SHA512:
-  metadata.gz: 2e783b77515eb3ec55b4684e11d3014fdea761b057d22fb3935eafd7064d9fcb7038b8f78f7fa7e9aa238d339c8a5c3d1cac358a430c207f6ba9cd19bf8ae55a
-  data.tar.gz: 77dace38e2de7a851547ee25502943f0d610cceefa8ee4d1194e084301852bf3b1e8fd93e79144f6c81865d7aadfe3c3705a210f3ca4dee2029e2daa5fbdf989
+  metadata.gz: 354478f3573f0934dcee72305069aa8f25333087d1667fc4e127963b8f8f955b66a02462925e4672f27f80e43b772640a68a4760266f461704bd9486ad66a3ef
+  data.tar.gz: f6d89c2db73d337b1aa52e9838ab08f6f3bda80e7053307482fa5b284b4b6f81ae0389e2a20411fbcc383c878b5309a320946d5849d43fa466e469037b9d5102

data/.gitlab-ci.yml

@@ -0,0 +1,10 @@
+image: "ruby:2.7"
+
+test:
+  before_script:
+    - bundle install --jobs $(nproc)
+  script:
+    - bundle exec rake test
+  cache:
+    paths:
+      - vendor/ruby

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-postgresql-csvlog (0.0.1)
+    fluent-plugin-postgresql-csvlog (0.0.2)
       fluentd (>= 1.0, < 2)
       pg_query (~> 1.3)
 

data/README.md

@@ -6,6 +6,7 @@ parse PostgreSQL CSV log files and extract slow log information:
 - `MultilineCSVParser`: Parses CSV files that span multiple lines
 - `PostgreSQLSlowLog`: Extracts slow log entries into `duration_s` and `statement` fields
 - `PostgreSQLRedactor`: Normalizes the SQL query and redacts sensitive information
+- `Marginalia`: Parses [Marginalia comments](https://github.com/basecamp/marginalia) into key-value pairs and stores them
 
 ## Installation
 
@@ -28,7 +29,6 @@ The configuration below shows how you might use these filters to
 ingest and parse PostgreSQL CSV logs:
 
 ```conf
-## PostgreSQL csvlog (enabled with 
 <source>
   @type tail
   tag postgres.postgres_csv
@@ -53,6 +53,11 @@ ingest and parse PostgreSQL CSV logs:
   @type postgresql_redactor
 </filter>
 
+<filter postgres.postgres_csv>
+  @type marginalia
+  key sql
+</filter>
+
 # Output resulting JSON file to a directory in /tmp
 <match postgres.*>
   @type file

data/Rakefile

@@ -0,0 +1,12 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/**/test_*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]

data/fluent-plugin-postgresql-csvlog.gemspec

@@ -2,7 +2,7 @@ $:.push File.expand_path('lib', __dir__)
 
 Gem::Specification.new do |s|
   s.name        = 'fluent-plugin-postgresql-csvlog'
-  s.version     = '0.0.1'
+  s.version     = '0.0.2'
   s.authors     = ['stanhu']
   s.email       = ['stanhu@gmail.com']
   s.homepage    = 'https://gitlab.com/gitlab-org/fluent-plugin-postgresql-csvlog'

data/lib/fluent/plugin/filter_marginalia.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require 'fluent/plugin/filter'
+
+module Fluent
+  module Plugin
+    # Filters SQL statements for Marginalia comments.
+    #
+    # Examples:
+    # SELECT COUNT(*) FROM "projects" /*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/
+    # /*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/ SELECT COUNT(*) FROM "projects"
+    #
+    class Marginalia < Filter
+      Fluent::Plugin.register_filter('marginalia', self)
+
+      desc 'Field to parse for Marginalia comments (key1:value1,key2:value2)'
+      config_param :key, :string, default: 'sql'
+
+      MARGINALIA_PREPENDED_REGEXP = %r{^(?<comment>/\*.*\*/).*}m.freeze
+      MARGINALIA_APPENDED_REGEXP = %r{.*(?<comment>/\*.*\*/)$}m.freeze
+      MARGINALIA_KEY_VALUE_REGEXP = /^(?<key>.*):?(?<value>.*)$/.freeze
+
+      def filter(_tag, _time, record)
+        parse_comments(record)
+
+        record
+      end
+
+      private
+
+      def parse_comments(record)
+        sql = record[@key]
+
+        return unless sql
+
+        comment_match = match_marginalia_comment(sql)
+
+        return unless comment_match
+
+        entries = extract_entries(comment_match['comment'])
+
+        entries.each do |component|
+          data = component.split(':', 2)
+
+          break unless data.length == 2
+
+          stored_key = store_key(record, data[0])
+          record[stored_key] = data[1]
+        end
+      end
+
+      def match_marginalia_comment(sql)
+        matched = MARGINALIA_PREPENDED_REGEXP.match(sql)
+
+        return matched if matched
+
+        MARGINALIA_APPENDED_REGEXP.match(sql)
+      end
+
+      def extract_entries(comment)
+        comment = scrub_comment(comment)
+
+        return [] unless comment
+
+        comment.split(',')
+      end
+
+      def scrub_comment(comment)
+        return unless comment
+
+        comment.strip!
+        comment.gsub!(%r{^/\*}, '')
+        comment.gsub!(%r{\*/$}, '')
+      end
+
+      def store_key(record, component_key)
+        if record.key?(component_key)
+          "#{@key}_#{component_key}"
+        else
+          component_key
+        end
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -13,3 +13,4 @@ Test::Unit::TestCase.include(Fluent::Test::Helpers)
 
 require 'fluent/plugin/filter_postgresql_slowlog'
 require 'fluent/plugin/filter_postgresql_redactor'
+require 'fluent/plugin/filter_marginalia'

data/test/plugin/test_filter_marginalia.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require_relative '../helper'
+
+class Marginalia < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    @tag = 'test.tag'
+  end
+
+  CONFIG = '
+    key statement
+  '
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::Marginalia).configure(conf)
+  end
+
+  test 'parses appended Marginalia comments' do
+    d = create_driver
+
+    inputs = [
+      { 'statement' => 'SELECT * FROM projects' },
+      { 'statement' => 'SELECT COUNT(*) FROM "projects" /* this is just a comment */' },
+      { 'statement' => 'SELECT COUNT(*) FROM "projects" /*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/' }
+    ]
+
+    d.run(default_tag: @tag) do
+      inputs.each { |input| d.feed(input) }
+    end
+
+    assert_equal(inputs[0].merge, d.filtered[0].last)
+    assert_equal(inputs[1].merge, d.filtered[1].last)
+    assert_equal(inputs[2].merge(
+                   {
+                     'application' => 'sidekiq',
+                     'correlation_id' => 'd67cae54c169e0cab7d73389e2934f0e',
+                     'jid' => '52a1c8a9e4c555ea573f20f0',
+                     'job_class' => 'Geo::MetricsUpdateWorker'
+                   }
+                 ),
+                 d.filtered[2].last)
+  end
+
+  test 'parses prepended Marginalia comments' do
+    d = create_driver
+
+    inputs = [
+      { 'statement' => '/* this is just a comment */ SELECT COUNT(*) FROM "projects"' },
+      { 'statement' => '/*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/ SELECT COUNT(*) FROM "projects"' },
+      { 'statement' => '/*application:sidekiq*/ SELECT COUNT(*) FROM "projects"',
+        'application' => 'test-conflict' }
+    ]
+
+    d.run(default_tag: @tag) do
+      inputs.each { |input| d.feed(input) }
+    end
+
+    assert_equal(inputs[0].merge, d.filtered[0].last)
+    assert_equal(inputs[1].merge(
+                   {
+                     'application' => 'sidekiq',
+                     'correlation_id' => 'd67cae54c169e0cab7d73389e2934f0e',
+                     'jid' => '52a1c8a9e4c555ea573f20f0',
+                     'job_class' => 'Geo::MetricsUpdateWorker'
+                   }
+                 ),
+                 d.filtered[1].last)
+    assert_equal(inputs[2].merge(
+                   {
+                     'statement_application' => 'sidekiq'
+                   }
+                 ),
+                 d.filtered[2].last)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-postgresql-csvlog
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - stanhu
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-10 00:00:00.000000000 Z
+date: 2021-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -79,15 +79,19 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitlab-ci.yml"
 - Gemfile
 - Gemfile.lock
 - LICENSE
 - README.md
+- Rakefile
 - fluent-plugin-postgresql-csvlog.gemspec
+- lib/fluent/plugin/filter_marginalia.rb
 - lib/fluent/plugin/filter_postgresql_redactor.rb
 - lib/fluent/plugin/filter_postgresql_slowlog.rb
 - lib/fluent/plugin/parser_multiline_csv.rb
 - test/helper.rb
+- test/plugin/test_filter_marginalia.rb
 - test/plugin/test_filter_postgresql_redactor.rb
 - test/plugin/test_filter_postgresql_slowlog.rb
 homepage: https://gitlab.com/gitlab-org/fluent-plugin-postgresql-csvlog
@@ -112,4 +116,8 @@ rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: fluentd plugins to work with PostgreSQL CSV logs
-test_files: []
+test_files:
+- test/helper.rb
+- test/plugin/test_filter_marginalia.rb
+- test/plugin/test_filter_postgresql_redactor.rb
+- test/plugin/test_filter_postgresql_slowlog.rb