fluent-plugin-postgresql-csvlog 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 9fd4d916e4d8cc9ebb39d62e847de1c2dca1ce0175d738ca55430a962b3283ab
+  data.tar.gz: b3e6baacf5bd683c39eca587c55ea0326ffe18a55d3b4f8b23ba435642641b8c
+SHA512:
+  metadata.gz: 2e783b77515eb3ec55b4684e11d3014fdea761b057d22fb3935eafd7064d9fcb7038b8f78f7fa7e9aa238d339c8a5c3d1cac358a430c207f6ba9cd19bf8ae55a
+  data.tar.gz: 77dace38e2de7a851547ee25502943f0d610cceefa8ee4d1194e084301852bf3b1e8fd93e79144f6c81865d7aadfe3c3705a210f3ca4dee2029e2daa5fbdf989

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,50 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-postgresql-csvlog (0.0.1)
+      fluentd (>= 1.0, < 2)
+      pg_query (~> 1.3)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    concurrent-ruby (1.1.8)
+    cool.io (1.7.0)
+    fluentd (1.12.0)
+      bundler
+      cool.io (>= 1.4.5, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      msgpack (>= 1.3.1, < 2.0.0)
+      serverengine (>= 2.2.2, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (>= 0.2.2, < 1.0.0)
+      tzinfo (>= 1.0, < 3.0)
+      tzinfo-data (~> 1.0)
+      yajl-ruby (~> 1.0)
+    http_parser.rb (0.6.0)
+    msgpack (1.4.2)
+    pg_query (1.3.0)
+    power_assert (2.0.0)
+    rake (13.0.3)
+    serverengine (2.2.2)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    strptime (0.2.5)
+    test-unit (3.4.0)
+      power_assert
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2021.1)
+      tzinfo (>= 1.0.0)
+    yajl-ruby (1.4.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  fluent-plugin-postgresql-csvlog!
+  rake
+  test-unit (~> 3.2)
+
+BUNDLED WITH
+   2.1.4

data/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2011-present GitLab B.V.
+
+Portions of this software are licensed as follows:
+
+* All content residing under the "doc/" directory of this repository is licensed under "Creative Commons: CC BY-SA 4.0 license".
+* All content that resides under the "ee/" directory of this repository, if that directory exists, is licensed under the license defined in "ee/LICENSE".
+* All client-side JavaScript (when served directly or after being compiled, arranged, augmented, or combined), is licensed under the "MIT Expat" license.
+* All third party components incorporated into the GitLab Software are licensed under the original license provided by the owner of the applicable component.
+* Content outside of the above mentioned directories or restrictions above is available under the "MIT Expat" license as defined below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,64 @@
+# Introduction
+
+This repository holds several [Fluentd](http://fluentd.org) plugins to
+parse PostgreSQL CSV log files and extract slow log information:
+
+- `MultilineCSVParser`: Parses CSV files that span multiple lines
+- `PostgreSQLSlowLog`: Extracts slow log entries into `duration_s` and `statement` fields
+- `PostgreSQLRedactor`: Normalizes the SQL query and redacts sensitive information
+
+## Installation
+
+Use RubyGems:
+
+    fluent-gem install fluent-plugin-postgresql-csvlog --no-document
+
+## Configuration
+
+### PostgreSQL configuration
+
+```conf
+log_destination = 'csvlog'
+log_collector = on
+```
+
+### Fluentd conf
+
+The configuration below shows how you might use these filters to
+ingest and parse PostgreSQL CSV logs:
+
+```conf
+## PostgreSQL csvlog (enabled with 
+<source>
+  @type tail
+  tag postgres.postgres_csv
+  path /var/log/postgresql/*.csv
+  pos_file /var/log/td-agent/postgres_csv.log.pos
+  <parse>
+    @type multiline_csv
+    format_firstline /^\d{4}-\d{1,2}-\d{1,2} \d{2}:\d{2}:\d{2}/
+    keys time,user_name,database_name,process_id,connection_from,session_id,session_line_num,command_tag,session_start_time,virtual_transaction_id,transaction_id,error_severity,sql_state_code,message,detail,hint,internal_query,internal_query_pos,context,query,query_pos,location,application_name
+    time_key time
+    time_format %Y-%m-%d %H:%M:%S.%N %Z
+    keep_time_key true
+    parser_type normal
+  </parse>
+</source>
+
+<filter postgres.postgres_csv>
+  @type postgresql_slowlog
+</filter>
+
+<filter postgres.postgres_csv>
+  @type postgresql_redactor
+</filter>
+
+# Output resulting JSON file to a directory in /tmp
+<match postgres.*>
+  @type file
+  path /tmp/fluent-test
+  <format>
+    @type json
+  </format>
+</match>
+```

data/fluent-plugin-postgresql-csvlog.gemspec

@@ -0,0 +1,22 @@
+$:.push File.expand_path('lib', __dir__)
+
+Gem::Specification.new do |s|
+  s.name        = 'fluent-plugin-postgresql-csvlog'
+  s.version     = '0.0.1'
+  s.authors     = ['stanhu']
+  s.email       = ['stanhu@gmail.com']
+  s.homepage    = 'https://gitlab.com/gitlab-org/fluent-plugin-postgresql-csvlog'
+  s.summary     = 'fluentd plugins to work with PostgreSQL CSV logs'
+  s.description = 'fluentd plugins to work with PostgreSQL CSV logs'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
+  s.require_paths = ['lib']
+
+  s.add_dependency 'fluentd', ['>= 1.0', '< 2']
+  s.add_dependency 'pg_query', '~> 1.3'
+
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'test-unit', '~> 3.2'
+end

data/lib/fluent/plugin/filter_postgresql_redactor.rb

@@ -0,0 +1,30 @@
+require 'fluent/plugin/filter'
+require 'pg_query'
+
+module Fluent::Plugin
+  class PostgreSQLRedactor < Filter
+    Fluent::Plugin.register_filter('postgresql_redactor', self)
+
+    def configure(conf)
+      super
+    end
+
+    def filter(_tag, _time, record)
+      statement = record['statement']
+
+      return record unless statement
+
+      normalized = PgQuery.normalize(statement)
+      record.delete('statement')
+      record['sql'] = normalized
+      record.delete('message')
+
+      record
+    rescue PgQuery::ParseError
+      # pg_query currently only supports PostgresQL 10:
+      # https://github.com/lfittl/pg_query/issues/184
+      record['pg_query_error'] = true
+      record
+    end
+  end
+end

data/lib/fluent/plugin/filter_postgresql_slowlog.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'fluent/plugin/filter'
+
+module Fluent
+  module Plugin
+    # Filters PostgreSQL slow log duration and statements from parsed record.
+    #
+    # Examples:
+    # duration: 2357.1 ms  execute <unnamed>: SELECT * FROM projects
+    # duration: 1873.345 ms  execute <unnamed>: SELECT COUNT(*) FROM "projects" /*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/
+    class PostgreSQLSlowLog < Filter
+      Fluent::Plugin.register_filter('postgresql_slowlog', self)
+
+      SLOWLOG_REGEXP = /^duration: (\d+(?:\.\d+)?) ms .*?:\s*(.*)/m.freeze
+
+      def filter(_tag, _time, record)
+        return record unless record.key?('message')
+
+        # rubocop:disable Style/PerlBackrefs
+        if record['message'] =~ SLOWLOG_REGEXP
+          record['duration_s'] = $1.to_f / 1000.0
+          record['statement'] = $2
+        end
+        # rubocop:enable Style/PerlBackrefs
+
+        record
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_multiline_csv.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'fluent/plugin/parser_csv'
+
+module Fluent
+  module Plugin
+    # This class can be used to parse CSV files that span multiple lines.
+    # Like the [multiline parser](https://docs.fluentd.org/parser/multiline),
+    # define a `format_firstline` for the `tail` input module to match against.
+    # Then use the [csv parser](https://docs.fluentd.org/parser/csv) parameters.
+    class MultilineCSVParser < CSVParser
+      Plugin.register_parser('multiline_csv', self)
+
+      desc 'Specify regexp pattern for start line of multiple lines'
+      config_param :format_firstline, :string, default: nil
+
+      def configure(conf)
+        super
+
+        if @format_firstline
+          check_format_regexp(@format_firstline, 'format_firstline')
+          @firstline_regex = Regexp.new(@format_firstline[1..-2])
+        end
+      end
+
+      # Used by in_tail:
+      # https://github.com/fluent/fluentd/blob/47be96e3a98fa247b59e479f7c62bfeff1a9ec55/lib/fluent/plugin/in_tail.rb#L523
+      def has_firstline?
+        !!@format_firstline
+      end
+
+      # Used by in_tail:
+      # https://github.com/fluent/fluentd/blob/47be96e3a98fa247b59e479f7c62bfeff1a9ec55/lib/fluent/plugin/in_tail.rb#L526
+      def firstline?(text)
+        @firstline_regex.match(text)
+      end
+
+      private
+
+      def check_format_regexp(format, key)
+        if format[0] == '/' && format[-1] == '/'
+          begin
+            Regexp.new(format[1..-2])
+          rescue StandardError => e
+            raise ConfigError, "Invalid regexp in #{key}: #{e}"
+          end
+        else
+          raise ConfigError, "format_firstline should be Regexp, need //: '#{format}'"
+        end
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
+$LOAD_PATH.unshift(__dir__)
+require 'fluent/test'
+require 'fluent/test/driver/filter'
+require 'fluent/test/helpers'
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+
+require 'fluent/plugin/filter_postgresql_slowlog'
+require 'fluent/plugin/filter_postgresql_redactor'

data/test/plugin/test_filter_postgresql_redactor.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative '../helper'
+
+class PostgreSQLRedactorTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    @tag = 'test.tag'
+  end
+
+  CONFIG = '
+    <filter test.tag>
+      @type postgresql_redactor
+    </filter>
+  '
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::PostgreSQLRedactor).configure(conf)
+  end
+
+  test 'filters SQL queries' do
+    d = create_driver
+
+    inputs = [
+      { 'message' => 'duration: 2357.1 ms  execute <unnamed>: SELECT * FROM projects WHERE id = 1',
+        'statement' => %(SELECT * FROM projects WHERE id = 1),
+        'duration_s' => 2.3571 }
+    ]
+
+    d.run(default_tag: @tag) do
+      inputs.each { |input| d.feed(input) }
+    end
+
+    assert_equal(%w[duration_s sql], d.filtered[0].last.keys.sort)
+    assert_equal('SELECT * FROM projects WHERE id = $1', d.filtered[0].last['sql'])
+  end
+
+  test 'handles parse errors' do
+    d = create_driver
+
+    input = { 'statement' => 'create index concurrently foo on test (bla) include (bar)' }
+
+    d.run(default_tag: @tag) do
+      d.feed(input)
+    end
+
+    assert_equal(%w[pg_query_error statement], d.filtered[0].last.keys.sort)
+    assert_equal(input['statement'], d.filtered[0].last['statement'])
+  end
+end

data/test/plugin/test_filter_postgresql_slowlog.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require_relative '../helper'
+
+class PostgreSQLSlowLogTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    @tag = 'test.tag'
+  end
+
+  CONFIG = '
+    <filter test.tag>
+      @type postgresql_slowlog
+    </filter>
+  '
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::PostgreSQLSlowLog).configure(conf)
+  end
+
+  test 'filters basic slow log' do
+    d = create_driver
+
+    inputs = [
+      { 'message' => 'duration: 2357.1 ms  execute <unnamed>: SELECT * FROM projects' },
+      { 'message' => 'duration: 1873.345 ms  execute <unnamed>: SELECT COUNT(*) FROM "projects" /*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/' }
+    ]
+
+    d.run(default_tag: @tag) do
+      inputs.each { |input| d.feed(input) }
+    end
+
+    assert_equal(inputs[0].merge(
+                   {
+                     'statement' => 'SELECT * FROM projects',
+                     'duration_s' => 2.3571
+                   }
+                 ),
+                 d.filtered[0].last)
+    assert_equal(inputs[1].merge(
+                   {
+                     'statement' => 'SELECT COUNT(*) FROM "projects" /*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/',
+                     'duration_s' => 1.873345
+                   }
+                 ),
+                 d.filtered[1].last)
+  end
+
+  test 'ignores messages not having to do with slow logs' do
+    d = create_driver
+    input = { 'message' => 'this is a test' }
+
+    d.run(default_tag: @tag) do
+      d.feed(input)
+    end
+
+    assert_equal(input, d.filtered[0].last)
+  end
+end

metadata

@@ -0,0 +1,115 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-postgresql-csvlog
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- stanhu
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-02-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: pg_query
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+description: fluentd plugins to work with PostgreSQL CSV logs
+email:
+- stanhu@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- fluent-plugin-postgresql-csvlog.gemspec
+- lib/fluent/plugin/filter_postgresql_redactor.rb
+- lib/fluent/plugin/filter_postgresql_slowlog.rb
+- lib/fluent/plugin/parser_multiline_csv.rb
+- test/helper.rb
+- test/plugin/test_filter_postgresql_redactor.rb
+- test/plugin/test_filter_postgresql_slowlog.rb
+homepage: https://gitlab.com/gitlab-org/fluent-plugin-postgresql-csvlog
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.4
+signing_key: 
+specification_version: 4
+summary: fluentd plugins to work with PostgreSQL CSV logs
+test_files: []