fluent-plugin-postgresql-csvlog 0.4.0 → 0.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e1cc2264997378ec801c6623768d57ca27ce1c28d6ac8ffcb6a905add2ce9f16
-  data.tar.gz: 79aab5d187209db5790f9bca181ea0ca8c909c808809590ad46cef0f6645902c
+  metadata.gz: 5fd63aaf4685f342a67b1ae1393d6f0ab7c4fab2d76b0e1b8f65a20d544c7fbf
+  data.tar.gz: faca443f2ff5b0bba6baa960918619e21ba03d86055cbb6a38d27278e56d903c
 SHA512:
-  metadata.gz: e38e47908ac33706c4b6679fabc97a395bb63f8a110cb6a836c93f984dbfaf1f77f2204209cfe8a0609ddf969ab960cd855ce363398388084a0026e5df0c4970
-  data.tar.gz: 94505a6d7ceee368c22b0c32081b85aaf215d28217ceed1ff8f58bab840367f5afe19ed725056394e57fd1d6ebfe3ae98fba66b39eb781cfa2f1f80a602385c6
+  metadata.gz: de5e28ed56dc6fcfeb7f0631538e796561556cadd3d5b03a3510cd4f575291c51bf9c335d1f49415569a7c8bdd217e1cb22a411258fe6cbeed023cad15cb2232
+  data.tar.gz: 176678ab47f25823b1dbce6bec33e4d86ee5c02ae72a7370541d399d91a0aa79cd9b575fb4a75a3fe53f0cf4400b7b55cd0102e071de7c52279e6b936f03ec20

data/.gitlab-ci.yml

@@ -10,10 +10,9 @@ test:
     paths:
       - vendor/ruby
 
-# integration tests
-itest:
+.iteration_test:
   services:
-    - name: postgres:12
+    - name: postgres:$POSTGRES_SERVER_VERSION
       alias: postgres
       command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all"]
   variables:
@@ -28,6 +27,18 @@ itest:
     paths:
       - vendor/ruby
 
+# integration tests for postgres 12
+itest_pg12:
+  extends: .iteration_test
+  variables:
+    POSTGRES_SERVER_VERSION: 12
+
+# integration tests for postgres 13
+itest_pg13:
+  extends: .iteration_test
+  variables:
+    POSTGRES_SERVER_VERSION: 13
+
 end_to_end_verification_test:
   image: docker:19.03.12
   services:

data/README.md

@@ -80,7 +80,8 @@ ingest and parse PostgreSQL CSV logs:
 To develop and debug locally, there is a `Dockerfile` and `docker-compose.yml` that will setup a local environment,
 complete with Postgres, suitable for testing purposes.
 
-1. `docker compose rm verifier --rm` - test the current configuration
+1. `docker compose build` - build the current configuration
+1. `docker compose run --rm verifier` - test the current configuration
 1. `docker compose up`
 
 ### Releasing a new version

data/docker-compose.yml

@@ -2,7 +2,7 @@
 version: "3.3"
 services:
   postgres:
-    image: postgres
+    image: postgres:13
     restart: "no"
     environment:
       - POSTGRES_USER=testuser

data/example-fluentd.conf

@@ -13,6 +13,10 @@
   time_slice_format %Y%m%d%H%M%S
   flush_interval 1s
   utc
+
+  <format>
+    @type json
+  </format>
 </match>
 
 <source>
@@ -23,3 +27,15 @@
   password testpass
   interval 1
 </source>
+
+<match postgres.pg_stat_activity>
+  @type file
+  path /var/log/pg/pg_stat_activity
+  time_slice_format %Y%m%d%H%M%S
+  flush_interval 1s
+  utc
+  <format>
+    @type json
+  </format>
+</match>
+

data/fluent-plugin-postgresql-csvlog.gemspec

@@ -2,7 +2,7 @@ $:.push File.expand_path('lib', __dir__)
 
 Gem::Specification.new do |s|
   s.name        = 'fluent-plugin-postgresql-csvlog'
-  s.version     = '0.4.0'
+  s.version     = '0.7.1'
   s.authors     = ['stanhu']
   s.email       = ['stanhu@gmail.com']
   s.homepage    = 'https://gitlab.com/gitlab-org/fluent-plugins/fluent-plugin-postgresql-csvlog'
@@ -16,7 +16,7 @@ Gem::Specification.new do |s|
 
   s.add_dependency 'fluentd', ['>= 1.0', '< 2']
   s.add_dependency 'pg', '~> 1.1'
-  s.add_dependency 'pg_query', '~> 2.1'
+  s.add_dependency 'pg_query', '~> 2.0'
 
   s.add_development_dependency 'rake'
   s.add_development_dependency 'test-unit', '~> 3.2'

data/lib/fluent/plugin/in_pg_stat_statements.rb

@@ -11,7 +11,7 @@ module Fluent::Plugin
   #  'fingerprint' => '8a6e9896bd9048a2',
   #  'query' => 'SELECT * FROM table ORDER BY queryid LIMIT $1',
   #  'query_length' => 58,
-  #  'queryid' => 3239318621761098074
+  #  'queryid' => '3239318621761098074'
   # }
   class PgStatStatementsInput < PollingPostgresInputPlugin
     Fluent::Plugin.register_input('pg_stat_statements', self)
@@ -19,6 +19,26 @@ module Fluent::Plugin
     desc 'Name of field to store SQL query fingerprint'
     config_param :fingerprint_key, :string, default: 'fingerprint'
 
+    POSTGRES_SERVER_VERSION_QUERY = "SELECT current_setting('server_version_num')"
+
+    PG12_STAT_STATEMENTS_QUERY = <<-SQL
+      SELECT queryid,
+             query,
+             calls,
+             rows,
+             total_time
+        FROM public.pg_stat_statements
+    SQL
+
+    PG13_STAT_STATEMENTS_QUERY = <<-SQL
+      SELECT queryid,
+             query,
+             calls,
+             rows,
+             (total_plan_time + total_exec_time) total_time
+        FROM public.pg_stat_statements
+    SQL
+
     protected
 
     def on_poll
@@ -29,13 +49,24 @@ module Fluent::Plugin
 
     public
 
+    def initialize
+      super
+      @postgres_server_version_num = nil
+    end
+
     # Returns a fluentd record for a query row
     def record_for_row(row)
       query = row['query']
 
       # We record the query_length as it will help in understanding whether unparseable
       # queries are truncated.
-      record = { 'queryid' => row['queryid'], 'query_length' => query&.length }
+      record = {
+        'queryid' => row['queryid'].to_s,
+        'query_length' => query&.length,
+        'calls' => row['calls']&.to_i,
+        'total_time_ms' => row['total_time']&.to_f,
+        'rows' => row['rows']&.to_i
+      }
 
       return record unless query
 
@@ -56,12 +87,32 @@ module Fluent::Plugin
       me = Fluent::MultiEventStream.new
 
       now = Fluent::Engine.now
-      conn.exec('SELECT queryid, query FROM public.pg_stat_statements').each do |row|
+
+      query = query_for_postgres_version(conn)
+
+      conn.exec(query).each do |row|
         record = record_for_row(row)
         me.add(now, record)
       end
 
       @router.emit_stream(@tag, me)
     end
+
+    # Returns the PG_VERSION_NUM value from the database
+    # will memoize the result
+    def postgres_server_version_num(conn)
+      return @postgres_server_version_num if @postgres_server_version_num
+
+      @postgres_server_version_num = conn.exec(POSTGRES_SERVER_VERSION_QUERY).getvalue(0,0).to_i
+    end
+
+    # pg_stat_statements columns changed in pg13, so we use different queries depending on the version
+    # https://www.postgresql.org/docs/12/pgstatstatements.html
+    # https://www.postgresql.org/docs/13/pgstatstatements.html
+    def query_for_postgres_version(conn)
+      return PG13_STAT_STATEMENTS_QUERY if postgres_server_version_num(conn) >= 13_00_00
+
+      PG12_STAT_STATEMENTS_QUERY
+    end
   end
 end

data/lib/fluent/plugin/marginalia_extractor.rb

@@ -7,7 +7,7 @@ module Fluent::Plugin
   # utility method, useful for extracting marginalia into fluentd records
   module MarginaliaExtractor
     MARGINALIA_PREPENDED_REGEXP = %r{^(?<comment>/\*.*\*/)(?<sql>.*)}m.freeze
-    MARGINALIA_APPENDED_REGEXP = %r{(?<sql>.*)(?<comment>/\*.*\*/)$}m.freeze
+    MARGINALIA_APPENDED_REGEXP = %r{(?<sql>.*)(?<comment>/\*.*\*/)\s*;?\s*$}m.freeze
 
     # Injects marginalia into a fluentd record
     def parse_marginalia_into_record(record, key, strip_comment)

data/lib/fluent/plugin/polling_pg_input_plugin.rb

@@ -60,7 +60,7 @@ module Fluent::Plugin
           on_poll
         rescue StandardError => e
           log.error 'unexpected error', error: e.message, error_class: e.class
-          log.error_backtrace e.backtrace
+          log.error_backtrace
         end
       end
     end

data/test/plugin/itest_in_pg_stat_statements.rb

@@ -98,8 +98,7 @@ class PgStatStatementsInputIntegrationTest < Test::Unit::TestCase
       expected_record = {
         'fingerprint' => '8a6e9896bd9048a2',
         'query' => 'SELECT * FROM pg_stat_statements ORDER BY queryid LIMIT $1',
-        'query_length' => 58,
-        'queryid' => 3_239_318_621_761_098_074
+        'query_length' => 58
       }
       known_statement_event = emits.find do |event|
         record = event[2]
@@ -114,7 +113,8 @@ class PgStatStatementsInputIntegrationTest < Test::Unit::TestCase
       assert_equal 'postgres.pg_stat_statements', tag
       assert_equal expected_record['fingerprint'], record['fingerprint']
       assert_equal expected_record['query_length'], record['query_length']
-      assert_true expected_record.include? 'queryid'
+      assert_true record.include? 'queryid'
+      assert_true record['queryid'].is_a? String
     end
   end
 end

data/test/plugin/test_in_pg_stat_statements.rb

@@ -42,13 +42,24 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
   sub_test_case 'execution' do
     test 'sql' do
       d = create_driver
-      record = d.instance.record_for_row({ 'queryid' => '1234', 'query' => 'SELECT * FROM users WHERE user_id = ?' })
+      row = {
+        'queryid' => '1234',
+        'query' => 'SELECT * FROM users WHERE user_id = ?',
+        'calls' => 22,
+        'rows' => 333,
+        'total_time' => 44.44
+      }
+
+      record = d.instance.record_for_row(row)
 
       expected = {
         'fingerprint' => 'c071dee80d466e7d',
         'query' => 'SELECT * FROM users WHERE user_id = ?',
         'query_length' => 37,
-        'queryid' => '1234'
+        'queryid' => '1234',
+        'calls' => 22,
+        'rows' => 333,
+        'total_time_ms' => 44.44
       }
 
       assert_equal expected, record
@@ -56,9 +67,22 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
 
     test 'nil query' do
       d = create_driver
-      record = d.instance.record_for_row({ 'queryid' => '1234', 'query' => nil })
+      row = {
+        'queryid' => '1234',
+        'query' => nil,
+        'calls' => nil,
+        'rows' => nil,
+        'total_time' => nil
+      }
+      record = d.instance.record_for_row(row)
 
-      expected = { 'query_length' => nil, 'queryid' => '1234' }
+      expected = {
+        'query_length' => nil,
+        'queryid' => '1234',
+        'calls' => nil,
+        'rows' => nil,
+        'total_time_ms' => nil
+      }
       assert_equal expected, record
     end
 
@@ -75,26 +99,48 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
         )
       SQL
 
-      record = d.instance.record_for_row({ 'queryid' => '1234', 'query' => ddl_sql })
+      row = {
+        'queryid' => 1234,
+        'query' => ddl_sql,
+        'calls' => 22,
+        'rows' => 333,
+        'total_time' => 44.44
+      }
+
+      record = d.instance.record_for_row(row)
 
       expected = {
         'fingerprint' => 'fa9c9d26757c4f9b',
         'query' => ddl_sql,
         'query_length' => 287,
-        'queryid' => '1234'
+        'queryid' => '1234',
+        'calls' => 22,
+        'rows' => 333,
+        'total_time_ms' => 44.44
       }
       assert_equal expected, record
     end
 
     test 'set command' do
       d = create_driver
-      record = d.instance.record_for_row({ 'queryid' => '1234', 'query' => "SET TIME ZONE 'PST8PDT'" })
+      row = {
+        'queryid' => 1234,
+        'query' => "SET TIME ZONE 'PST8PDT'",
+        'calls' => 22,
+        'rows' => 333,
+        'total_time' => 44.44
+      }
+
+      record = d.instance.record_for_row(row)
 
       expected = {
         'fingerprint' => '23f8d6eb1d3125c3',
         'query' => 'SET TIME ZONE $1',
         'query_length' => 23,
-        'queryid' => '1234'
+        'queryid' => '1234',
+        'calls' => 22,
+        'rows' => 333,
+        'total_time_ms' => 44.44
       }
 
       assert_equal expected, record
@@ -102,9 +148,17 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
 
     test 'unparseable sql' do
       d = create_driver
-      record = d.instance.record_for_row({ 'queryid' => '1234', 'query' => 'SELECT * FROM' })
+      record = d.instance.record_for_row({ 'queryid' => 1234, 'query' => 'SELECT * FROM' })
+
+      expected = {
+        'query_length' => 13,
+        'query_unparseable' => true,
+        'queryid' => '1234',
+        'calls' => nil,
+        'rows' => nil,
+        'total_time_ms' => nil
+      }
 
-      expected = { 'query_length' => 13, 'query_unparseable' => true, 'queryid' => '1234' }
       assert_equal expected, record
     end
   end

data/test/plugin/test_marginalia_extractor.rb

@@ -56,6 +56,14 @@ class Marginalia < Test::Unit::TestCase
     test_parse(sql, {}, 'sql', true, expected)
   end
 
+  test 'normal comment appended with trailing semicolon' do
+    sql = 'SELECT COUNT(*) FROM "projects" /* this is just a comment */ ; '
+    expected = {
+      "sql" => 'SELECT COUNT(*) FROM "projects"'
+    }
+    test_parse(sql, {}, 'sql', true, expected)
+  end
+
   test 'marginalia prepended for sidekiq' do
     sql = '/*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/ SELECT COUNT(*) FROM "projects"'
     expected = {

data/test/verify-docker-compose.sh

@@ -5,7 +5,8 @@
 
 cleanup() {
   echo "# removing all logs"
-  rm -rf /var/log/pg/*
+  find /var/log/pg/ -name "pg_stat_statements.*.log" -delete
+  find /var/log/pg/ -name "pg_stat_activity.*.log" -delete
 }
 
 die() {
@@ -17,6 +18,15 @@ die() {
 cleanup
 echo "# sleeping 10, awaiting logs"
 sleep 10;
-find /var/log/pg/ -name "pg_stat_statements.*.log" || die "No pg_stat_statements files created"
+
+echo "# looking for pg_stat_statements"
+
+(find /var/log/pg/ -name "pg_stat_statements.*.log" | grep .  >/dev/null) || die "No pg_stat_statements files created"
+cat /var/log/pg/pg_stat_statements.*.log | tail -10
+
+echo "# looking for pg_stat_activity"
+
+(find /var/log/pg/ -name "pg_stat_activity.*.log" | grep . >/dev/null) || die "No pg_stat_activity files created"
+cat /var/log/pg/pg_stat_activity.*.log | tail -10
 
 cleanup

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-postgresql-csvlog
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.7.1
 platform: ruby
 authors:
 - stanhu
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-23 00:00:00.000000000 Z
+date: 2021-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -50,14 +50,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -125,7 +125,7 @@ files:
 homepage: https://gitlab.com/gitlab-org/fluent-plugins/fluent-plugin-postgresql-csvlog
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -140,8 +140,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key: 
+rubygems_version: 3.2.28
+signing_key:
 specification_version: 4
 summary: fluentd plugins to work with PostgreSQL CSV logs
 test_files: