fluent-plugin-postgresql-csvlog 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e36a1382a11ed800637467180b1af1a28b16f1125ecd5bf2c5b7dd0b4c708223
-  data.tar.gz: 36f9ea068ffd9674c7a0450feffd3ff7344774a88e63fb8934e3871155ca40a6
+  metadata.gz: 38090772bf1e0302ecd53d759d8da4ffb2df0cdf0a6f001ec0e0d12390472276
+  data.tar.gz: 22bc8479b81f0b0fd7615cfb6cc1b491b317fc74d612f21373243a516e671be4
 SHA512:
-  metadata.gz: e5fe1bd18ada66464c1c6fa57d3cf31f389aa6831e248095d63e2c6d30b737dd363ce72dcf7b80baf771e93921bada05c7d26267969ab161f862d0dc35cc0e57
-  data.tar.gz: bff45c6bf1911f8106b4dd36248df2b14bd8c8d08f1d3214b95d5930c85c385ccfd1ecd23d2d712df8e044c49f5bad4e543f1b18f4de7ab81ecdd0d9001f831c
+  metadata.gz: c3ecfef7d3290ab00a71613c7914c0aae1d05501d37574614d9d6f4a2f5de9548474ecb7b6fdad4ef778c4ea48510a1e31a89294e317ecf7ab6004e094579cc9
+  data.tar.gz: 2b91a1704d53f07ab5139bc2d623c0c17afde53b416ef5f018dd7e6d345ce8a8da1d0aa8964bcef638b8e6ab351b2d7c16b83be73cbdcfba560dda69962bdd17

data/.gitlab-ci.yml

@@ -10,10 +10,9 @@ test:
     paths:
       - vendor/ruby
 
-# integration tests
-itest:
+.iteration_test:
   services:
-    - name: postgres:12
+    - name: postgres:$POSTGRES_SERVER_VERSION
       alias: postgres
       command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all"]
   variables:
@@ -28,6 +27,18 @@ itest:
     paths:
       - vendor/ruby
 
+# integration tests for postgres 12
+itest_pg12:
+  extends: .iteration_test
+  variables:
+    POSTGRES_SERVER_VERSION: 12
+
+# integration tests for postgres 13
+itest_pg13:
+  extends: .iteration_test
+  variables:
+    POSTGRES_SERVER_VERSION: 13
+
 end_to_end_verification_test:
   image: docker:19.03.12
   services:

data/README.md

@@ -80,7 +80,8 @@ ingest and parse PostgreSQL CSV logs:
 To develop and debug locally, there is a `Dockerfile` and `docker-compose.yml` that will setup a local environment,
 complete with Postgres, suitable for testing purposes.
 
-1. `docker compose rm verifier --rm` - test the current configuration
+1. `docker compose build` - build the current configuration
+1. `docker compose run --rm verifier` - test the current configuration
 1. `docker compose up`
 
 ### Releasing a new version

data/docker-compose.yml

@@ -2,7 +2,7 @@
 version: "3.3"
 services:
   postgres:
-    image: postgres
+    image: postgres:13
     restart: "no"
     environment:
       - POSTGRES_USER=testuser

data/fluent-plugin-postgresql-csvlog.gemspec

@@ -2,7 +2,7 @@ $:.push File.expand_path('lib', __dir__)
 
 Gem::Specification.new do |s|
   s.name        = 'fluent-plugin-postgresql-csvlog'
-  s.version     = '0.6.0'
+  s.version     = '0.7.0'
   s.authors     = ['stanhu']
   s.email       = ['stanhu@gmail.com']
   s.homepage    = 'https://gitlab.com/gitlab-org/fluent-plugins/fluent-plugin-postgresql-csvlog'

data/lib/fluent/plugin/in_pg_stat_statements.rb

@@ -19,6 +19,26 @@ module Fluent::Plugin
     desc 'Name of field to store SQL query fingerprint'
     config_param :fingerprint_key, :string, default: 'fingerprint'
 
+    POSTGRES_SERVER_VERSION_QUERY = "SELECT current_setting('server_version_num')"
+
+    PG12_STAT_STATEMENTS_QUERY = <<-SQL
+      SELECT queryid,
+             query,
+             calls,
+             rows,
+             total_time
+        FROM public.pg_stat_statements
+    SQL
+
+    PG13_STAT_STATEMENTS_QUERY = <<-SQL
+      SELECT queryid,
+             query,
+             calls,
+             rows,
+             (total_plan_time + total_exec_time) total_time
+        FROM public.pg_stat_statements
+    SQL
+
     protected
 
     def on_poll
@@ -29,13 +49,24 @@ module Fluent::Plugin
 
     public
 
+    def initialize
+      super
+      @postgres_server_version_num = nil
+    end
+
     # Returns a fluentd record for a query row
     def record_for_row(row)
       query = row['query']
 
       # We record the query_length as it will help in understanding whether unparseable
       # queries are truncated.
-      record = { 'queryid' => row['queryid'].to_s, 'query_length' => query&.length }
+      record = {
+        'queryid' => row['queryid'].to_s,
+        'query_length' => query&.length,
+        'calls' => row['calls']&.to_i,
+        'total_time_ms' => row['total_time']&.to_f,
+        'rows' => row['rows']&.to_i
+      }
 
       return record unless query
 
@@ -56,12 +87,32 @@ module Fluent::Plugin
       me = Fluent::MultiEventStream.new
 
       now = Fluent::Engine.now
-      conn.exec('SELECT queryid, query FROM public.pg_stat_statements').each do |row|
+
+      query = query_for_postgres_version(conn)
+
+      conn.exec(query).each do |row|
         record = record_for_row(row)
         me.add(now, record)
       end
 
       @router.emit_stream(@tag, me)
     end
+
+    # Returns the PG_VERSION_NUM value from the database
+    # will memoize the result
+    def postgres_server_version_num(conn)
+      return @postgres_server_version_num if @postgres_server_version_num
+
+      @postgres_server_version_num = conn.exec(POSTGRES_SERVER_VERSION_QUERY).getvalue(0,0).to_i
+    end
+
+    # pg_stat_statements columns changed in pg13, so we use different queries depending on the version
+    # https://www.postgresql.org/docs/12/pgstatstatements.html
+    # https://www.postgresql.org/docs/13/pgstatstatements.html
+    def query_for_postgres_version(conn)
+      return PG13_STAT_STATEMENTS_QUERY if postgres_server_version_num(conn) >= 13_00_00
+
+      PG12_STAT_STATEMENTS_QUERY
+    end
   end
 end

data/lib/fluent/plugin/polling_pg_input_plugin.rb

@@ -60,7 +60,7 @@ module Fluent::Plugin
           on_poll
         rescue StandardError => e
           log.error 'unexpected error', error: e.message, error_class: e.class
-          log.error_backtrace e.backtrace
+          log.error_backtrace
         end
       end
     end

data/test/plugin/test_in_pg_stat_statements.rb

@@ -42,13 +42,24 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
   sub_test_case 'execution' do
     test 'sql' do
       d = create_driver
-      record = d.instance.record_for_row({ 'queryid' => '1234', 'query' => 'SELECT * FROM users WHERE user_id = ?' })
+      row = {
+        'queryid' => '1234',
+        'query' => 'SELECT * FROM users WHERE user_id = ?',
+        'calls' => 22,
+        'rows' => 333,
+        'total_time' => 44.44
+      }
+
+      record = d.instance.record_for_row(row)
 
       expected = {
         'fingerprint' => 'c071dee80d466e7d',
         'query' => 'SELECT * FROM users WHERE user_id = ?',
         'query_length' => 37,
-        'queryid' => '1234'
+        'queryid' => '1234',
+        'calls' => 22,
+        'rows' => 333,
+        'total_time_ms' => 44.44
       }
 
       assert_equal expected, record
@@ -56,9 +67,22 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
 
     test 'nil query' do
       d = create_driver
-      record = d.instance.record_for_row({ 'queryid' => '1234', 'query' => nil })
+      row = {
+        'queryid' => '1234',
+        'query' => nil,
+        'calls' => nil,
+        'rows' => nil,
+        'total_time' => nil
+      }
+      record = d.instance.record_for_row(row)
 
-      expected = { 'query_length' => nil, 'queryid' => '1234' }
+      expected = {
+        'query_length' => nil,
+        'queryid' => '1234',
+        'calls' => nil,
+        'rows' => nil,
+        'total_time_ms' => nil
+      }
       assert_equal expected, record
     end
 
@@ -75,26 +99,48 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
         )
       SQL
 
-      record = d.instance.record_for_row({ 'queryid' => 1234, 'query' => ddl_sql })
+      row = {
+        'queryid' => 1234,
+        'query' => ddl_sql,
+        'calls' => 22,
+        'rows' => 333,
+        'total_time' => 44.44
+      }
+
+      record = d.instance.record_for_row(row)
 
       expected = {
         'fingerprint' => 'fa9c9d26757c4f9b',
         'query' => ddl_sql,
         'query_length' => 287,
-        'queryid' => '1234'
+        'queryid' => '1234',
+        'calls' => 22,
+        'rows' => 333,
+        'total_time_ms' => 44.44
       }
       assert_equal expected, record
     end
 
     test 'set command' do
       d = create_driver
-      record = d.instance.record_for_row({ 'queryid' => 1234, 'query' => "SET TIME ZONE 'PST8PDT'" })
+      row = {
+        'queryid' => 1234,
+        'query' => "SET TIME ZONE 'PST8PDT'",
+        'calls' => 22,
+        'rows' => 333,
+        'total_time' => 44.44
+      }
+
+      record = d.instance.record_for_row(row)
 
       expected = {
         'fingerprint' => '23f8d6eb1d3125c3',
         'query' => 'SET TIME ZONE $1',
         'query_length' => 23,
-        'queryid' => '1234'
+        'queryid' => '1234',
+        'calls' => 22,
+        'rows' => 333,
+        'total_time_ms' => 44.44
       }
 
       assert_equal expected, record
@@ -104,7 +150,15 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
       d = create_driver
       record = d.instance.record_for_row({ 'queryid' => 1234, 'query' => 'SELECT * FROM' })
 
-      expected = { 'query_length' => 13, 'query_unparseable' => true, 'queryid' => '1234' }
+      expected = {
+        'query_length' => 13,
+        'query_unparseable' => true,
+        'queryid' => '1234',
+        'calls' => nil,
+        'rows' => nil,
+        'total_time_ms' => nil
+      }
+
       assert_equal expected, record
     end
   end

data/test/verify-docker-compose.sh

@@ -5,8 +5,8 @@
 
 cleanup() {
   echo "# removing all logs"
-  rm -rf /var/log/pg/pg_stat_statements.*.log
-  rm -rf /var/log/pg/pg_stat_activity.*.log
+  find /var/log/pg/ -name "pg_stat_statements.*.log" -delete
+  find /var/log/pg/ -name "pg_stat_activity.*.log" -delete
 }
 
 die() {
@@ -19,10 +19,14 @@ cleanup
 echo "# sleeping 10, awaiting logs"
 sleep 10;
 
-(find /var/log/pg/ -name "pg_stat_statements.*.log" | grep .) || die "No pg_stat_statements files created"
-cat /var/log/pg/pg_stat_statements.*.log | tail -1
+echo "# looking for pg_stat_statements"
 
-(find /var/log/pg/ -name "pg_stat_activity.*.log" | grep .) || die "No pg_stat_activity files created"
-cat /var/log/pg/pg_stat_activity.*.log | tail -1
+(find /var/log/pg/ -name "pg_stat_statements.*.log" | grep .  >/dev/null) || die "No pg_stat_statements files created"
+cat /var/log/pg/pg_stat_statements.*.log | tail -10
+
+echo "# looking for pg_stat_activity"
+
+(find /var/log/pg/ -name "pg_stat_activity.*.log" | grep . >/dev/null) || die "No pg_stat_activity files created"
+cat /var/log/pg/pg_stat_activity.*.log | tail -10
 
 cleanup

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-postgresql-csvlog
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - stanhu
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-29 00:00:00.000000000 Z
+date: 2021-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -125,7 +125,7 @@ files:
 homepage: https://gitlab.com/gitlab-org/fluent-plugins/fluent-plugin-postgresql-csvlog
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -140,8 +140,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key: 
+rubygems_version: 3.2.28
+signing_key:
 specification_version: 4
 summary: fluentd plugins to work with PostgreSQL CSV logs
 test_files: