fluent-plugin-postgresql-csvlog 0.10.0 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2393ec76502b4127cd874027d99eada9e184dba85edfec3c2a9cfe1a88de54ee
-  data.tar.gz: 2f61b0b0d4e27a302f90449af833dba0e985406d98f15be773ac379146a958d1
+  metadata.gz: af7b81970b58d90863db57c0386b096ca5fe62f4b5c100cd62a6a40b416c15a1
+  data.tar.gz: 53fdf00e4405b46f5db1a40abc29a1d8e1ffe2daeb9f3597027f96fb9fb2d1c0
 SHA512:
-  metadata.gz: b537e7458f9fa38bb0b88672e137a345739f22ab92f8a3d9fe3cf0e26f9e958995c2f7fdb51afc649ac5f55e608ddd431ef904dd757e651f1b223be58703d7d6
-  data.tar.gz: 7724c6961993e7e50ef38ebfa22b874fd4578eb8d8057d2cac76fe761f723a5631e1a227c791c7debf24c2631bc4eea1c2febe929f723a5584c52adbb2b91a49
+  metadata.gz: f69402f67122706714e07283efc6b131f8c8b4f06851ed77b01d5188d305dd9d57e5e62a3d732cba0272b06db6c5e266d953fb19edb5af27d4396e715ca05be7
+  data.tar.gz: ae1f30e585ace5fdba2d81921f22eec5f2a99a0ae3cec8ae71e2e19fa27b89e10f4f81e7057edbea913582920dc3d4bdcb81195a42e13c21077bccd1667da368

data/fluent-plugin-postgresql-csvlog.gemspec

@@ -2,7 +2,7 @@ $:.push File.expand_path('lib', __dir__)
 
 Gem::Specification.new do |s|
   s.name        = 'fluent-plugin-postgresql-csvlog'
-  s.version     = '0.10.0'
+  s.version     = '0.11.0'
   s.authors     = ['stanhu']
   s.email       = ['stanhu@gmail.com']
   s.homepage    = 'https://gitlab.com/gitlab-org/fluent-plugins/fluent-plugin-postgresql-csvlog'

data/lib/fluent/plugin/filter_postgresql_redactor.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
 require 'fluent/plugin/filter'
-require 'pg_query'
+
+require_relative './query_normalizer'
 
 module Fluent::Plugin
   class PostgreSQLRedactor < Filter
@@ -19,32 +20,17 @@ module Fluent::Plugin
     desc 'Truncate the query if it exceeds the number of bytes'
     config_param :max_length, :integer, default: 3 * 1024 * 1024
 
-    def filter(_tag, _time, record)
-      statement = record[@input_key]
-
-      return record unless statement
-
-      normalized = PgQuery.normalize(statement)
-      record[@fingerprint_key] = PgQuery.fingerprint(normalized) if @fingerprint_key
-
-      record.delete(@input_key)
-      record[@output_key] = truncate_query(record, normalized)
+    include QueryNormalizer
 
-      record
-    rescue PgQuery::ParseError
-      record['pg_query_error'] = true
-      record
-    end
-
-    private
-
-    def truncate_query(record, normalized)
-      return normalized if normalized.length < @max_length
-
-      record['truncated_query'] = true
-      # Assume UTF-8 encoding for PostgreSQL queries
-      utf8_query = normalized.encode('UTF-8', invalid: :replace, undef: :replace, replace: '')
-      utf8_query[0, @max_length]
+    def filter(_tag, _time, record)
+      opts = {
+        input_key: @input_key,
+        output_key: @output_key,
+        fingerprint_key: @fingerprint_key,
+        max_length: @max_length
+      }
+
+      normalize_and_fingerprint_query(record, opts)
     end
   end
 end

data/lib/fluent/plugin/in_pg_stat_activity.rb

@@ -12,6 +12,8 @@ module Fluent::Plugin
   # Fingerprints of the queries are also included for easier aggregation
   class PgStatActivityInput < PollingPostgresInputPlugin
     include MarginaliaExtractor
+    include QueryNormalizer
+
     Fluent::Plugin.register_input('pg_stat_activity', self)
 
     ACTIVITY_QUERY = <<-SQL
@@ -42,6 +44,9 @@ module Fluent::Plugin
     desc 'Name of field to store SQL query fingerprint'
     config_param :fingerprint_key, :string, default: 'fingerprint'
 
+    desc 'Truncate the query if it exceeds the number of bytes'
+    config_param :max_length, :integer, default: 3 * 1024 * 1024
+
     protected
 
     def on_poll
@@ -94,31 +99,14 @@ module Fluent::Plugin
       # Inject marginalia into record
       parse_marginalia_into_record(record, 'query', true)
 
-      # Normalize query and fingerprint
-      # Note that `record['query']` was updated in previous step
-      # To strip off marginalia comments
-      record.merge!(fingerprint_query(record['query']))
-
-      record
-    end
-
-    def fingerprint_query(query)
-      # We record the query_length as it will help in understanding whether unparseable
-      # queries are truncated.
-      record = { 'query_length' => query&.length, 'query' => nil }
-
-      return record unless query
-
-      normalized = PgQuery.normalize(query)
-      record['query'] = normalized
-
-      record[@fingerprint_key] = PgQuery.fingerprint(normalized) if @fingerprint_key
-
-      record
-    rescue PgQuery::ParseError
-      record['query_unparseable'] = true
+      opts = {
+        input_key: 'query',
+        output_key: 'query',
+        fingerprint_key: @fingerprint_key,
+        max_length: @max_length
+      }
 
-      record
+      normalize_and_fingerprint_query(record, opts)
     end
   end
 end

data/lib/fluent/plugin/in_pg_stat_statements.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative './polling_pg_input_plugin'
-require 'pg_query'
+require_relative './query_normalizer'
 
 module Fluent::Plugin
   # PgStatStatementsInput will periodically poll postgres, querying pg_stat_statements
@@ -19,6 +19,11 @@ module Fluent::Plugin
     desc 'Name of field to store SQL query fingerprint'
     config_param :fingerprint_key, :string, default: 'fingerprint'
 
+    desc 'Truncate the query if it exceeds the number of bytes'
+    config_param :max_length, :integer, default: 3 * 1024 * 1024
+
+    include QueryNormalizer
+
     POSTGRES_SERVER_VERSION_QUERY = "SELECT current_setting('server_version_num')"
 
     PG12_STAT_STATEMENTS_QUERY = <<-SQL
@@ -56,30 +61,22 @@ module Fluent::Plugin
 
     # Returns a fluentd record for a query row
     def record_for_row(row)
-      query = row['query']
-
-      # We record the query_length as it will help in understanding whether unparseable
-      # queries are truncated.
       record = {
+        'query' => row['query'],
         'queryid' => row['queryid'].to_s,
-        'query_length' => query&.length,
         'calls' => row['calls']&.to_i,
         'total_time_ms' => row['total_time']&.to_f,
         'rows' => row['rows']&.to_i
       }
 
-      return record unless query
-
-      normalized = PgQuery.normalize(query)
-      record['query'] = normalized
-
-      record[@fingerprint_key] = PgQuery.fingerprint(normalized) if @fingerprint_key
-
-      record
-    rescue PgQuery::ParseError
-      record['query_unparseable'] = true
+      opts = {
+        input_key: 'query',
+        output_key: 'query',
+        fingerprint_key: @fingerprint_key,
+        max_length: @max_length
+      }
 
-      record
+      normalize_and_fingerprint_query(record, opts)
     end
 
     # Query the database and emit statements to fluentd router

data/lib/fluent/plugin/query_normalizer.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'pg_query'
+
+module Fluent::Plugin
+  module QueryNormalizer
+    def normalize_and_fingerprint_query(record, opts)
+      input_key = opts[:input_key]
+      output_key = opts[:output_key]
+      fingerprint_key = opts[:fingerprint_key]
+      max_length = opts[:max_length]
+      statement = record[input_key]
+
+      return record unless statement
+
+      # We record the query_length as it will help in understanding whether unparseable
+      # queries are truncated.
+      record['query_length'] = statement&.length
+      normalized = PgQuery.normalize(statement)
+
+      record.delete(input_key)
+      record[fingerprint_key] = PgQuery.fingerprint(normalized) if fingerprint_key
+      record[output_key] = truncate_query(record, normalized, max_length)
+
+      record
+    rescue PgQuery::ParseError
+      record['query_unparseable'] = true
+      record
+    end
+
+    private
+
+    def truncate_query(record, normalized, max_length)
+      return normalized if normalized.length < max_length
+
+      record['truncated_query'] = true
+      # Assume UTF-8 encoding for PostgreSQL queries
+      utf8_query = normalized.encode('UTF-8', invalid: :replace, undef: :replace, replace: '')
+      utf8_query[0, max_length]
+    end
+  end
+end

data/test/plugin/test_filter_postgresql_redactor.rb

@@ -27,7 +27,7 @@ class PostgreSQLRedactorTest < Test::Unit::TestCase
       inputs.each { |input| d.feed(input) }
     end
 
-    assert_equal(%w[duration_s fingerprint message sql], d.filtered[0].last.keys.sort)
+    assert_equal(%w[duration_s fingerprint message query_length sql], d.filtered[0].last.keys.sort)
     assert_equal('SELECT * FROM projects WHERE id = $1', d.filtered[0].last['sql'])
   end
 
@@ -47,7 +47,7 @@ class PostgreSQLRedactorTest < Test::Unit::TestCase
       inputs.each { |input| d.feed(input) }
     end
 
-    assert_equal(%w[duration_s fingerprint message sql truncated_query], d.filtered[0].last.keys.sort)
+    assert_equal(%w[duration_s fingerprint message query_length sql truncated_query], d.filtered[0].last.keys.sort)
     assert_equal('SELECT * F', d.filtered[0].last['sql'])
   end
 
@@ -60,7 +60,7 @@ class PostgreSQLRedactorTest < Test::Unit::TestCase
       d.feed(input)
     end
 
-    assert_equal(%w[pg_query_error query], d.filtered[0].last.keys.sort)
+    assert_equal(%w[query query_length query_unparseable], d.filtered[0].last.keys.sort)
     assert_equal(input['query'], d.filtered[0].last['query'])
   end
 
@@ -83,7 +83,7 @@ class PostgreSQLRedactorTest < Test::Unit::TestCase
       inputs.each { |input| d.feed(input) }
     end
 
-    assert_equal(%w[duration_s fingerprint message out_sql], d.filtered[0].last.keys.sort)
+    assert_equal(%w[duration_s fingerprint message out_sql query_length], d.filtered[0].last.keys.sort)
     assert_equal('SELECT * FROM projects WHERE id = $1', d.filtered[0].last['out_sql'])
   end
 end

data/test/plugin/test_in_pg_stat_activity.rb

@@ -109,7 +109,6 @@ class PgStatActivityInputTest < Test::Unit::TestCase
         'pid' => nil,
         'query' => nil,
         'query_age_s' => nil,
-        'query_length' => nil,
         'query_start' => nil,
         'state' => nil,
         'state_age_s' => nil,
@@ -159,7 +158,7 @@ class PgStatActivityInputTest < Test::Unit::TestCase
         'datid' => 16384,
         'datname' => 'testuser',
         'pid' => 376,
-        'query' => nil,
+        'query' => "SELECT * FROM users WHERE user_se=",
         'query_age_s' => 0.001894,
         'query_length' => 34,
         'query_start' => '2021-07-23T12:55:25.000+00:00',

data/test/plugin/test_in_pg_stat_statements.rb

@@ -77,8 +77,8 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
       record = d.instance.record_for_row(row)
 
       expected = {
-        'query_length' => nil,
         'queryid' => '1234',
+        'query' => nil,
         'calls' => nil,
         'rows' => nil,
         'total_time_ms' => nil
@@ -153,6 +153,7 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
       expected = {
         'query_length' => 13,
         'query_unparseable' => true,
+        'query' => 'SELECT * FROM',
         'queryid' => '1234',
         'calls' => nil,
         'rows' => nil,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-postgresql-csvlog
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.11.0
 platform: ruby
 authors:
 - stanhu
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-10-07 00:00:00.000000000 Z
+date: 2024-10-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -111,6 +111,7 @@ files:
 - lib/fluent/plugin/marginalia_extractor.rb
 - lib/fluent/plugin/parser_multiline_csv.rb
 - lib/fluent/plugin/polling_pg_input_plugin.rb
+- lib/fluent/plugin/query_normalizer.rb
 - sql/create_extension.sql
 - test/helper.rb
 - test/plugin/itest_in_pg_stat_activity.rb