fluent-plugin-postgresql-csvlog 0.9.2 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5bbac50b30c3a585cf23b58ae413fa4ff40c664346680548e342c70bff796696
-  data.tar.gz: 25f733282fc1c10704bd8ea0ae2af563f6d569ee2d68e63e3df38b8ea31631f9
+  metadata.gz: 2393ec76502b4127cd874027d99eada9e184dba85edfec3c2a9cfe1a88de54ee
+  data.tar.gz: 2f61b0b0d4e27a302f90449af833dba0e985406d98f15be773ac379146a958d1
 SHA512:
-  metadata.gz: aaaecacfe5ed442dfcec714c29d4e08255bc202b0672f8e4c39e5a0b71303e5ded858b4615085ec3962c7b62d11f576b7c8700758e2217aa15a43dc9a4e491b5
-  data.tar.gz: 3c17fef4a6f58b9203b125aece0b2d9806af8ce1589b1c4b09c44cfcd12f45fc6e3e9bbd30d0934eef7ab26567b44dd767423e87bcf75914500e9acd6aa750e0
+  metadata.gz: b537e7458f9fa38bb0b88672e137a345739f22ab92f8a3d9fe3cf0e26f9e958995c2f7fdb51afc649ac5f55e608ddd431ef904dd757e651f1b223be58703d7d6
+  data.tar.gz: 7724c6961993e7e50ef38ebfa22b874fd4578eb8d8057d2cac76fe761f723a5631e1a227c791c7debf24c2631bc4eea1c2febe929f723a5584c52adbb2b91a49

data/fluent-plugin-postgresql-csvlog.gemspec

@@ -2,7 +2,7 @@ $:.push File.expand_path('lib', __dir__)
 
 Gem::Specification.new do |s|
   s.name        = 'fluent-plugin-postgresql-csvlog'
-  s.version     = '0.9.2'
+  s.version     = '0.10.0'
   s.authors     = ['stanhu']
   s.email       = ['stanhu@gmail.com']
   s.homepage    = 'https://gitlab.com/gitlab-org/fluent-plugins/fluent-plugin-postgresql-csvlog'

data/lib/fluent/plugin/filter_postgresql_redactor.rb

@@ -16,6 +16,9 @@ module Fluent::Plugin
     desc 'Name of field to store SQL query fingerprint'
     config_param :fingerprint_key, :string, default: 'fingerprint'
 
+    desc 'Truncate the query if it exceeds the number of bytes'
+    config_param :max_length, :integer, default: 3 * 1024 * 1024
+
     def filter(_tag, _time, record)
       statement = record[@input_key]
 
@@ -25,12 +28,23 @@ module Fluent::Plugin
       record[@fingerprint_key] = PgQuery.fingerprint(normalized) if @fingerprint_key
 
       record.delete(@input_key)
-      record[@output_key] = normalized
+      record[@output_key] = truncate_query(record, normalized)
 
       record
     rescue PgQuery::ParseError
       record['pg_query_error'] = true
       record
     end
+
+    private
+
+    def truncate_query(record, normalized)
+      return normalized if normalized.length < @max_length
+
+      record['truncated_query'] = true
+      # Assume UTF-8 encoding for PostgreSQL queries
+      utf8_query = normalized.encode('UTF-8', invalid: :replace, undef: :replace, replace: '')
+      utf8_query[0, @max_length]
+    end
   end
 end

data/test/plugin/test_filter_postgresql_redactor.rb

@@ -31,6 +31,26 @@ class PostgreSQLRedactorTest < Test::Unit::TestCase
     assert_equal('SELECT * FROM projects WHERE id = $1', d.filtered[0].last['sql'])
   end
 
+  test 'truncates logs' do
+    d = create_driver(<<~CONF
+      max_length 10
+    CONF
+                     )
+
+    inputs = [
+      { 'message' => 'duration: 2357.1 ms  execute <unnamed>: SELECT * FROM projects WHERE id = 1',
+        'query' => %(SELECT * FROM projects WHERE id = 1),
+        'duration_s' => 2.3571 }
+    ]
+
+    d.run(default_tag: @tag) do
+      inputs.each { |input| d.feed(input) }
+    end
+
+    assert_equal(%w[duration_s fingerprint message sql truncated_query], d.filtered[0].last.keys.sort)
+    assert_equal('SELECT * F', d.filtered[0].last['sql'])
+  end
+
   test 'handles parse errors' do
     d = create_driver
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-postgresql-csvlog
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.10.0
 platform: ruby
 authors:
 - stanhu
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-09-26 00:00:00.000000000 Z
+date: 2024-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd