fluent-plugin-postgresql-csvlog 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1c99895582e6133a97c194d676882f42d0fad0ab52aff6cfe1b3fbbbcf32263a
-  data.tar.gz: 71135c5dddefbf8a61f8f6e3453daddd61df187b082201fb00dff4680ce0bdbe
+  metadata.gz: bb322f50f848a7b196962dae916a75f30a77f1461167bb314494ae203f8ef9d6
+  data.tar.gz: 077266e6c7f3589813d0c4db8ddccfde76f7d6ecd0b6d6388b67a4e6328ad858
 SHA512:
-  metadata.gz: 518a30b929f02f82a6a6d80d03a4a1a9fa66174759c8f9b89a3243a7bf4d8168a43fe08631882630790afea4fb24eed4179215f85d7454c8fc63fc6e2aae1c6e
-  data.tar.gz: 43921a20185c91042274d0acbbd4c70ef94cb7c199675833890ab2cc946aa49065b8c0a624eb6691684e55bfa0945a02e9996f305211657ffae492e856deebe2
+  metadata.gz: 9b130aa84b1285a62466b3bf68ff6d02acc7e95cd53cc91ee7e34ec81ee4440e06358738790f86f8d436cb8a72ad232f12ac2c0f9d8aca16fd5c6ba19bbdfe49
+  data.tar.gz: e9480110be0cadfad86c5fcc554eb5d1d7264559714e4fd88ea396ae9bb913027b15782c70cd2f4c06e1f4cc5a5bbafe55632ca8eba02ce57e610d98f8440c09

data/.gitlab-ci.yml

@@ -9,3 +9,21 @@ test:
   cache:
     paths:
       - vendor/ruby
+
+# integration tests
+itest:
+  services:
+    - name: postgres:12
+      alias: postgres
+      command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all"]
+  variables:
+    POSTGRES_USER: testuser
+    POSTGRES_PASSWORD: testpass
+  before_script:
+    - bundle config set path vendor
+    - bundle install --jobs $(nproc)
+  script:
+    - bundle exec rake itest
+  cache:
+    paths:
+      - vendor/ruby

data/Rakefile

@@ -9,4 +9,10 @@ Rake::TestTask.new(:test) do |test|
   test.verbose = true
 end
 
+Rake::TestTask.new(:itest) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/**/itest_*.rb']
+  test.verbose = true
+end
+
 task :default => [:build]

data/fluent-plugin-postgresql-csvlog.gemspec

@@ -2,10 +2,10 @@ $:.push File.expand_path('lib', __dir__)
 
 Gem::Specification.new do |s|
   s.name        = 'fluent-plugin-postgresql-csvlog'
-  s.version     = '0.2.1'
+  s.version     = '0.3.0'
   s.authors     = ['stanhu']
   s.email       = ['stanhu@gmail.com']
-  s.homepage    = 'https://gitlab.com/gitlab-org/fluent-plugin-postgresql-csvlog'
+  s.homepage    = 'https://gitlab.com/gitlab-org/fluent-plugins/fluent-plugin-postgresql-csvlog'
   s.summary     = 'fluentd plugins to work with PostgreSQL CSV logs'
   s.description = 'fluentd plugins to work with PostgreSQL CSV logs'
 
@@ -15,6 +15,7 @@ Gem::Specification.new do |s|
   s.require_paths = ['lib']
 
   s.add_dependency 'fluentd', ['>= 1.0', '< 2']
+  s.add_dependency 'pg', '~> 1.1'
   s.add_dependency 'pg_query', '~> 2.0'
 
   s.add_development_dependency 'rake'

data/lib/fluent/plugin/input_pg_stat_statements.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+require 'fluent/input'
+require 'pg'
+require 'pg_query'
+
+module Fluent
+  # PgStatStatementsInput will periodically poll postgres, querying pg_stat_statements
+  # for queryid to query mappings. These are then normalized for security purposes
+  # fingerprinted and emitted as records with the following format:
+  # {
+  #  'fingerprint' => '8a6e9896bd9048a2',
+  #  'query' => 'SELECT * FROM table ORDER BY queryid LIMIT $1',
+  #  'query_length' => 58,
+  #  'queryid' => 3239318621761098074
+  # }
+  class PgStatStatementsInput < Input
+    Fluent::Plugin.register_input('pg_stat_statements', self)
+
+    desc 'PostgreSQL host'
+    config_param :host, :string
+
+    desc 'RDBMS port (default: 5432)'
+    config_param :port, :integer, default: 5432
+
+    desc 'login user name'
+    config_param :username, :string, default: nil
+
+    desc 'postgres db'
+    config_param :dbname, :string, default: nil
+
+    desc 'login password'
+    config_param :password, :string, default: nil, secret: true
+
+    # See https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLMODE
+    # for options
+    desc 'postgres sslmode'
+    config_param :sslmode, :string, default: 'prefer'
+
+    desc 'tag'
+    config_param :tag, :string, default: nil
+
+    desc 'interval in second to run query'
+    config_param :interval, :time, default: 300
+
+    desc 'Name of field to store SQL query fingerprint'
+    config_param :fingerprint_key, :string, default: 'fingerprint'
+
+    def start
+      @stop_flag = false
+      @thread = Thread.new(&method(:thread_main))
+    end
+
+    def shutdown
+      @stop_flag = true
+
+      # Interrupt thread and wait for it to finish
+      Thread.new { @thread.run } if @thread
+      @thread.join
+    end
+
+    def thread_main
+      until @stop_flag
+        sleep @interval
+        break if @stop_flag
+
+        begin
+          with_connection do |conn|
+            emit_statements_to_stream(conn)
+          end
+        rescue StandardError => e
+          log.error 'unexpected error', error: e.message, error_class: e.class
+          log.error_backtrace e.backtrace
+        end
+      end
+    end
+
+    # Returns a fluentd record for a query row
+    def record_for_row(row)
+      query = row['query']
+
+      # We record the query_length as it will help in understanding whether unparseable
+      # queries are truncated.
+      record = { 'queryid' => row['queryid'], 'query_length' => query&.length }
+
+      return record unless query
+
+      normalized = PgQuery.normalize(query)
+      record['query'] = normalized
+
+      record[@fingerprint_key] = PgQuery.parse(normalized).fingerprint if @fingerprint_key
+
+      record
+    rescue PgQuery::ParseError
+      record['query_unparseable'] = true
+
+      record
+    end
+
+    private
+
+    # Query the database and emit statements to fluentd router
+    def emit_statements_to_stream(conn)
+      me = MultiEventStream.new
+
+      now = Engine.now
+      conn.exec('SELECT queryid, query FROM pg_stat_statements').each do |row|
+        record = record_for_row(row)
+        me.add(now, record)
+      end
+
+      @router.emit_stream(@tag, me)
+    end
+
+    # Since this query is very infrequent, and it may be communicating directly
+    # with postgres without pgbouncer, don't use a persistent connection and
+    # ensure that it is properly closed
+    def with_connection(&block)
+      conn = PG.connect(
+        host: @host,
+        dbname: @dbname,
+        sslmode: @sslmode,
+        user: @username,
+        password: @password
+      )
+      conn.type_map_for_results = PG::BasicTypeMapForResults.new conn
+
+      begin
+        block.call(conn)
+      ensure
+        # Always close the connection
+        conn.finish
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -7,10 +7,13 @@ $LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
 $LOAD_PATH.unshift(__dir__)
 require 'fluent/test'
 require 'fluent/test/driver/filter'
+require 'fluent/test/driver/input'
 require 'fluent/test/helpers'
 
 Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)
 
 require 'fluent/plugin/filter_postgresql_slowlog'
 require 'fluent/plugin/filter_postgresql_redactor'
 require 'fluent/plugin/filter_marginalia'
+require 'fluent/plugin/input_pg_stat_statements'

data/test/plugin/itest_input_pg_stat_statements.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require_relative '../helper'
+
+class PgStatStatementsInputIntegrationTest < Test::Unit::TestCase
+  # These items are configured in .gitlab-ci.yml on the postgres service
+  HOST = 'postgres'
+  USERNAME = 'testuser'
+  PASSWORD = 'testpass'
+
+  def setup
+    Fluent::Test.setup
+
+    @conn = PG.connect(
+      host: HOST,
+      user: USERNAME,
+      password: PASSWORD
+    )
+
+    try_setup_extension
+    create_known_statement
+  end
+
+  def teardown
+    @conn&.finish
+  end
+
+  # Setup pg_stat_statements extension
+  def try_setup_extension
+    @conn.exec('CREATE EXTENSION pg_stat_statements')
+  rescue PG::DuplicateObject
+  end
+
+  # This statement gives us something to look for in the emitted stream
+  def create_known_statement
+    @conn.exec('SELECT * FROM pg_stat_statements ORDER BY queryid LIMIT 1')
+  end
+
+  VALID_CONFIG = %(
+    tag postgres.pg_stat_statements
+    host #{HOST}
+    username #{USERNAME}
+    password #{PASSWORD}
+    interval 1
+  )
+
+  INVALID_CONFIG = %(
+    host 'invalid_host.dne'
+    port 1234
+    username #{USERNAME}
+    password #{PASSWORD}
+    interval 1
+  )
+
+  def create_driver(config)
+    Fluent::Test::InputTestDriver.new(Fluent::PgStatStatementsInput).configure(config)
+  end
+
+  sub_test_case 'configuration' do
+    test 'connects' do
+      d = create_driver(VALID_CONFIG)
+
+      emits = []
+      # wait 50 * 0.05, "see fluentd/lib/fluent/test/base.rb:79 num_waits.times { sleep 0.05 }
+      d.run(num_waits = 50) do
+        emits = d.emits
+      end
+
+      assert_false emits.empty?
+    end
+
+    # Why do we have this test? If postgres is still starting up, we don't want to cause the
+    # the fluentd configuration to fail. We would rather retry until we get a connection
+    test 'connects for an invalid config' do
+      d = create_driver(INVALID_CONFIG)
+
+      emits = []
+      # wait 50 * 0.05, "see fluentd/lib/fluent/test/base.rb:79 num_waits.times { sleep 0.05 }
+      d.run(num_waits = 50) do
+        emits = d.emits
+      end
+
+      assert_true emits.empty?
+    end
+  end
+
+  sub_test_case 'execution' do
+    test 'connects' do
+      d = create_driver(VALID_CONFIG)
+
+      emits = []
+      # wait 50 * 0.05, "see fluentd/lib/fluent/test/base.rb:79 num_waits.times { sleep 0.05 }
+      d.run(num_waits = 50) do
+        emits = d.emits
+      end
+
+      expected_record = {
+        'fingerprint' => '8a6e9896bd9048a2',
+        'query' => 'SELECT * FROM pg_stat_statements ORDER BY queryid LIMIT $1',
+        'query_length' => 58,
+        'queryid' => 3_239_318_621_761_098_074
+      }
+      known_statement_event = emits.find do |event|
+        record = event[2]
+        record['query'] == expected_record['query']
+      end
+
+      assert_false known_statement_event.nil?
+
+      tag = known_statement_event[0]
+      record = known_statement_event[2]
+
+      assert_equal 'postgres.pg_stat_statements', tag
+      assert_equal expected_record['fingerprint'], record['fingerprint']
+      assert_equal expected_record['query_length'], record['query_length']
+      assert_true expected_record.include? 'queryid'
+    end
+  end
+end

data/test/plugin/test_input_pg_stat_statements.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require_relative '../helper'
+
+class PgStatStatementsInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %(
+    tag postgres.pg_stat_statements
+    host localhost
+    port 1234
+    dbname gitlab
+    sslmode require
+    username moo
+    password secret
+    interval 600
+    fingerprint_key fingerprint
+  )
+
+  def create_driver
+    Fluent::Test::InputTestDriver.new(Fluent::PgStatStatementsInput).configure(CONFIG)
+  end
+
+  sub_test_case 'configuration' do
+    test 'basic configuration' do
+      d = create_driver
+
+      assert_equal 'postgres.pg_stat_statements', d.instance.tag
+      assert_equal 'localhost', d.instance.host
+      assert_equal 1234, d.instance.port
+      assert_equal 'gitlab', d.instance.dbname
+      assert_equal 'require', d.instance.sslmode
+      assert_equal 'moo', d.instance.username
+      assert_equal 'secret', d.instance.password
+      assert_equal 600, d.instance.interval
+      assert_equal 'fingerprint', d.instance.fingerprint_key
+    end
+  end
+
+  sub_test_case 'execution' do
+    test 'sql' do
+      d = create_driver
+      record = d.instance.record_for_row({ 'queryid' => '1234', 'query' => 'SELECT * FROM users WHERE user_id = ?' })
+
+      expected = {
+        'fingerprint' => 'c071dee80d466e7d',
+        'query' => 'SELECT * FROM users WHERE user_id = ?',
+        'query_length' => 37,
+        'queryid' => '1234'
+      }
+
+      assert_equal expected, record
+    end
+
+    test 'nil query' do
+      d = create_driver
+      record = d.instance.record_for_row({ 'queryid' => '1234', 'query' => nil })
+
+      expected = { 'query_length' => nil, 'queryid' => '1234' }
+      assert_equal expected, record
+    end
+
+    test 'ddl query' do
+      d = create_driver
+      ddl_sql = <<-SQL
+        CREATE TABLE accounts (
+          user_id serial PRIMARY KEY,
+          username VARCHAR(50) UNIQUE NOT NULL,
+          password VARCHAR(50) NOT NULL,
+          email VARCHAR(255) UNIQUE NOT NULL,
+          created_on TIMESTAMP NOT NULL,
+          last_login TIMESTAMP
+        )
+      SQL
+
+      record = d.instance.record_for_row({ 'queryid' => '1234', 'query' => ddl_sql })
+
+      expected = {
+        'fingerprint' => 'fa9c9d26757c4f9b',
+        'query' => ddl_sql,
+        'query_length' => 287,
+        'queryid' => '1234'
+      }
+      assert_equal expected, record
+    end
+
+    test 'set command' do
+      d = create_driver
+      record = d.instance.record_for_row({ 'queryid' => '1234', 'query' => "SET TIME ZONE 'PST8PDT'" })
+
+      expected = {
+        'fingerprint' => '23f8d6eb1d3125c3',
+        'query' => 'SET TIME ZONE $1',
+        'query_length' => 23,
+        'queryid' => '1234'
+      }
+
+      assert_equal expected, record
+    end
+
+    test 'unparseable sql' do
+      d = create_driver
+      record = d.instance.record_for_row({ 'queryid' => '1234', 'query' => 'SELECT * FROM' })
+
+      expected = { 'query_length' => 13, 'query_unparseable' => true, 'queryid' => '1234' }
+      assert_equal expected, record
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-postgresql-csvlog
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - stanhu
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-23 00:00:00.000000000 Z
+date: 2021-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -30,6 +30,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: pg_query
   requirement: !ruby/object:Gem::Requirement
@@ -88,12 +102,15 @@ files:
 - lib/fluent/plugin/filter_marginalia.rb
 - lib/fluent/plugin/filter_postgresql_redactor.rb
 - lib/fluent/plugin/filter_postgresql_slowlog.rb
+- lib/fluent/plugin/input_pg_stat_statements.rb
 - lib/fluent/plugin/parser_multiline_csv.rb
 - test/helper.rb
+- test/plugin/itest_input_pg_stat_statements.rb
 - test/plugin/test_filter_marginalia.rb
 - test/plugin/test_filter_postgresql_redactor.rb
 - test/plugin/test_filter_postgresql_slowlog.rb
-homepage: https://gitlab.com/gitlab-org/fluent-plugin-postgresql-csvlog
+- test/plugin/test_input_pg_stat_statements.rb
+homepage: https://gitlab.com/gitlab-org/fluent-plugins/fluent-plugin-postgresql-csvlog
 licenses: []
 metadata: {}
 post_install_message: 
@@ -117,6 +134,8 @@ specification_version: 4
 summary: fluentd plugins to work with PostgreSQL CSV logs
 test_files:
 - test/helper.rb
+- test/plugin/itest_input_pg_stat_statements.rb
 - test/plugin/test_filter_marginalia.rb
 - test/plugin/test_filter_postgresql_redactor.rb
 - test/plugin/test_filter_postgresql_slowlog.rb
+- test/plugin/test_input_pg_stat_statements.rb