fluent-plugin-postgresql-csvlog 0.3.2 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5d61fc31718e43c6d1dff46139a1b03d56384405d45b69c9772dc6f7b6a66dbf
-  data.tar.gz: 5d2d23a4a7b5f277b19f181aa515ab74718f6ef93824e2c958264d9bbdf9c9aa
+  metadata.gz: 38090772bf1e0302ecd53d759d8da4ffb2df0cdf0a6f001ec0e0d12390472276
+  data.tar.gz: 22bc8479b81f0b0fd7615cfb6cc1b491b317fc74d612f21373243a516e671be4
 SHA512:
-  metadata.gz: f2106f60749b6fa8fc931ccd3d85f51595b0ec60eb9f425935072b1749ed068e8a068c6d513f1557a20e8c5a0613acad80c6690c7ce2aa20cca3027d686c388a
-  data.tar.gz: 9ceef623cbd5256e047dea817d487d2711cc8b387ff068a10913100fe649e77174a600ac019b4729403390152c1a3e6f162accf4cc50f020561032b6493d965f
+  metadata.gz: c3ecfef7d3290ab00a71613c7914c0aae1d05501d37574614d9d6f4a2f5de9548474ecb7b6fdad4ef778c4ea48510a1e31a89294e317ecf7ab6004e094579cc9
+  data.tar.gz: 2b91a1704d53f07ab5139bc2d623c0c17afde53b416ef5f018dd7e6d345ce8a8da1d0aa8964bcef638b8e6ab351b2d7c16b83be73cbdcfba560dda69962bdd17

data/.gitignore

@@ -0,0 +1 @@
+Gemfile.lock

data/.gitlab-ci.yml

@@ -10,10 +10,9 @@ test:
     paths:
       - vendor/ruby
 
-# integration tests
-itest:
+.iteration_test:
   services:
-    - name: postgres:12
+    - name: postgres:$POSTGRES_SERVER_VERSION
       alias: postgres
       command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all"]
   variables:
@@ -27,3 +26,28 @@ itest:
   cache:
     paths:
       - vendor/ruby
+
+# integration tests for postgres 12
+itest_pg12:
+  extends: .iteration_test
+  variables:
+    POSTGRES_SERVER_VERSION: 12
+
+# integration tests for postgres 13
+itest_pg13:
+  extends: .iteration_test
+  variables:
+    POSTGRES_SERVER_VERSION: 13
+
+end_to_end_verification_test:
+  image: docker:19.03.12
+  services:
+    - docker:19.03.12-dind
+  tags:
+    - gitlab-org-docker
+  variables:
+    DOCKER_TLS_CERTDIR: ""
+  before_script:
+    - apk add --no-cache docker-compose
+  script:
+    - docker-compose run --rm verifier

data/README.md

@@ -7,6 +7,8 @@ parse PostgreSQL CSV log files and extract slow log information:
 - `PostgreSQLSlowLog`: Extracts slow log entries into `duration_s` and `statement` fields
 - `PostgreSQLRedactor`: Normalizes the SQL query and redacts sensitive information
 - `Marginalia`: Parses [Marginalia comments](https://github.com/basecamp/marginalia) into key-value pairs and stores them
+- `PgStatStatementsInput`: polls the [`pg_stat_statements`](https://www.postgresql.org/docs/current/pgstatstatements.html) postgres plugin and emits fluentd events.
+- `PgStatActivityInput`: polls the [`postges activity monitor`](https://www.postgresql.org/docs/current/monitoring-stats.html) and emits fluentd events.
 
 ## Installation
 
@@ -72,3 +74,18 @@ ingest and parse PostgreSQL CSV logs:
   </format>
 </match>
 ```
+
+## Developing `fluent-plugin-postgresql-csvlog`
+
+To develop and debug locally, there is a `Dockerfile` and `docker-compose.yml` that will setup a local environment,
+complete with Postgres, suitable for testing purposes.
+
+1. `docker compose build` - build the current configuration
+1. `docker compose run --rm verifier` - test the current configuration
+1. `docker compose up`
+
+### Releasing a new version
+
+1. Update the version in `fluent-plugin-postgresql-csvlog.gemspec`.
+1. Create a merge request and merge the changes to `master`.
+1. Run `bundle exec rake release`.

data/docker-compose.yml

@@ -1,14 +1,9 @@
 # Docker Compose setup useful for testing and development purposes
-version: "3.9"
+version: "3.3"
 services:
-  fluentd:
-    build: .
-    links:
-      - postgres
-    entrypoint: /usr/bin/fluentd -vvv -c /src/example-fluentd.conf
   postgres:
-    image: postgres
-    restart: always
+    image: postgres:13
+    restart: "no"
     environment:
       - POSTGRES_USER=testuser
       - POSTGRES_PASSWORD=testpass
@@ -17,3 +12,26 @@ services:
     command: postgres -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all
     volumes:
       - ./sql/create_extension.sql:/docker-entrypoint-initdb.d/create_extension.sql
+
+  fluentd:
+    build: .
+    restart: "no"
+    links:
+      - postgres
+    entrypoint: /usr/bin/fluentd -vvv -c /src/example-fluentd.conf
+    volumes:
+      - ./example-fluentd.conf:/src/example-fluentd.conf
+      - log-volume:/var/log/pg/
+
+  verifier:
+    image: alpine:3.13
+    restart: "no"
+    links:
+      - fluentd
+    command: /bin/sh /src/verify-docker-compose.sh
+    volumes:
+      - ./test/verify-docker-compose.sh:/src/verify-docker-compose.sh
+      - log-volume:/var/log/pg/
+
+volumes:
+  log-volume:

data/example-fluentd.conf

@@ -8,5 +8,34 @@
 </source>
 
 <match postgres.pg_stat_statements>
-  @type stdout
+  @type file
+  path /var/log/pg/pg_stat_statements
+  time_slice_format %Y%m%d%H%M%S
+  flush_interval 1s
+  utc
+
+  <format>
+    @type json
+  </format>
 </match>
+
+<source>
+  @type pg_stat_activity
+  tag postgres.pg_stat_activity
+  host postgres
+  username testuser
+  password testpass
+  interval 1
+</source>
+
+<match postgres.pg_stat_activity>
+  @type file
+  path /var/log/pg/pg_stat_activity
+  time_slice_format %Y%m%d%H%M%S
+  flush_interval 1s
+  utc
+  <format>
+    @type json
+  </format>
+</match>
+

data/fluent-plugin-postgresql-csvlog.gemspec

@@ -2,7 +2,7 @@ $:.push File.expand_path('lib', __dir__)
 
 Gem::Specification.new do |s|
   s.name        = 'fluent-plugin-postgresql-csvlog'
-  s.version     = '0.3.2'
+  s.version     = '0.7.0'
   s.authors     = ['stanhu']
   s.email       = ['stanhu@gmail.com']
   s.homepage    = 'https://gitlab.com/gitlab-org/fluent-plugins/fluent-plugin-postgresql-csvlog'

data/lib/fluent/plugin/filter_marginalia.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'fluent/plugin/filter'
+require_relative './marginalia_extractor'
 
 module Fluent
   module Plugin
@@ -11,6 +12,7 @@ module Fluent
     # /*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/ SELECT COUNT(*) FROM "projects"
     #
     class Marginalia < Filter
+      include MarginaliaExtractor
       Fluent::Plugin.register_filter('marginalia', self)
 
       desc 'Field to parse for Marginalia comments (key1:value1,key2:value2)'
@@ -19,77 +21,11 @@ module Fluent
       desc 'Whether to strip the comment from the record specified by key'
       config_param :strip_comment, :bool, default: true
 
-      MARGINALIA_PREPENDED_REGEXP = %r{^(?<comment>/\*.*\*/)(?<sql>.*)}m.freeze
-      MARGINALIA_APPENDED_REGEXP = %r{(?<sql>.*)(?<comment>/\*.*\*/)$}m.freeze
-
       def filter(_tag, _time, record)
-        parse_comments(record)
+        parse_marginalia_into_record(record, @key, @strip_comment)
 
         record
       end
-
-      private
-
-      def parse_comments(record)
-        sql = record[@key]
-
-        return unless sql
-
-        comment_match = match_marginalia_comment(sql)
-
-        return unless comment_match
-
-        entries = extract_entries(comment_match['comment'])
-        parse_entries(entries, record)
-
-        record[@key] = comment_match['sql'].strip if @strip_comment
-      end
-
-      def match_marginalia_comment(sql)
-        matched = MARGINALIA_PREPENDED_REGEXP.match(sql)
-
-        return matched if matched
-
-        MARGINALIA_APPENDED_REGEXP.match(sql)
-      end
-
-      def extract_entries(comment)
-        comment = scrub_comment(comment)
-
-        return [] unless comment
-
-        comment.split(',')
-      end
-
-      def scrub_comment(comment)
-        return unless comment
-
-        comment.strip!
-        comment.gsub!(%r{^/\*}, '')
-        comment.gsub!(%r{\*/$}, '')
-      end
-
-      def parse_entries(entries, record)
-        entries.each do |component|
-          data = component.split(':', 2)
-
-          break unless data.length == 2
-
-          stored_key = store_key(record, data[0])
-          record[stored_key] = data[1]
-        end
-      end
-
-      def store_key(record, component_key)
-        # In case there is a conflict with the Marginalia key
-        # (e.g. `correlation_id`), we use the base key
-        # (`sql_correlation_id`) instead.
-        if record.key?(component_key)
-          "#{@key}_#{component_key}"
-        else
-          component_key
-        end
-      end
     end
   end
 end

data/lib/fluent/plugin/in_pg_stat_activity.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require_relative './polling_pg_input_plugin'
+require 'pg_query'
+require_relative './marginalia_extractor'
+require 'time'
+
+module Fluent::Plugin
+  # PgStatActivityInput polls the `pg_stat_activity` table
+  # emitting normalized versions of the queries currently running on
+  # the postgres server.
+  # Fingerprints of the queries are also included for easier aggregation
+  class PgStatActivityInput < PollingPostgresInputPlugin
+    include MarginaliaExtractor
+    Fluent::Plugin.register_input('pg_stat_activity', self)
+
+    ACTIVITY_QUERY = <<-SQL
+      SELECT
+        datid,
+        datname,
+        pid,
+        usesysid,
+        usename,
+        application_name,
+        host(client_addr) as client_addr,
+        client_hostname,
+        client_port,
+        xact_start,
+        extract(epoch from clock_timestamp() - xact_start) xact_age_s,
+        query_start,
+        extract(epoch from clock_timestamp() - query_start) query_age_s,
+        state_change,
+        extract(epoch from clock_timestamp() - state_change) state_age_s,
+        state,
+        query
+      FROM pg_stat_activity
+      WHERE usename IS NOT NULL
+    SQL
+
+    desc 'Name of field to store SQL query fingerprint'
+    config_param :fingerprint_key, :string, default: 'fingerprint'
+
+    protected
+
+    def on_poll
+      with_connection do |conn|
+        emit_activity_to_stream(conn)
+      end
+    end
+
+    public
+
+    # Query the database and emit statements to fluentd router
+    def emit_activity_to_stream(conn)
+      me = Fluent::MultiEventStream.new
+
+      now = Fluent::Engine.now
+      conn.exec(ACTIVITY_QUERY).each do |row|
+        record = record_for_row(row)
+        me.add(now, record)
+      end
+
+      @router.emit_stream(@tag, me)
+    end
+
+    # Returns a fluentd record for a query row
+    def record_for_row(row)
+      record = {
+        'datid' => row['datid'],
+        'datname' => row['datname'],
+        'pid' => row['pid'],
+        'usesysid' => row['usesysid'],
+        'usename' => row['usename'],
+        'application_name' => row['application_name'],
+        'client_addr' => row['client_addr'],
+        'client_hostname' => row['client_hostname'],
+        'client_port' => row['client_port'],
+        'xact_start' => row['xact_start']&.iso8601(3),
+        'xact_age_s' => row['xact_age_s'],
+        'query_start' => row['query_start']&.iso8601(3),
+        'query_age_s' => row['query_age_s'],
+        'state_change' => row['state_change']&.iso8601(3),
+        'state_age_s' => row['state_age_s'],
+        'state' => row['state'],
+        'query' => row['query'] # This will be stripped, normalized etc
+      }
+
+      # Inject marginalia into record
+      parse_marginalia_into_record(record, 'query', true)
+
+      # Normalize query and fingerprint
+      # Note that `record['query']` was updated in previous step
+      # To strip off marginalia comments
+      record.merge!(fingerprint_query(record['query']))
+
+      record
+    end
+
+    def fingerprint_query(query)
+      # We record the query_length as it will help in understanding whether unparseable
+      # queries are truncated.
+      record = { 'query_length' => query&.length, 'query' => nil }
+
+      return record unless query
+
+      normalized = PgQuery.normalize(query)
+      record['query'] = normalized
+
+      record[@fingerprint_key] = PgQuery.parse(normalized).fingerprint if @fingerprint_key
+
+      record
+    rescue PgQuery::ParseError
+      record['query_unparseable'] = true
+
+      record
+    end
+  end
+end

data/lib/fluent/plugin/in_pg_stat_statements.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
-require 'fluent/plugin/input'
-require 'pg'
+require_relative './polling_pg_input_plugin'
 require 'pg_query'
 
 module Fluent::Plugin
@@ -12,67 +11,47 @@ module Fluent::Plugin
   #  'fingerprint' => '8a6e9896bd9048a2',
   #  'query' => 'SELECT * FROM table ORDER BY queryid LIMIT $1',
   #  'query_length' => 58,
-  #  'queryid' => 3239318621761098074
+  #  'queryid' => '3239318621761098074'
   # }
-  class PgStatStatementsInput < Input
+  class PgStatStatementsInput < PollingPostgresInputPlugin
     Fluent::Plugin.register_input('pg_stat_statements', self)
 
-    desc 'PostgreSQL host'
-    config_param :host, :string
-
-    desc 'RDBMS port (default: 5432)'
-    config_param :port, :integer, default: 5432
-
-    desc 'login user name'
-    config_param :username, :string, default: nil
-
-    desc 'postgres db'
-    config_param :dbname, :string, default: nil
-
-    desc 'login password'
-    config_param :password, :string, default: nil, secret: true
-
-    # See https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLMODE
-    # for options
-    desc 'postgres sslmode'
-    config_param :sslmode, :string, default: 'prefer'
-
-    desc 'tag'
-    config_param :tag, :string, default: nil
-
-    desc 'interval in second to run query'
-    config_param :interval, :time, default: 300
-
     desc 'Name of field to store SQL query fingerprint'
     config_param :fingerprint_key, :string, default: 'fingerprint'
 
-    def start
-      @stop_flag = false
-      @thread = Thread.new(&method(:thread_main))
+    POSTGRES_SERVER_VERSION_QUERY = "SELECT current_setting('server_version_num')"
+
+    PG12_STAT_STATEMENTS_QUERY = <<-SQL
+      SELECT queryid,
+             query,
+             calls,
+             rows,
+             total_time
+        FROM public.pg_stat_statements
+    SQL
+
+    PG13_STAT_STATEMENTS_QUERY = <<-SQL
+      SELECT queryid,
+             query,
+             calls,
+             rows,
+             (total_plan_time + total_exec_time) total_time
+        FROM public.pg_stat_statements
+    SQL
+
+    protected
+
+    def on_poll
+      with_connection do |conn|
+        emit_statements_to_stream(conn)
+      end
     end
 
-    def shutdown
-      @stop_flag = true
-
-      # Interrupt thread and wait for it to finish
-      Thread.new { @thread.run } if @thread
-      @thread.join
-    end
+    public
 
-    def thread_main
-      until @stop_flag
-        sleep @interval
-        break if @stop_flag
-
-        begin
-          with_connection do |conn|
-            emit_statements_to_stream(conn)
-          end
-        rescue StandardError => e
-          log.error 'unexpected error', error: e.message, error_class: e.class
-          log.error_backtrace e.backtrace
-        end
-      end
+    def initialize
+      super
+      @postgres_server_version_num = nil
     end
 
     # Returns a fluentd record for a query row
@@ -81,7 +60,13 @@ module Fluent::Plugin
 
       # We record the query_length as it will help in understanding whether unparseable
       # queries are truncated.
-      record = { 'queryid' => row['queryid'], 'query_length' => query&.length }
+      record = {
+        'queryid' => row['queryid'].to_s,
+        'query_length' => query&.length,
+        'calls' => row['calls']&.to_i,
+        'total_time_ms' => row['total_time']&.to_f,
+        'rows' => row['rows']&.to_i
+      }
 
       return record unless query
 
@@ -97,14 +82,15 @@ module Fluent::Plugin
       record
     end
 
-    private
-
     # Query the database and emit statements to fluentd router
     def emit_statements_to_stream(conn)
       me = Fluent::MultiEventStream.new
 
       now = Fluent::Engine.now
-      conn.exec('SELECT queryid, query FROM public.pg_stat_statements').each do |row|
+
+      query = query_for_postgres_version(conn)
+
+      conn.exec(query).each do |row|
         record = record_for_row(row)
         me.add(now, record)
       end
@@ -112,25 +98,21 @@ module Fluent::Plugin
       @router.emit_stream(@tag, me)
     end
 
-    # Since this query is very infrequent, and it may be communicating directly
-    # with postgres without pgbouncer, don't use a persistent connection and
-    # ensure that it is properly closed
-    def with_connection(&block)
-      conn = PG.connect(
-        host: @host,
-        dbname: @dbname,
-        sslmode: @sslmode,
-        user: @username,
-        password: @password
-      )
-      conn.type_map_for_results = PG::BasicTypeMapForResults.new conn
-
-      begin
-        block.call(conn)
-      ensure
-        # Always close the connection
-        conn.finish
-      end
+    # Returns the PG_VERSION_NUM value from the database
+    # will memoize the result
+    def postgres_server_version_num(conn)
+      return @postgres_server_version_num if @postgres_server_version_num
+
+      @postgres_server_version_num = conn.exec(POSTGRES_SERVER_VERSION_QUERY).getvalue(0,0).to_i
+    end
+
+    # pg_stat_statements columns changed in pg13, so we use different queries depending on the version
+    # https://www.postgresql.org/docs/12/pgstatstatements.html
+    # https://www.postgresql.org/docs/13/pgstatstatements.html
+    def query_for_postgres_version(conn)
+      return PG13_STAT_STATEMENTS_QUERY if postgres_server_version_num(conn) >= 13_00_00
+
+      PG12_STAT_STATEMENTS_QUERY
     end
   end
 end

data/lib/fluent/plugin/marginalia_extractor.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require 'fluent/plugin/filter'
+
+module Fluent::Plugin
+  # MarginaliaExtractor provides the parse_marginalia_into_record
+  # utility method, useful for extracting marginalia into fluentd records
+  module MarginaliaExtractor
+    MARGINALIA_PREPENDED_REGEXP = %r{^(?<comment>/\*.*\*/)(?<sql>.*)}m.freeze
+    MARGINALIA_APPENDED_REGEXP = %r{(?<sql>.*)(?<comment>/\*.*\*/)$}m.freeze
+
+    # Injects marginalia into a fluentd record
+    def parse_marginalia_into_record(record, key, strip_comment)
+      sql = record[key]
+      return unless sql
+
+      comment_match = match_marginalia_comment(sql)
+
+      return unless comment_match
+
+      entries = extract_entries(comment_match['comment'])
+      parse_entries(entries, key, record)
+
+      record[key] = comment_match['sql'].strip if strip_comment
+    end
+
+    def match_marginalia_comment(sql)
+      matched = MARGINALIA_PREPENDED_REGEXP.match(sql)
+
+      return matched if matched
+
+      MARGINALIA_APPENDED_REGEXP.match(sql)
+    end
+
+    def extract_entries(comment)
+      comment = scrub_comment(comment)
+
+      return [] unless comment
+
+      comment.split(',')
+    end
+
+    def scrub_comment(comment)
+      return unless comment
+
+      comment.strip!
+      comment.gsub!(%r{^/\*}, '')
+      comment.gsub!(%r{\*/$}, '')
+    end
+
+    def parse_entries(entries, key, record)
+      entries.each do |component|
+        data = component.split(':', 2)
+
+        break unless data.length == 2
+
+        stored_key = store_key(record, key, data[0])
+        record[stored_key] = data[1]
+      end
+    end
+
+    def store_key(record, key, component_key)
+      # In case there is a conflict with the Marginalia key
+      # (e.g. `correlation_id`), we use the base key
+      # (`sql_correlation_id`) instead.
+      if record.key?(component_key)
+        "#{key}_#{component_key}"
+      else
+        component_key
+      end
+    end
+  end
+end