fluent-plugin-postgresql_csv 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 74da41159df683170a38a0dec1389413446a7a9598f0958a787804eef2bdfc4b
+  data.tar.gz: 522819601c6a2eea5f4dc7e54b2880d73069e9e2dc15d2378480e4e46c58cc7c
+SHA512:
+  metadata.gz: f159d28274c110d1320c5e28cb0d92fa6c6ccdc816ad00c7589a90ae1a15563538f8253ec1f8b379b3cdd6b8fc7fb09c2ce25321f489900225a68515994e10fb
+  data.tar.gz: 4f1866eb5c8ff4f71d11fc6b78a6127ffa70773a74cff14edfa82f521a10b3e280bef68d600aefb75faf057b9ce0dcfc296da752f9829cb311aa0938f7888015

data/.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Rakefile

@@ -0,0 +1,3 @@
+require 'bundler/gem_tasks'
+
+task :default => :test

data/fluent-plugin-postgresql-csv.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-postgresql_csv"
+  gem.version       = "0.0.3"
+  gem.summary       = %q{Fluentd filter plugin for PostgreSQL logs in CSV format}
+  gem.authors       = ["Oleg Gurov"]
+  gem.homepage      = "https://github.com/masterlee998/"
+  gem.license       = 'MIT'
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.has_rdoc      = false
+
+  gem.required_ruby_version = ['>= 2.1.0', '< 2.5.0']
+  gem.add_runtime_dependency 'fluentd', '>= 0.14.0'
+  gem.add_development_dependency 'bundler', '~> 1.3'
+  gem.add_development_dependency 'test-unit', '~> 3.2.8'
+  gem.add_development_dependency 'rake'
+end

data/lib/fluent/plugin/filter_postgresql_csv.rb

@@ -0,0 +1,69 @@
+require 'fluent/plugin/filter'
+
+module Fluent::Plugin
+class PostgreSQL_CSV_Filter < Filter
+  Fluent::Plugin.register_filter('postgresql_csv', self)
+
+  def initialize
+    super
+    require 'csv'
+    require 'date'
+  end
+
+  def configure(conf)
+    super
+  end
+
+  def filter(tag, time, record)
+    record_orig = record.clone
+    begin
+      csv = CSV.parse(record['log'])[0]
+      if csv.length != 23
+        return record
+      end
+
+      begin
+        DateTime.strptime(csv[0], '%Y-%m-%d %H:%M:%S.%L %Z')
+      rescue ArgumentError
+        return record
+      end
+
+      parsed_value = {}
+      parsed_value['log_time']                 = csv[0]
+      parsed_value['user_name']                = csv[1]
+      parsed_value['database_name']            = csv[2]
+      parsed_value['process_id']               = csv[3]
+      parsed_value['connection_from']          = csv[4]
+      parsed_value['session_id']               = csv[5]
+      parsed_value['session_line_num']         = csv[6]
+      parsed_value['command_tag']              = csv[7]
+      parsed_value['session_start_time']       = csv[8]
+      parsed_value['virtual_transaction_id']   = csv[9]
+      parsed_value['transaction_id']           = csv[10]
+      parsed_value['error_severity']           = csv[11]
+      parsed_value['sql_state_code']           = csv[12]
+      parsed_value['message']                  = csv[13]
+      parsed_value['detail']                   = csv[14]
+      parsed_value['hint']                     = csv[15]
+      parsed_value['internal_query']           = csv[16]
+      parsed_value['internal_query_pos']       = csv[17]
+      parsed_value['context']                  = csv[18]
+      parsed_value['query']                    = csv[19]
+      parsed_value['query_pos']                = csv[20]
+      parsed_value['location']                 = csv[21]
+      parsed_value['application_name']         = csv[22]
+
+      duration = parsed_value['message'].scan(/duration: (\d+(?:\.\d+)?) ms  statement: (.+)/m)[0]
+      if duration != nil and duration.kind_of?(Array)
+        parsed_value['duration'] = duration[0]
+        #parsed_value['duration'] = duration[0].to_f
+      end
+
+      record['postgresql'] = parsed_value
+    rescue CSV::MalformedCSVError
+      return record_orig
+    end
+    return record
+  end
+end
+end

data/test/helper.rb

@@ -0,0 +1,27 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require "fluent/test"
+require "fluent/test/driver/filter"
+require "fluent/test/helpers"
+
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+require 'fluent/plugin/filter_postgresql_csv'

data/test/plugin/test_filter_postgresql_csv.rb

@@ -0,0 +1,133 @@
+require_relative '../helper'
+
+class PostgreSQL_CSV_Filter < Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    dummy_param dummy_value
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::PostgreSQL_CSV_Filter).configure(conf)
+  end
+
+  def filter(config, messages)
+    d = create_driver(config)
+    d.run(default_tag: "input.access") do
+      messages.each do |message|
+        d.feed(message)
+      end
+    end
+    d.filtered_records
+  end
+
+  sub_test_case 'parse log line and add fields' do
+    test "error message" do
+      conf = CONFIG
+      message = [{ "log" => "2018-10-24 02:59:54.936 MSK,\"root\",\"root\",1919,\"10.233.102.145:37540\",5bcfb5fa." \
+  "77f,1,\"authentication\",2018-10-24 02:59:54 MSK,3/14755,0,FATAL,28P01,\"password authentication failed for user" \
+  " \"\"root\"\"\",\"Role \"\"root\"\" does not exist.
+Connection matched pg_hba.conf line 95: \"\"host all all all md5\"\"\",,,,,,,,\"\"" }]
+
+      expected = [{ "log" => "2018-10-24 02:59:54.936 MSK,\"root\",\"root\",1919,\"10.233.102.145:37540\",5bcfb5fa." \
+  "77f,1,\"authentication\",2018-10-24 02:59:54 MSK,3/14755,0,FATAL,28P01,\"password authentication failed for user" \
+  " \"\"root\"\"\",\"Role \"\"root\"\" does not exist.
+Connection matched pg_hba.conf line 95: \"\"host all all all md5\"\"\",,,,,,,,\"\"",
+            "postgresql" => {
+                  "log_time" => "2018-10-24 02:59:54.936 MSK",
+                  "user_name" => "root",
+                  "database_name" => "root",
+                  "process_id" => "1919",
+                  "connection_from" => "10.233.102.145:37540",
+                  "session_id" => "5bcfb5fa.77f",
+                  "session_line_num" => "1",
+                  "command_tag" => "authentication",
+                  "session_start_time" => "2018-10-24 02:59:54 MSK",
+                  "virtual_transaction_id" => "3/14755",
+                  "transaction_id" => "0",
+                  "error_severity" => "FATAL",
+                  "sql_state_code" => "28P01",
+                  "message" => "password authentication failed for user \"root\"",
+                  "detail" => "Role \"root\" does not exist.\nConnection matched pg_hba.conf line 95: \"host all all all md5\"",
+                  "hint" => nil,
+                  "internal_query" => nil,
+                  "internal_query_pos" => nil,
+                  "context" => nil,
+                  "query" => nil,
+                  "query_pos" => nil,
+                  "location" => nil,
+                  "application_name" => ""
+            }
+         }
+      ]
+
+      filtered_records = filter(conf, message)
+      assert(expected == filtered_records)
+    end
+  end
+
+  sub_test_case 'parse log line and add fields' do
+    test "duration parsing" do
+      conf = CONFIG
+      message = [{ "log" => "2018-10-29 15:14:07.142 MSK,\"app_user\",\"app_user\",30960,\"127.0.0.1:37710\",5bd6f927" \
+  ".78f0,3,\"SELECT\",2018-10-29 15:12:23 MSK,5/0,0,LOG,00000,\"duration: 4004.345 ms  statement: select pg_sleep(4)" \
+  "\",,,,,,,,,\"pgAdmin 4 - CONN:3711814\"" }]
+
+      expected = [{ "log" => "2018-10-29 15:14:07.142 MSK,\"app_user\",\"app_user\",30960,\"127.0.0.1:37710\",5bd6f927" \
+  ".78f0,3,\"SELECT\",2018-10-29 15:12:23 MSK,5/0,0,LOG,00000,\"duration: 4004.345 ms  statement: select pg_sleep(4)" \
+  "\",,,,,,,,,\"pgAdmin 4 - CONN:3711814\"",
+             "postgresql" => {
+                    "log_time" => "2018-10-29 15:14:07.142 MSK",
+                    "user_name" => "app_user",
+                    "database_name" => "app_user",
+                    "process_id" => "30960",
+                    "connection_from" => "127.0.0.1:37710",
+                    "session_id" => "5bd6f927.78f0",
+                    "session_line_num" => "3",
+                    "command_tag" => "SELECT",
+                    "session_start_time" => "2018-10-29 15:12:23 MSK",
+                    "virtual_transaction_id" => "5/0",
+                    "transaction_id" => "0",
+                    "error_severity" => "LOG",
+                    "sql_state_code" => "00000",
+                    "message" => "duration: 4004.345 ms  statement: select pg_sleep(4)",
+                    "detail" => nil,
+                    "hint" => nil,
+                    "internal_query" => nil,
+                    "internal_query_pos" => nil,
+                    "context" => nil,
+                    "query" => nil,
+                    "query_pos" => nil,
+                    "location" => nil,
+                    "application_name" => "pgAdmin 4 - CONN:3711814",
+                    "duration" => "4004.345"
+             }
+          }
+      ]
+
+      filtered_records = filter(conf, message)
+      assert(expected == filtered_records)
+    end
+  end
+
+  sub_test_case 'any string' do
+    test "transfer 'as is'" do
+      conf = CONFIG
+      message = [{"log" => "some,text"}]
+      expected = [{"log" => "some,text"}]
+      filtered_records = filter(conf, message)
+      assert(expected == filtered_records)
+    end
+    test "fake csv" do
+      conf = CONFIG
+      message = [{"log" => "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23"}]
+      expected = [{"log" => "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23"}]
+      filtered_records = filter(conf, message)
+      assert(expected == filtered_records)
+    end
+  end
+end

metadata

@@ -0,0 +1,111 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-postgresql_csv
+version: !ruby/object:Gem::Version
+  version: 0.0.3
+platform: ruby
+authors:
+- Oleg Gurov
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-11-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.8
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.8
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".idea/vcs.xml"
+- Gemfile
+- Rakefile
+- fluent-plugin-postgresql-csv.gemspec
+- lib/fluent/plugin/filter_postgresql_csv.rb
+- test/helper.rb
+- test/plugin/test_filter_postgresql_csv.rb
+homepage: https://github.com/masterlee998/
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.1.0
+  - - "<"
+    - !ruby/object:Gem::Version
+      version: 2.5.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.8
+signing_key: 
+specification_version: 4
+summary: Fluentd filter plugin for PostgreSQL logs in CSV format
+test_files:
+- test/helper.rb
+- test/plugin/test_filter_postgresql_csv.rb