fluent-plugin-postgresql_csv 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 74da41159df683170a38a0dec1389413446a7a9598f0958a787804eef2bdfc4b
4
+ data.tar.gz: 522819601c6a2eea5f4dc7e54b2880d73069e9e2dc15d2378480e4e46c58cc7c
5
+ SHA512:
6
+ metadata.gz: f159d28274c110d1320c5e28cb0d92fa6c6ccdc816ad00c7589a90ae1a15563538f8253ec1f8b379b3cdd6b8fc7fb09c2ce25321f489900225a68515994e10fb
7
+ data.tar.gz: 4f1866eb5c8ff4f71d11fc6b78a6127ffa70773a74cff14edfa82f521a10b3e280bef68d600aefb75faf057b9ce0dcfc296da752f9829cb311aa0938f7888015
@@ -0,0 +1,6 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project version="4">
3
+ <component name="VcsDirectoryMappings">
4
+ <mapping directory="$PROJECT_DIR$" vcs="Git" />
5
+ </component>
6
+ </project>
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gemspec
@@ -0,0 +1,3 @@
1
+ require 'bundler/gem_tasks'
2
+
3
+ task :default => :test
@@ -0,0 +1,24 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+
5
+ Gem::Specification.new do |gem|
6
+ gem.name = "fluent-plugin-postgresql_csv"
7
+ gem.version = "0.0.3"
8
+ gem.summary = %q{Fluentd filter plugin for PostgreSQL logs in CSV format}
9
+ gem.authors = ["Oleg Gurov"]
10
+ gem.homepage = "https://github.com/masterlee998/"
11
+ gem.license = 'MIT'
12
+
13
+ gem.files = `git ls-files`.split($/)
14
+ gem.executables = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
15
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
16
+ gem.require_paths = ["lib"]
17
+ gem.has_rdoc = false
18
+
19
+ gem.required_ruby_version = ['>= 2.1.0', '< 2.5.0']
20
+ gem.add_runtime_dependency 'fluentd', '>= 0.14.0'
21
+ gem.add_development_dependency 'bundler', '~> 1.3'
22
+ gem.add_development_dependency 'test-unit', '~> 3.2.8'
23
+ gem.add_development_dependency 'rake'
24
+ end
@@ -0,0 +1,69 @@
1
+ require 'fluent/plugin/filter'
2
+
3
+ module Fluent::Plugin
4
+ class PostgreSQL_CSV_Filter < Filter
5
+ Fluent::Plugin.register_filter('postgresql_csv', self)
6
+
7
+ def initialize
8
+ super
9
+ require 'csv'
10
+ require 'date'
11
+ end
12
+
13
+ def configure(conf)
14
+ super
15
+ end
16
+
17
+ def filter(tag, time, record)
18
+ record_orig = record.clone
19
+ begin
20
+ csv = CSV.parse(record['log'])[0]
21
+ if csv.length != 23
22
+ return record
23
+ end
24
+
25
+ begin
26
+ DateTime.strptime(csv[0], '%Y-%m-%d %H:%M:%S.%L %Z')
27
+ rescue ArgumentError
28
+ return record
29
+ end
30
+
31
+ parsed_value = {}
32
+ parsed_value['log_time'] = csv[0]
33
+ parsed_value['user_name'] = csv[1]
34
+ parsed_value['database_name'] = csv[2]
35
+ parsed_value['process_id'] = csv[3]
36
+ parsed_value['connection_from'] = csv[4]
37
+ parsed_value['session_id'] = csv[5]
38
+ parsed_value['session_line_num'] = csv[6]
39
+ parsed_value['command_tag'] = csv[7]
40
+ parsed_value['session_start_time'] = csv[8]
41
+ parsed_value['virtual_transaction_id'] = csv[9]
42
+ parsed_value['transaction_id'] = csv[10]
43
+ parsed_value['error_severity'] = csv[11]
44
+ parsed_value['sql_state_code'] = csv[12]
45
+ parsed_value['message'] = csv[13]
46
+ parsed_value['detail'] = csv[14]
47
+ parsed_value['hint'] = csv[15]
48
+ parsed_value['internal_query'] = csv[16]
49
+ parsed_value['internal_query_pos'] = csv[17]
50
+ parsed_value['context'] = csv[18]
51
+ parsed_value['query'] = csv[19]
52
+ parsed_value['query_pos'] = csv[20]
53
+ parsed_value['location'] = csv[21]
54
+ parsed_value['application_name'] = csv[22]
55
+
56
+ duration = parsed_value['message'].scan(/duration: (\d+(?:\.\d+)?) ms statement: (.+)/m)[0]
57
+ if duration != nil and duration.kind_of?(Array)
58
+ parsed_value['duration'] = duration[0]
59
+ #parsed_value['duration'] = duration[0].to_f
60
+ end
61
+
62
+ record['postgresql'] = parsed_value
63
+ rescue CSV::MalformedCSVError
64
+ return record_orig
65
+ end
66
+ return record
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,27 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+ begin
4
+ Bundler.setup(:default, :development)
5
+ rescue Bundler::BundlerError => e
6
+ $stderr.puts e.message
7
+ $stderr.puts "Run `bundle install` to install missing gems"
8
+ exit e.status_code
9
+ end
10
+
11
+ $LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
12
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
13
+
14
+ require "fluent/test"
15
+ require "fluent/test/driver/filter"
16
+ require "fluent/test/helpers"
17
+
18
+ unless ENV.has_key?('VERBOSE')
19
+ nulllogger = Object.new
20
+ nulllogger.instance_eval {|obj|
21
+ def method_missing(method, *args)
22
+ # pass
23
+ end
24
+ }
25
+ $log = nulllogger
26
+ end
27
+ require 'fluent/plugin/filter_postgresql_csv'
@@ -0,0 +1,133 @@
1
+ require_relative '../helper'
2
+
3
+ class PostgreSQL_CSV_Filter < Test::Unit::TestCase
4
+ include Fluent::Test::Helpers
5
+
6
+ def setup
7
+ Fluent::Test.setup
8
+ end
9
+
10
+ CONFIG = %[
11
+ dummy_param dummy_value
12
+ ]
13
+
14
+ def create_driver(conf = CONFIG)
15
+ Fluent::Test::Driver::Filter.new(Fluent::Plugin::PostgreSQL_CSV_Filter).configure(conf)
16
+ end
17
+
18
+ def filter(config, messages)
19
+ d = create_driver(config)
20
+ d.run(default_tag: "input.access") do
21
+ messages.each do |message|
22
+ d.feed(message)
23
+ end
24
+ end
25
+ d.filtered_records
26
+ end
27
+
28
+ sub_test_case 'parse log line and add fields' do
29
+ test "error message" do
30
+ conf = CONFIG
31
+ message = [{ "log" => "2018-10-24 02:59:54.936 MSK,\"root\",\"root\",1919,\"10.233.102.145:37540\",5bcfb5fa." \
32
+ "77f,1,\"authentication\",2018-10-24 02:59:54 MSK,3/14755,0,FATAL,28P01,\"password authentication failed for user" \
33
+ " \"\"root\"\"\",\"Role \"\"root\"\" does not exist.
34
+ Connection matched pg_hba.conf line 95: \"\"host all all all md5\"\"\",,,,,,,,\"\"" }]
35
+
36
+ expected = [{ "log" => "2018-10-24 02:59:54.936 MSK,\"root\",\"root\",1919,\"10.233.102.145:37540\",5bcfb5fa." \
37
+ "77f,1,\"authentication\",2018-10-24 02:59:54 MSK,3/14755,0,FATAL,28P01,\"password authentication failed for user" \
38
+ " \"\"root\"\"\",\"Role \"\"root\"\" does not exist.
39
+ Connection matched pg_hba.conf line 95: \"\"host all all all md5\"\"\",,,,,,,,\"\"",
40
+ "postgresql" => {
41
+ "log_time" => "2018-10-24 02:59:54.936 MSK",
42
+ "user_name" => "root",
43
+ "database_name" => "root",
44
+ "process_id" => "1919",
45
+ "connection_from" => "10.233.102.145:37540",
46
+ "session_id" => "5bcfb5fa.77f",
47
+ "session_line_num" => "1",
48
+ "command_tag" => "authentication",
49
+ "session_start_time" => "2018-10-24 02:59:54 MSK",
50
+ "virtual_transaction_id" => "3/14755",
51
+ "transaction_id" => "0",
52
+ "error_severity" => "FATAL",
53
+ "sql_state_code" => "28P01",
54
+ "message" => "password authentication failed for user \"root\"",
55
+ "detail" => "Role \"root\" does not exist.\nConnection matched pg_hba.conf line 95: \"host all all all md5\"",
56
+ "hint" => nil,
57
+ "internal_query" => nil,
58
+ "internal_query_pos" => nil,
59
+ "context" => nil,
60
+ "query" => nil,
61
+ "query_pos" => nil,
62
+ "location" => nil,
63
+ "application_name" => ""
64
+ }
65
+ }
66
+ ]
67
+
68
+ filtered_records = filter(conf, message)
69
+ assert(expected == filtered_records)
70
+ end
71
+ end
72
+
73
+ sub_test_case 'parse log line and add fields' do
74
+ test "duration parsing" do
75
+ conf = CONFIG
76
+ message = [{ "log" => "2018-10-29 15:14:07.142 MSK,\"app_user\",\"app_user\",30960,\"127.0.0.1:37710\",5bd6f927" \
77
+ ".78f0,3,\"SELECT\",2018-10-29 15:12:23 MSK,5/0,0,LOG,00000,\"duration: 4004.345 ms statement: select pg_sleep(4)" \
78
+ "\",,,,,,,,,\"pgAdmin 4 - CONN:3711814\"" }]
79
+
80
+ expected = [{ "log" => "2018-10-29 15:14:07.142 MSK,\"app_user\",\"app_user\",30960,\"127.0.0.1:37710\",5bd6f927" \
81
+ ".78f0,3,\"SELECT\",2018-10-29 15:12:23 MSK,5/0,0,LOG,00000,\"duration: 4004.345 ms statement: select pg_sleep(4)" \
82
+ "\",,,,,,,,,\"pgAdmin 4 - CONN:3711814\"",
83
+ "postgresql" => {
84
+ "log_time" => "2018-10-29 15:14:07.142 MSK",
85
+ "user_name" => "app_user",
86
+ "database_name" => "app_user",
87
+ "process_id" => "30960",
88
+ "connection_from" => "127.0.0.1:37710",
89
+ "session_id" => "5bd6f927.78f0",
90
+ "session_line_num" => "3",
91
+ "command_tag" => "SELECT",
92
+ "session_start_time" => "2018-10-29 15:12:23 MSK",
93
+ "virtual_transaction_id" => "5/0",
94
+ "transaction_id" => "0",
95
+ "error_severity" => "LOG",
96
+ "sql_state_code" => "00000",
97
+ "message" => "duration: 4004.345 ms statement: select pg_sleep(4)",
98
+ "detail" => nil,
99
+ "hint" => nil,
100
+ "internal_query" => nil,
101
+ "internal_query_pos" => nil,
102
+ "context" => nil,
103
+ "query" => nil,
104
+ "query_pos" => nil,
105
+ "location" => nil,
106
+ "application_name" => "pgAdmin 4 - CONN:3711814",
107
+ "duration" => "4004.345"
108
+ }
109
+ }
110
+ ]
111
+
112
+ filtered_records = filter(conf, message)
113
+ assert(expected == filtered_records)
114
+ end
115
+ end
116
+
117
+ sub_test_case 'any string' do
118
+ test "transfer 'as is'" do
119
+ conf = CONFIG
120
+ message = [{"log" => "some,text"}]
121
+ expected = [{"log" => "some,text"}]
122
+ filtered_records = filter(conf, message)
123
+ assert(expected == filtered_records)
124
+ end
125
+ test "fake csv" do
126
+ conf = CONFIG
127
+ message = [{"log" => "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23"}]
128
+ expected = [{"log" => "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23"}]
129
+ filtered_records = filter(conf, message)
130
+ assert(expected == filtered_records)
131
+ end
132
+ end
133
+ end
metadata ADDED
@@ -0,0 +1,111 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-postgresql_csv
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.3
5
+ platform: ruby
6
+ authors:
7
+ - Oleg Gurov
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2018-11-11 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: fluentd
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: 0.14.0
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: 0.14.0
27
+ - !ruby/object:Gem::Dependency
28
+ name: bundler
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '1.3'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.3'
41
+ - !ruby/object:Gem::Dependency
42
+ name: test-unit
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: 3.2.8
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: 3.2.8
55
+ - !ruby/object:Gem::Dependency
56
+ name: rake
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ description:
70
+ email:
71
+ executables: []
72
+ extensions: []
73
+ extra_rdoc_files: []
74
+ files:
75
+ - ".idea/vcs.xml"
76
+ - Gemfile
77
+ - Rakefile
78
+ - fluent-plugin-postgresql-csv.gemspec
79
+ - lib/fluent/plugin/filter_postgresql_csv.rb
80
+ - test/helper.rb
81
+ - test/plugin/test_filter_postgresql_csv.rb
82
+ homepage: https://github.com/masterlee998/
83
+ licenses:
84
+ - MIT
85
+ metadata: {}
86
+ post_install_message:
87
+ rdoc_options: []
88
+ require_paths:
89
+ - lib
90
+ required_ruby_version: !ruby/object:Gem::Requirement
91
+ requirements:
92
+ - - ">="
93
+ - !ruby/object:Gem::Version
94
+ version: 2.1.0
95
+ - - "<"
96
+ - !ruby/object:Gem::Version
97
+ version: 2.5.0
98
+ required_rubygems_version: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - ">="
101
+ - !ruby/object:Gem::Version
102
+ version: '0'
103
+ requirements: []
104
+ rubyforge_project:
105
+ rubygems_version: 2.7.8
106
+ signing_key:
107
+ specification_version: 4
108
+ summary: Fluentd filter plugin for PostgreSQL logs in CSV format
109
+ test_files:
110
+ - test/helper.rb
111
+ - test/plugin/test_filter_postgresql_csv.rb