logstash-output-charrington 0.3.24 → 0.3.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +6 -4
- data/lib/logstash/outputs/charrington/alter_postgres_table.rb +41 -41
- data/lib/logstash/outputs/charrington/alter_redshift_table.rb +50 -49
- data/lib/logstash/outputs/charrington/create_postgres_table.rb +33 -32
- data/lib/logstash/outputs/charrington/create_redshift_table.rb +34 -32
- data/lib/logstash/outputs/charrington/insert.rb +64 -64
- data/lib/logstash/outputs/charrington/process.rb +23 -19
- data/lib/logstash/outputs/charrington/service.rb +4 -0
- data/lib/logstash/outputs/charrington/transform_postgres.rb +10 -6
- data/lib/logstash/outputs/charrington/transform_redshift.rb +40 -36
- data/lib/logstash/outputs/charrington.rb +32 -37
- data/lib/logstash-output-charrington_jars.rb +2 -1
- data/logstash-output-charrington.gemspec +9 -9
- data/spec/charrington_spec_helper.rb +30 -35
- data/spec/{logstash-output-charrington_test_jars.rb → logstash_output_charrington_test_jars.rb} +2 -1
- data/spec/outputs/charrington_mysql_spec.rb +1 -0
- data/spec/outputs/charrington_postgres_spec.rb +1 -1
- data/spec/outputs/charrington_spec.rb +33 -29
- metadata +35 -57
@@ -1,4 +1,5 @@
|
|
1
|
-
#
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require 'logstash/outputs/base'
|
3
4
|
require 'logstash/namespace'
|
4
5
|
require 'concurrent'
|
@@ -7,10 +8,10 @@ require 'java'
|
|
7
8
|
require 'logstash-output-charrington_jars'
|
8
9
|
require 'json'
|
9
10
|
require 'bigdecimal'
|
10
|
-
require File.join(File.dirname(__FILE__),
|
11
|
-
require File.join(File.dirname(__FILE__),
|
12
|
-
require File.join(File.dirname(__FILE__),
|
13
|
-
require File.join(File.dirname(__FILE__),
|
11
|
+
require File.join(File.dirname(__FILE__), 'charrington/process')
|
12
|
+
require File.join(File.dirname(__FILE__), 'charrington/transform_postgres')
|
13
|
+
require File.join(File.dirname(__FILE__), 'charrington/transform_redshift')
|
14
|
+
require File.join(File.dirname(__FILE__), 'charrington/insert')
|
14
15
|
|
15
16
|
# Write events to a SQL engine, using JDBC.
|
16
17
|
# It is upto the user of the plugin to correctly configure the plugin.
|
@@ -20,12 +21,11 @@ require File.join(File.dirname(__FILE__), "charrington/insert")
|
|
20
21
|
# is attempted. If that fails, it will try to either
|
21
22
|
# create a table via Charrington::CreateTable
|
22
23
|
# or alter an existing one via Charrington::AlterTable
|
23
|
-
|
24
|
-
class LogStash::Outputs::Charrington < LogStash::Outputs::Base
|
24
|
+
class LogStash::Outputs::Charrington < LogStash::Outputs::Base # rubocop:disable Metrics/ClassLength
|
25
25
|
concurrency :shared
|
26
26
|
config_name 'charrington'
|
27
27
|
|
28
|
-
STRFTIME_FMT = '%Y-%m-%d %T.%L'
|
28
|
+
STRFTIME_FMT = '%Y-%m-%d %T.%L'
|
29
29
|
|
30
30
|
# Driver class - Reintroduced for https://github.com/theangryangel/logstash-output-jdbc/issues/26
|
31
31
|
config :driver_class, validate: :string
|
@@ -98,23 +98,23 @@ class LogStash::Outputs::Charrington < LogStash::Outputs::Base
|
|
98
98
|
|
99
99
|
def multi_receive(events)
|
100
100
|
events.each do |event|
|
101
|
-
|
102
|
-
break unless
|
101
|
+
conn = connection
|
102
|
+
break unless conn
|
103
103
|
|
104
104
|
schema = get_schema(event)
|
105
105
|
|
106
|
-
opts = { connection:
|
106
|
+
opts = { connection: conn,
|
107
107
|
schema: schema,
|
108
108
|
max_retries: @max_flush_exceptions,
|
109
109
|
retry_initial_interval: @retry_initial_interval,
|
110
110
|
driver: driver,
|
111
111
|
transformer: @transformer }
|
112
|
-
Charrington::Process.call(
|
113
|
-
rescue => e
|
112
|
+
Charrington::Process.call(conn, event, opts)
|
113
|
+
rescue StandardError => e
|
114
114
|
@logger.error("Unable to process event. Event dropped. #{e.message}")
|
115
115
|
next
|
116
116
|
ensure
|
117
|
-
|
117
|
+
conn&.close
|
118
118
|
end
|
119
119
|
end
|
120
120
|
|
@@ -129,9 +129,9 @@ class LogStash::Outputs::Charrington < LogStash::Outputs::Base
|
|
129
129
|
def driver
|
130
130
|
case @driver_class
|
131
131
|
when /redshift/
|
132
|
-
|
132
|
+
'redshift'
|
133
133
|
else
|
134
|
-
|
134
|
+
'postgres'
|
135
135
|
end
|
136
136
|
end
|
137
137
|
|
@@ -147,7 +147,7 @@ class LogStash::Outputs::Charrington < LogStash::Outputs::Base
|
|
147
147
|
|
148
148
|
validate_connection_timeout = (@connection_timeout / 1000) / 2
|
149
149
|
|
150
|
-
if !@connection_test_query.nil?
|
150
|
+
if !@connection_test_query.nil? && (@connection_test_query.length > 1)
|
151
151
|
@pool.setConnectionTestQuery(@connection_test_query)
|
152
152
|
@pool.setConnectionInitSql(@connection_test_query)
|
153
153
|
end
|
@@ -156,9 +156,7 @@ class LogStash::Outputs::Charrington < LogStash::Outputs::Base
|
|
156
156
|
|
157
157
|
# Test connection
|
158
158
|
test_connection = @pool.getConnection
|
159
|
-
unless test_connection.isValid(validate_connection_timeout)
|
160
|
-
@logger.warn('JDBC - Connection is not reporting as validate. Either connection is invalid, or driver is not getting the appropriate response.')
|
161
|
-
end
|
159
|
+
@logger.warn('JDBC - Connection is not reporting as validate. Either connection is invalid, or driver is not getting the appropriate response.') unless test_connection.isValid(validate_connection_timeout)
|
162
160
|
test_connection.close
|
163
161
|
end
|
164
162
|
|
@@ -166,6 +164,7 @@ class LogStash::Outputs::Charrington < LogStash::Outputs::Base
|
|
166
164
|
def load_jar_files!
|
167
165
|
unless @driver_jar_path.nil?
|
168
166
|
raise LogStash::ConfigurationError, 'JDBC - Could not find jar file at given path. Check config.' unless File.exist? @driver_jar_path
|
167
|
+
|
169
168
|
require @driver_jar_path
|
170
169
|
return
|
171
170
|
end
|
@@ -192,40 +191,36 @@ class LogStash::Outputs::Charrington < LogStash::Outputs::Base
|
|
192
191
|
if !@schema.nil?
|
193
192
|
@schema
|
194
193
|
else
|
195
|
-
case event.to_hash[
|
196
|
-
when
|
197
|
-
|
198
|
-
when
|
199
|
-
|
194
|
+
case event.to_hash['app_name']
|
195
|
+
when 'Web App'
|
196
|
+
'dea_webapp'
|
197
|
+
when 'Mobile App'
|
198
|
+
'dea_mobileapp'
|
200
199
|
else
|
201
|
-
|
200
|
+
''
|
202
201
|
end
|
203
202
|
end
|
204
203
|
end
|
205
204
|
|
206
|
-
def
|
207
|
-
|
208
|
-
rescue => e
|
205
|
+
def connection
|
206
|
+
@pool.getConnection
|
207
|
+
rescue StandardError => e
|
209
208
|
log_jdbc_exception(e, true, nil)
|
210
209
|
false
|
211
210
|
end
|
212
211
|
|
213
212
|
def log_jdbc_exception(exception, retrying, event)
|
214
213
|
current_exception = exception
|
215
|
-
log_text =
|
214
|
+
log_text = "JDBC - Exception. #{retrying ? 'Retrying' : 'Not retrying'}"
|
216
215
|
|
217
216
|
log_method = (retrying ? 'warn' : 'error')
|
218
217
|
|
219
218
|
loop do
|
220
|
-
@logger.send(log_method, log_text, :
|
219
|
+
@logger.send(log_method, log_text, exception: current_exception, event: event)
|
221
220
|
|
222
|
-
if current_exception.respond_to? 'getNextException'
|
223
|
-
current_exception = current_exception.getNextException()
|
224
|
-
else
|
225
|
-
current_exception = nil
|
226
|
-
end
|
221
|
+
current_exception = (current_exception.getNextException if current_exception.respond_to? 'getNextException')
|
227
222
|
|
228
|
-
break if current_exception
|
223
|
+
break if current_exception.nil?
|
229
224
|
end
|
230
225
|
end
|
231
226
|
end
|
@@ -1,16 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
Gem::Specification.new do |s|
|
2
4
|
s.name = 'logstash-output-charrington'
|
3
|
-
s.version = '0.3.
|
5
|
+
s.version = '0.3.25'
|
4
6
|
|
5
7
|
s.licenses = ['Apache-2.0']
|
6
8
|
s.homepage = 'https://gitlab.podium.com/engineering/analytics/logstash-output-charrington'
|
7
9
|
s.summary = 'This plugin allows you to output to SQL, via JDBC'
|
8
10
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install 'logstash-output-charrington'. This gem is not a stand-alone program"
|
9
|
-
s.authors = [
|
11
|
+
s.authors = %w[dconger brianbroderick spencerdcarlson]
|
10
12
|
s.require_paths = ['lib']
|
11
13
|
|
12
14
|
# Files
|
13
|
-
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','Gemfile','LICENSE.txt']
|
15
|
+
s.files = Dir['lib/**/*', 'spec/**/*', 'vendor/**/*', '*.gemspec', '*.md', 'Gemfile', 'LICENSE.txt']
|
14
16
|
# Tests
|
15
17
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
16
18
|
|
@@ -18,19 +20,17 @@ Gem::Specification.new do |s|
|
|
18
20
|
s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'output' }
|
19
21
|
|
20
22
|
# Gem dependencies
|
21
|
-
s.add_runtime_dependency 'logstash-core-plugin-api', ">= 1.60", "<= 2.99"
|
22
23
|
s.add_runtime_dependency 'logstash-codec-plain', '~> 3.0', '>= 3.0.6'
|
24
|
+
s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
|
23
25
|
|
24
26
|
# The 'install_jars' rake task will download these jars from Maven and put them into the vendor directory
|
25
27
|
# See jar-dependencies gem's wiki - https://github.com/mkristian/jar-dependencies/wiki/declare-jars-inside-gemspec
|
26
28
|
s.requirements << "jar 'com.zaxxer:HikariCP', '2.7.2'"
|
27
29
|
s.requirements << "jar 'org.apache.logging.log4j:log4j-slf4j-impl', '2.16.0'"
|
28
30
|
|
29
|
-
s.add_development_dependency 'logstash-devutils'
|
30
31
|
s.add_development_dependency 'jar-dependencies', '~> 0.4.1'
|
31
|
-
s.add_development_dependency 'ruby-maven', '~> 3.3.12'
|
32
|
-
s.add_development_dependency 'rubocop', '0.41.2'
|
33
|
-
s.add_development_dependency 'logstash-input-generator', '~> 3.0', '>= 3.0.6'
|
34
32
|
s.add_development_dependency 'logstash-codec-json', '~> 3.0', '>= 3.0.5'
|
35
|
-
s.add_development_dependency '
|
33
|
+
s.add_development_dependency 'logstash-devutils', '~> 1.3', '>= 1.3.1'
|
34
|
+
s.add_development_dependency 'logstash-input-generator', '~> 3.0', '>= 3.0.6'
|
35
|
+
s.add_development_dependency 'rubocop', '0.41.2'
|
36
36
|
end
|
@@ -1,15 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'logstash/devutils/rspec/spec_helper'
|
2
4
|
require 'logstash/outputs/charrington'
|
3
5
|
require 'stud/temporary'
|
4
6
|
require 'java'
|
5
|
-
require '
|
7
|
+
require 'logstash_output_charrington_test_jars'
|
6
8
|
require 'securerandom'
|
7
9
|
java_import java.util.ArrayList
|
8
10
|
java_import java.nio.file.Paths
|
9
11
|
|
10
12
|
RSpec::Support::ObjectFormatter.default_instance.max_formatted_output_length = 80000
|
11
|
-
RSpec.configure do |c|
|
12
|
-
end
|
13
13
|
|
14
14
|
RSpec.shared_context 'pipeline' do
|
15
15
|
let(:run_pipeline) do
|
@@ -18,15 +18,9 @@ RSpec.shared_context 'pipeline' do
|
|
18
18
|
end
|
19
19
|
end
|
20
20
|
|
21
|
-
RSpec.shared_context 'postgres' do
|
22
|
-
def start_database(
|
23
|
-
|
24
|
-
config = Java::RuYandexQatoolsEmbedPostgresql::EmbeddedPostgres::cachedRuntimeConfig(Paths::get('/tmp/charrington-test-db-cache')) # avoid archive extraction every time
|
25
|
-
db = Java::RuYandexQatoolsEmbedPostgresql::EmbeddedPostgres.new
|
26
|
-
@url = db.start(config, host, port, database, user, password, ArrayList.new(["-E", "SQL_ASCII", "--locale=C", "--lc-collate=C", "--lc-ctype=C"]))
|
27
|
-
else
|
28
|
-
@url = "jdbc:postgresql://#{host}/#{database}?user=#{user}&password=#{password}";
|
29
|
-
end
|
21
|
+
RSpec.shared_context 'postgres' do # rubocop:disable Metrics/BlockLength
|
22
|
+
def start_database(host, _port, database, user, password)
|
23
|
+
@url = "jdbc:postgresql://#{host}/#{database}?user=#{user}&password=#{password}"
|
30
24
|
|
31
25
|
# setup connection manager
|
32
26
|
@connection_manager = Java::ComZaxxerHikari::HikariDataSource.new
|
@@ -37,41 +31,42 @@ RSpec.shared_context 'postgres' do
|
|
37
31
|
end
|
38
32
|
|
39
33
|
before(:all) do
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
34
|
+
database_host = ENV.fetch('DATABASE_HOST', 'postgres')
|
35
|
+
database_port = ENV.fetch('DATABASE_PORT', 5432)
|
36
|
+
database_name = ENV.fetch('DATABASE_NAME', 'winston')
|
37
|
+
database_user = ENV.fetch('DATABASE_USER', 'postgres')
|
38
|
+
database_password = ENV.fetch('DATABASE_PASSWORD', 'postgres')
|
39
|
+
start_database(database_host, database_port, database_name, database_user, database_password)
|
45
40
|
end
|
46
41
|
|
47
|
-
let(:driver_path)
|
42
|
+
let(:driver_path) do
|
48
43
|
ENV.fetch('DRIVER_JAR_PATH', Pathname.new("#{Dir.pwd}/vendor/jar-dependencies/test-jars/postgresql-42.2.5.jar").to_s)
|
49
|
-
|
44
|
+
end
|
50
45
|
|
51
46
|
def query(sql)
|
52
|
-
execute(sql, true)
|
47
|
+
execute(sql, results: true)
|
53
48
|
end
|
54
49
|
|
55
50
|
def create(sql)
|
56
|
-
execute(sql
|
51
|
+
execute(sql)
|
57
52
|
end
|
58
53
|
|
59
54
|
def insert(sql)
|
60
55
|
execute(sql)
|
61
56
|
end
|
62
57
|
|
63
|
-
def execute(sql, results
|
58
|
+
def execute(sql, results: false)
|
64
59
|
conn = @connection_manager.getConnection
|
65
|
-
stmt = conn.prepareStatement(sql)
|
60
|
+
stmt = conn.prepareStatement(sql)
|
66
61
|
if !results
|
67
62
|
stmt.execute
|
68
|
-
|
63
|
+
nil
|
69
64
|
else
|
70
|
-
rs = stmt.
|
71
|
-
meta = rs.getMetaData
|
72
|
-
n = meta.getColumnCount
|
65
|
+
rs = stmt.execute_query
|
66
|
+
meta = rs.getMetaData
|
67
|
+
n = meta.getColumnCount
|
73
68
|
results = []
|
74
|
-
while rs.next
|
69
|
+
while rs.next
|
75
70
|
row = {}
|
76
71
|
(1..n).each do |i|
|
77
72
|
row[meta.getColumnName(i).to_sym] = rs.getString(i)
|
@@ -80,17 +75,17 @@ RSpec.shared_context 'postgres' do
|
|
80
75
|
end
|
81
76
|
results
|
82
77
|
end
|
83
|
-
rescue => e
|
78
|
+
rescue StandardError => e
|
84
79
|
puts "Error executing query. sql=#{sql} #{e.message}"
|
85
80
|
false
|
86
81
|
ensure
|
87
|
-
stmt.close if !stmt.nil?
|
88
|
-
conn.close if !conn.nil?
|
82
|
+
stmt.close if !stmt.nil? && !stmt.isClosed
|
83
|
+
conn.close if !conn.nil? && !conn.isClosed
|
89
84
|
end
|
90
85
|
|
91
86
|
def create_tracks_table(schema = '')
|
92
87
|
sql = <<-SQL
|
93
|
-
CREATE TABLE IF NOT EXISTS #{
|
88
|
+
CREATE TABLE IF NOT EXISTS #{schema.empty? ? schema : "#{schema}."}tracks
|
94
89
|
(
|
95
90
|
id VARCHAR(512) NOT NULL CONSTRAINT tracks_pkey PRIMARY KEY,
|
96
91
|
action VARCHAR(512),
|
@@ -138,11 +133,11 @@ RSpec.shared_context 'postgres' do
|
|
138
133
|
stmt = conn.createStatement
|
139
134
|
stmt.executeUpdate(sql)
|
140
135
|
true
|
141
|
-
rescue => e
|
136
|
+
rescue StandardError => e
|
142
137
|
puts "Error executing update. sql=#{sql} #{e.message}"
|
143
138
|
false
|
144
139
|
ensure
|
145
|
-
stmt.close if !stmt.nil?
|
146
|
-
conn.close if !conn.nil?
|
140
|
+
stmt.close if !stmt.nil? && !stmt.isClosed
|
141
|
+
conn.close if !conn.nil? && !conn.isClosed
|
147
142
|
end
|
148
143
|
end
|
@@ -1,3 +1,6 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# rubocop:disable Metrics/BlockLength
|
1
4
|
require_relative '../charrington_spec_helper'
|
2
5
|
|
3
6
|
describe LogStash::Outputs::Charrington do
|
@@ -6,7 +9,7 @@ describe LogStash::Outputs::Charrington do
|
|
6
9
|
|
7
10
|
describe 'a new payload with one event that truncates strings correctly' do
|
8
11
|
let(:too_big) do
|
9
|
-
|
12
|
+
'0123456789' * 30
|
10
13
|
end
|
11
14
|
let(:config) do
|
12
15
|
<<-CONFIG
|
@@ -34,8 +37,8 @@ describe LogStash::Outputs::Charrington do
|
|
34
37
|
create_tracks_table
|
35
38
|
drop_table('schemaless')
|
36
39
|
run_pipeline
|
37
|
-
expect(query('SELECT * FROM schemaless')).to match_array([{id:
|
38
|
-
expect(query('SELECT COUNT(1) FROM schemaless').first[:count]).to eq(
|
40
|
+
expect(query('SELECT * FROM schemaless')).to match_array([{ id: '1', app_name: 'Web App', event: 'schemaless', inserted_at: a_kind_of(String), meta_type: 'XML', meta_too_big: too_big[0, 254], published_at: '2019-07-29 20:09:18' }])
|
41
|
+
expect(query('SELECT COUNT(1) FROM schemaless').first[:count]).to eq('1')
|
39
42
|
end
|
40
43
|
end
|
41
44
|
|
@@ -64,8 +67,8 @@ describe LogStash::Outputs::Charrington do
|
|
64
67
|
it 'creates a table and inserts a record' do
|
65
68
|
drop_table('with_plus')
|
66
69
|
run_pipeline
|
67
|
-
expect(query('SELECT * FROM with_plus')).to match_array([{id:
|
68
|
-
expect(query('SELECT COUNT(1) FROM with_plus').first[:count]).to eq(
|
70
|
+
expect(query('SELECT * FROM with_plus')).to match_array([{ id: '1', app_name: 'Web App', event: 'with plus +', inserted_at: a_kind_of(String), meta_type: 'XML' }])
|
71
|
+
expect(query('SELECT COUNT(1) FROM with_plus').first[:count]).to eq('1')
|
69
72
|
end
|
70
73
|
end
|
71
74
|
|
@@ -100,10 +103,10 @@ describe LogStash::Outputs::Charrington do
|
|
100
103
|
drop_table('metadata')
|
101
104
|
run_pipeline
|
102
105
|
expect(query('SELECT * FROM metadata')).to match_array([
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
expect(query('SELECT COUNT(1) FROM metadata').first[:count]).to eq(
|
106
|
+
{ app_name: 'Web App', event: 'metadata', id: '1', inserted_at: a_kind_of(String), meta_file_name: nil, meta_type: 'XML' },
|
107
|
+
{ app_name: 'Web App', event: 'metadata', id: '2', inserted_at: a_kind_of(String), meta_file_name: 'virus.pdf', meta_type: 'XML' }
|
108
|
+
])
|
109
|
+
expect(query('SELECT COUNT(1) FROM metadata').first[:count]).to eq('2')
|
107
110
|
end
|
108
111
|
end
|
109
112
|
|
@@ -136,18 +139,18 @@ describe LogStash::Outputs::Charrington do
|
|
136
139
|
drop_table('existing')
|
137
140
|
create_table('CREATE TABLE existing (id SERIAL PRIMARY KEY, inserted_at TIMESTAMP DEFAULT NOW(), app_name VARCHAR(255))')
|
138
141
|
insert("INSERT INTO existing (app_name) VALUES ('Not Agent')")
|
139
|
-
expect(query('SELECT * FROM existing')).to match_array([{id:
|
142
|
+
expect(query('SELECT * FROM existing')).to match_array([{ id: '1', inserted_at: a_kind_of(String), app_name: 'Not Agent' }])
|
140
143
|
|
141
144
|
run_pipeline
|
142
145
|
expect(query('SELECT * FROM existing')).to match_array([
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
+
{ id: '1', app_name: 'Not Agent', event: nil, inserted_at: a_kind_of(String), meta_type: nil },
|
147
|
+
{ id: '2', app_name: 'Web App', event: 'existing', inserted_at: a_kind_of(String), meta_type: 'XML' }
|
148
|
+
])
|
146
149
|
end
|
147
150
|
end
|
148
151
|
|
149
152
|
describe '2 event payloads with different metadata and different schema' do
|
150
|
-
let(:schema) {
|
153
|
+
let(:schema) { 'dea_test' }
|
151
154
|
let(:config) do
|
152
155
|
<<-CONFIG
|
153
156
|
input {
|
@@ -179,10 +182,10 @@ describe LogStash::Outputs::Charrington do
|
|
179
182
|
drop_table("#{schema}.from_agent")
|
180
183
|
run_pipeline
|
181
184
|
expect(query("SELECT * FROM #{schema}.from_agent")).to match_array([
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
expect(query("SELECT COUNT(1) FROM #{schema}.from_agent").first[:count]).to eq(
|
185
|
+
{ app_name: 'Web App', event: 'From Agent', id: '1', inserted_at: a_kind_of(String), meta_file_name: nil, meta_type: 'XML' },
|
186
|
+
{ app_name: 'Web App', event: 'From Agent', id: '2', inserted_at: a_kind_of(String), meta_file_name: 'virus.pdf', meta_type: 'XML' }
|
187
|
+
])
|
188
|
+
expect(query("SELECT COUNT(1) FROM #{schema}.from_agent").first[:count]).to eq('2')
|
186
189
|
end
|
187
190
|
end
|
188
191
|
|
@@ -211,8 +214,8 @@ describe LogStash::Outputs::Charrington do
|
|
211
214
|
it 'creates a table and inserts a record' do
|
212
215
|
drop_table('camel_case_this')
|
213
216
|
run_pipeline
|
214
|
-
expect(query('SELECT * FROM camel_case_this')).to match_array([{id:
|
215
|
-
expect(query('SELECT COUNT(1) FROM camel_case_this').first[:count]).to eq(
|
217
|
+
expect(query('SELECT * FROM camel_case_this')).to match_array([{ id: '1', app_name: 'Web App', event: 'camelCase this', inserted_at: a_kind_of(String), meta_type: 'XML' }])
|
218
|
+
expect(query('SELECT COUNT(1) FROM camel_case_this').first[:count]).to eq('1')
|
216
219
|
end
|
217
220
|
end
|
218
221
|
|
@@ -244,12 +247,12 @@ describe LogStash::Outputs::Charrington do
|
|
244
247
|
drop_table('schemaless')
|
245
248
|
run_pipeline
|
246
249
|
expect(query('SELECT * FROM schemaless')).to match_array([
|
247
|
-
|
248
|
-
|
249
|
-
expect(query('SELECT COUNT(1) FROM schemaless').first[:count]).to eq(
|
250
|
-
expect(query(
|
251
|
-
|
252
|
-
|
250
|
+
{ anonymous_id: '', action: 'click', app_name: 'Web App', event: 'schemaless', event_text: 'schemaless', id: a_kind_of(String), original_timestamp: a_kind_of(String), received_at: a_kind_of(String), sent_at: '2019-07-29 20:09:18', timestamp: a_kind_of(String), type: 'XML', user_id: '123', user_uid: '456', uuid_ts: a_kind_of(String) }
|
251
|
+
])
|
252
|
+
expect(query('SELECT COUNT(1) FROM schemaless').first[:count]).to eq('1')
|
253
|
+
expect(query('SELECT * FROM tracks')).to match_array([
|
254
|
+
{ id: a_kind_of(String), action: 'click', anonymous_id: '', app_name: 'Web App', context_campaign_content: nil, context_campaign_medium: nil, context_campaign_name: nil, context_campaign_source: nil, context_ip: nil, context_library_name: nil, context_library_version: nil, context_page_path: nil, context_page_referrer: nil, context_page_search: nil, context_page_title: nil, context_page_url: nil, context_user_agent: nil, event: 'schemaless', event_text: 'schemaless', original_timestamp: '2019-07-29 20:09:18', received_at: '2019-07-29 20:09:18', segment_dedupe_id: nil, sent_at: '2019-07-29 20:09:18', timestamp: '2019-07-29 20:09:18', user_id: '123', user_uid: '456', uuid: nil, uuid_ts: a_kind_of(String) }
|
255
|
+
])
|
253
256
|
end
|
254
257
|
end
|
255
258
|
|
@@ -281,9 +284,10 @@ describe LogStash::Outputs::Charrington do
|
|
281
284
|
drop_table('schemaless')
|
282
285
|
run_pipeline
|
283
286
|
expect(query('SELECT * FROM schemaless')).to match_array([
|
284
|
-
|
285
|
-
|
286
|
-
expect(query('SELECT COUNT(1) FROM schemaless').first[:count]).to eq(
|
287
|
+
{ anonymous_id: '', app_name: 'Web App', event: 'schemaless', event_text: 'schemaless', id: a_kind_of(String), original_timestamp: '2019-07-29 20:09:18', received_at: '2019-07-29 20:09:18', sent_at: '2019-07-29 20:09:18', timestamp: '2019-07-29 20:09:18', type: 'XML', user_id: '123', user_uid: '456', uuid_ts: a_kind_of(String) }
|
288
|
+
])
|
289
|
+
expect(query('SELECT COUNT(1) FROM schemaless').first[:count]).to eq('1')
|
287
290
|
end
|
288
291
|
end
|
289
292
|
end
|
293
|
+
# rubocop:enable Metrics/BlockLength
|