logstash-output-charrington 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d16654dc603c8a1f7ad5e0c394c09c8a1bc018f040f81f22d16b2e13df0cba6d
|
4
|
+
data.tar.gz: 3d74abf0777655dbcd133dce2254dd6f026764ca31b0d76853d04c241f48bcfb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a2e8d0be10c3491ee93548351d898e631f2c5011e3b4998671aaa447ca77d07ee940dc4162d6ae6ae590bdac11bcdeed4574fa0d33b9d55c88caf7dd2022a4b4
|
7
|
+
data.tar.gz: 31eb24bbab66fbd069e723c684073fd1ce7c777146e538aab34709c2249a30eb45e543f1af75d1d80b38d6558cb6ff80bf008ee4468d3d4830250e66af24cd05
|
@@ -39,27 +39,17 @@ module Charrington
|
|
39
39
|
|
40
40
|
def call
|
41
41
|
insert_stmt = insert_statement
|
42
|
+
|
42
43
|
stmt = connection.prepareStatement(insert_stmt)
|
43
44
|
stmt = add_statement_event_params(stmt)
|
44
45
|
stmt.execute
|
45
46
|
should_retry
|
46
|
-
rescue Java::OrgPostgresqlUtil::PSQLException => e
|
47
|
-
case e.getSQLState()
|
48
|
-
when "42P01"
|
49
|
-
should_retry = Charrington::CreatePostgresTable.call(connection, event, schema, table_name, columns)
|
50
|
-
when "42703"
|
51
|
-
should_retry = Charrington::AlterPostgresTable.call(connection, event, schema, table_name, columns)
|
52
|
-
else
|
53
|
-
raise InsertFailed, "Charrington: Rescue from SQLException #{e.message}"
|
54
|
-
end
|
55
|
-
should_retry
|
56
47
|
rescue Java::JavaSql::SQLException => e
|
57
|
-
puts "catching SQLException #{e.message}"
|
58
48
|
case e.getSQLState()
|
59
49
|
when "42P01"
|
60
|
-
should_retry =
|
50
|
+
should_retry = create_table
|
61
51
|
when "42703"
|
62
|
-
should_retry =
|
52
|
+
should_retry = alter_table
|
63
53
|
else
|
64
54
|
raise InsertFailed, "Charrington: Rescue from SQLException #{e.message}"
|
65
55
|
end
|
@@ -74,6 +64,22 @@ module Charrington
|
|
74
64
|
|
75
65
|
private
|
76
66
|
|
67
|
+
def create_table
|
68
|
+
if driver == "postgres"
|
69
|
+
Charrington::CreatePostgresTable.call(connection, event, schema, table_name, columns)
|
70
|
+
elsif driver == "redshift"
|
71
|
+
Charrington::CreateRedshiftTable.call(connection, event, schema, table_name, columns)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def alter_table
|
76
|
+
if driver == "postgres"
|
77
|
+
Charrington::AlterPostgresTable.call(connection, event, schema, table_name, columns)
|
78
|
+
elsif driver == "redshift"
|
79
|
+
Charrington::AlterRedshiftTable.call(connection, event, schema, table_name, columns)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
77
83
|
def cleanup
|
78
84
|
@columns.clear if clearable(@columns)
|
79
85
|
end
|
@@ -9,7 +9,7 @@ describe LogStash::Outputs::Charrington do
|
|
9
9
|
<<-CONFIG
|
10
10
|
input {
|
11
11
|
generator {
|
12
|
-
message => '{"app_name": "Web App", "event": "
|
12
|
+
message => '{"app_name": "Web App", "event": "schemaless", "meta": { "type": "XML" } }'
|
13
13
|
codec => 'json'
|
14
14
|
count => 1
|
15
15
|
}
|
@@ -25,10 +25,10 @@ describe LogStash::Outputs::Charrington do
|
|
25
25
|
end
|
26
26
|
|
27
27
|
it 'creates a table and inserts a record' do
|
28
|
-
drop_table('
|
28
|
+
drop_table('schemaless')
|
29
29
|
run_pipeline
|
30
|
-
expect(query('SELECT * FROM
|
31
|
-
expect(query('SELECT COUNT(1) FROM
|
30
|
+
expect(query('SELECT * FROM schemaless')).to match_array([{id: "1", app_name: "Web App", event: "schemaless", inserted_at: a_kind_of(String), :meta_type => "XML"}])
|
31
|
+
expect(query('SELECT COUNT(1) FROM schemaless').first[:count]).to eq("1")
|
32
32
|
end
|
33
33
|
end
|
34
34
|
|
@@ -38,8 +38,8 @@ describe LogStash::Outputs::Charrington do
|
|
38
38
|
input {
|
39
39
|
generator {
|
40
40
|
lines => [
|
41
|
-
'{"app_name": "Web App", "event": "
|
42
|
-
'{"app_name": "Web App", "event": "
|
41
|
+
'{"app_name": "Web App", "event": "metadata", "meta": { "type": "XML" } }',
|
42
|
+
'{"app_name": "Web App", "event": "metadata", "meta": { "type": "XML", "file_name": "virus.pdf" } }'
|
43
43
|
]
|
44
44
|
codec => 'json'
|
45
45
|
count => 1
|
@@ -56,13 +56,13 @@ describe LogStash::Outputs::Charrington do
|
|
56
56
|
end
|
57
57
|
|
58
58
|
it 'creates a table and inserts the first record and alters the table for the second record' do
|
59
|
-
drop_table('
|
59
|
+
drop_table('metadata')
|
60
60
|
run_pipeline
|
61
|
-
expect(query('SELECT * FROM
|
62
|
-
{app_name: 'Web App', event: '
|
63
|
-
{app_name: 'Web App', event: '
|
61
|
+
expect(query('SELECT * FROM metadata')).to match_array([
|
62
|
+
{app_name: 'Web App', event: 'metadata', id: '1', inserted_at: a_kind_of(String), meta_file_name: nil, meta_type: 'XML'},
|
63
|
+
{app_name: 'Web App', event: 'metadata', id: '2', inserted_at: a_kind_of(String), meta_file_name: 'virus.pdf', meta_type: 'XML'}
|
64
64
|
])
|
65
|
-
expect(query('SELECT COUNT(1) FROM
|
65
|
+
expect(query('SELECT COUNT(1) FROM metadata').first[:count]).to eq("2")
|
66
66
|
end
|
67
67
|
end
|
68
68
|
|
@@ -71,7 +71,7 @@ describe LogStash::Outputs::Charrington do
|
|
71
71
|
<<-CONFIG
|
72
72
|
input {
|
73
73
|
generator {
|
74
|
-
message => '{"app_name": "Web App", "event": "
|
74
|
+
message => '{"app_name": "Web App", "event": "existing", "meta": { "type": "XML" } }'
|
75
75
|
codec => 'json'
|
76
76
|
count => 1
|
77
77
|
}
|
@@ -88,15 +88,15 @@ describe LogStash::Outputs::Charrington do
|
|
88
88
|
|
89
89
|
it 'can insert into an existing table' do
|
90
90
|
# setup pre-existing data
|
91
|
-
drop_table('
|
92
|
-
create_table('CREATE TABLE
|
93
|
-
insert("INSERT INTO
|
94
|
-
expect(query('SELECT * FROM
|
91
|
+
drop_table('existing')
|
92
|
+
create_table('CREATE TABLE existing (id SERIAL PRIMARY KEY, inserted_at TIMESTAMP DEFAULT NOW(), app_name VARCHAR(255))')
|
93
|
+
insert("INSERT INTO existing (app_name) VALUES ('Not Agent')")
|
94
|
+
expect(query('SELECT * FROM existing')).to match_array([{id: "1", inserted_at: a_kind_of(String), app_name: "Not Agent"}])
|
95
95
|
|
96
96
|
run_pipeline
|
97
|
-
expect(query('SELECT * FROM
|
97
|
+
expect(query('SELECT * FROM existing')).to match_array([
|
98
98
|
{id: '1', app_name: 'Not Agent', event: nil, inserted_at: a_kind_of(String), meta_type: nil},
|
99
|
-
{id: '2', app_name: 'Web App', event: '
|
99
|
+
{id: '2', app_name: 'Web App', event: 'existing', inserted_at: a_kind_of(String), meta_type: 'XML'}
|
100
100
|
])
|
101
101
|
end
|
102
102
|
end
|