logstash-output-charrington 0.2.2 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (27) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +38 -1
  3. data/lib/logstash/outputs/charrington.rb +32 -5
  4. data/lib/logstash/outputs/charrington/{alter_table.rb → alter_postgres_table.rb} +6 -5
  5. data/lib/logstash/outputs/charrington/alter_redshift_table.rb +109 -0
  6. data/lib/logstash/outputs/charrington/{create_table.rb → create_postgres_table.rb} +5 -4
  7. data/lib/logstash/outputs/charrington/create_redshift_table.rb +88 -0
  8. data/lib/logstash/outputs/charrington/insert.rb +27 -9
  9. data/lib/logstash/outputs/charrington/process.rb +8 -2
  10. data/lib/logstash/outputs/charrington/{transform.rb → transform_postgres.rb} +1 -1
  11. data/lib/logstash/outputs/charrington/transform_redshift.rb +102 -0
  12. data/logstash-output-charrington.gemspec +11 -9
  13. data/spec/charrington_spec_helper.rb +75 -188
  14. data/spec/logstash-output-charrington_test_jars.rb +5 -0
  15. data/spec/outputs/charrington_spec.rb +118 -39
  16. metadata +52 -34
  17. data/lib/commons-io/commons-io/2.4/commons-io-2.4.jar +0 -0
  18. data/lib/de/flapdoodle/embed/de.flapdoodle.embed.process/2.0.2/de.flapdoodle.embed.process-2.0.2.jar +0 -0
  19. data/lib/net/java/dev/jna/jna-platform/4.0.0/jna-platform-4.0.0.jar +0 -0
  20. data/lib/net/java/dev/jna/jna/4.0.0/jna-4.0.0.jar +0 -0
  21. data/lib/org/apache/commons/commons-compress/1.10/commons-compress-1.10.jar +0 -0
  22. data/lib/org/apache/commons/commons-lang3/3.1/commons-lang3-3.1.jar +0 -0
  23. data/lib/org/postgresql/postgresql/42.2.5/postgresql-42.2.5.jar +0 -0
  24. data/lib/ru/yandex/qatools/embed/postgresql-embedded/2.10/postgresql-embedded-2.10.jar +0 -0
  25. data/lib/ru/yandex/qatools/embed/postgresql-embedded/2.8/postgresql-embedded-2.8.jar +0 -0
  26. data/vendor/postgresql-42.2.5.jar +0 -0
  27. data/vendor/redshift.jar +0 -0
data/spec/logstash-output-charrington_test_jars.rb ADDED
@@ -0,0 +1,5 @@
1
+ # encoding: utf-8
2
+ require 'logstash/environment'
3
+
4
+ root_dir = File.expand_path(File.join(File.dirname(__FILE__), '..'))
5
+ LogStash::Environment.load_test_jars! File.join(root_dir, 'vendor')
data/spec/outputs/charrington_spec.rb CHANGED
@@ -1,44 +1,46 @@
1
1
  require_relative '../charrington_spec_helper'
2
- require 'insist'
3
2
 
4
3
  describe LogStash::Outputs::Charrington do
5
- describe 'when initializing' do
6
- it 'shouldn\'t register without a config' do
7
- expect do
8
- LogStash::Plugin.lookup('output', 'charrington').new
9
- end.to raise_error(LogStash::ConfigurationError)
4
+ include_context 'postgres'
5
+ include_context 'pipeline'
6
+
7
+ describe 'a new payload with one event' do
8
+ let(:config) do
9
+ <<-CONFIG
10
+ input {
11
+ generator {
12
+ message => '{"app_name": "Web App", "event": "From Agent", "meta": { "type": "XML" } }'
13
+ codec => 'json'
14
+ count => 1
15
+ }
16
+ }
17
+
18
+ output {
19
+ charrington {
20
+ connection_string => '#{@url}'
21
+ driver_jar_path => '#{driver_path}'
22
+ }
23
+ }
24
+ CONFIG
25
+ end
26
+
27
+ it 'creates a table and inserts a record' do
28
+ drop_table('from_agent')
29
+ run_pipeline
30
+ expect(query('SELECT * FROM from_agent')).to match_array([{id: "1", app_name: "Web App", event: "From Agent", inserted_at: a_kind_of(String), :meta_type => "XML"}])
31
+ expect(query('SELECT COUNT(1) FROM from_agent').first[:count]).to eq("1")
10
32
  end
11
33
  end
12
34
 
13
- # describe 'integration tests with agent' do
14
- # config <<-HEREDOC
15
- # input {
16
- # generator {
17
- # message => '{"app_name": "Web App", "event": "From Agent"}'
18
- # codec => 'json'
19
- # count => 1
20
- # }
21
- # }
22
-
23
- # output {
24
- # charrington {
25
- # connection_string => 'jdbc:postgresql://localhost:5432/winston?user=postgres&password=postgres'
26
- # driver_jar_path => '/projects/logstash-output-charrington/vendor/postgresql-42.2.5.jar'
27
- # schema => 'dea'
28
- # }
29
- # }
30
- # HEREDOC
31
-
32
- # agent do
33
- # puts "IT'S WORKING!!!!!"
34
- # end
35
- # end
36
-
37
- describe 'integration tests' do
38
- let(:config) do <<-CONFIG
35
+ describe '2 event payloads with different metadata' do
36
+ let(:config) do
37
+ <<-CONFIG
39
38
  input {
40
39
  generator {
41
- message => '{"app_name": "Web App", "event": "Hi - Dan"}'
40
+ lines => [
41
+ '{"app_name": "Web App", "event": "From Agent", "meta": { "type": "XML" } }',
42
+ '{"app_name": "Web App", "event": "From Agent", "meta": { "type": "XML", "file_name": "virus.pdf" } }'
43
+ ]
42
44
  codec => 'json'
43
45
  count => 1
44
46
  }
@@ -46,19 +48,96 @@ describe LogStash::Outputs::Charrington do
46
48
 
47
49
  output {
48
50
  charrington {
49
- connection_string => 'jdbc:postgresql://localhost:5432/winston?user=postgres&password=postgres'
50
- driver_jar_path => '/projects/logstash-output-charrington/vendor/postgresql-42.2.5.jar'
51
- schema => 'dea'
51
+ connection_string => '#{@url}'
52
+ driver_jar_path => '#{driver_path}'
52
53
  }
53
54
  }
54
55
  CONFIG
55
56
  end
56
57
 
57
- it("isn't effed") do
58
- pipeline = new_pipeline_from_string(config)
59
- pipeline.run
58
+ it 'creates a table and inserts the first record and alters the table for the second record' do
59
+ drop_table('from_agent')
60
+ run_pipeline
61
+ expect(query('SELECT * FROM from_agent')).to match_array([
62
+ {app_name: 'Web App', event: 'From Agent', id: '1', inserted_at: a_kind_of(String), meta_file_name: nil, meta_type: 'XML'},
63
+ {app_name: 'Web App', event: 'From Agent', id: '2', inserted_at: a_kind_of(String), meta_file_name: 'virus.pdf', meta_type: 'XML'}
64
+ ])
65
+ expect(query('SELECT COUNT(1) FROM from_agent').first[:count]).to eq("2")
60
66
  end
61
67
  end
68
+
69
+ describe 'event payload for an existing table with existing data' do
70
+ let(:config) do
71
+ <<-CONFIG
72
+ input {
73
+ generator {
74
+ message => '{"app_name": "Web App", "event": "From Agent", "meta": { "type": "XML" } }'
75
+ codec => 'json'
76
+ count => 1
77
+ }
78
+ }
79
+
80
+ output {
81
+ charrington {
82
+ connection_string => '#{@url}'
83
+ driver_jar_path => '#{driver_path}'
84
+ }
85
+ }
86
+ CONFIG
87
+ end
88
+
89
+ it 'can insert into an existing table' do
90
+ # setup pre-existing data
91
+ drop_table('from_agent')
92
+ create_table('CREATE TABLE from_agent (id SERIAL PRIMARY KEY, inserted_at TIMESTAMP DEFAULT NOW(), app_name VARCHAR(255))')
93
+ insert("INSERT INTO from_agent (app_name) VALUES ('Not Agent')")
94
+ expect(query('SELECT * FROM from_agent')).to match_array([{id: "1", inserted_at: a_kind_of(String), app_name: "Not Agent"}])
95
+
96
+ run_pipeline
97
+ expect(query('SELECT * FROM from_agent')).to match_array([
98
+ {id: '1', app_name: 'Not Agent', event: nil, inserted_at: a_kind_of(String), meta_type: nil},
99
+ {id: '2', app_name: 'Web App', event: 'From Agent', inserted_at: a_kind_of(String), meta_type: 'XML'}
100
+ ])
101
+ end
102
+ end
103
+
104
+ describe '2 event payloads with different metadata and different schema' do
105
+ let(:schema) { "dea_test" }
106
+ let(:config) do
107
+ <<-CONFIG
108
+ input {
109
+ generator {
110
+ lines => [
111
+ '{"app_name": "Web App", "event": "From Agent", "meta": { "type": "XML" } }',
112
+ '{"app_name": "Web App", "event": "From Agent", "meta": { "type": "XML", "file_name": "virus.pdf" } }'
113
+ ]
114
+ codec => 'json'
115
+ count => 1
116
+ }
117
+ }
118
+
119
+ output {
120
+ charrington {
121
+ connection_string => '#{@url}'
122
+ driver_jar_path => '#{driver_path}'
123
+ schema => '#{schema}'
124
+ }
125
+ }
126
+ CONFIG
127
+ end
128
+
129
+ it 'creates a table and inserts the first record and alters the table for the second record' do
130
+ create("CREATE SCHEMA IF NOT EXISTS #{schema}")
131
+ drop_table("#{schema}.from_agent")
132
+ run_pipeline
133
+ expect(query("SELECT * FROM #{schema}.from_agent")).to match_array([
134
+ {app_name: 'Web App', event: 'From Agent', id: '1', inserted_at: a_kind_of(String), meta_file_name: nil, meta_type: 'XML'},
135
+ {app_name: 'Web App', event: 'From Agent', id: '2', inserted_at: a_kind_of(String), meta_file_name: 'virus.pdf', meta_type: 'XML'}
136
+ ])
137
+ expect(query("SELECT COUNT(1) FROM #{schema}.from_agent").first[:count]).to eq("2")
138
+ end
139
+ end
140
+
62
141
  end
63
142
 
64
143
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-charrington
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.2
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - dconger
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2019-06-19 00:00:00.000000000 Z
13
+ date: 2019-07-27 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  requirement: !ruby/object:Gem::Requirement
@@ -35,45 +35,57 @@ dependencies:
35
35
  - !ruby/object:Gem::Dependency
36
36
  requirement: !ruby/object:Gem::Requirement
37
37
  requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '3.0'
38
41
  - - ">="
39
42
  - !ruby/object:Gem::Version
40
- version: '0'
43
+ version: 3.0.6
41
44
  name: logstash-codec-plain
42
45
  prerelease: false
43
46
  type: :runtime
44
47
  version_requirements: !ruby/object:Gem::Requirement
45
48
  requirements:
49
+ - - "~>"
50
+ - !ruby/object:Gem::Version
51
+ version: '3.0'
46
52
  - - ">="
47
53
  - !ruby/object:Gem::Version
48
- version: '0'
54
+ version: 3.0.6
49
55
  - !ruby/object:Gem::Dependency
50
56
  requirement: !ruby/object:Gem::Requirement
51
57
  requirements:
58
+ - - "~>"
59
+ - !ruby/object:Gem::Version
60
+ version: '1.3'
52
61
  - - ">="
53
62
  - !ruby/object:Gem::Version
54
- version: '0'
63
+ version: 1.3.1
55
64
  name: logstash-devutils
56
65
  prerelease: false
57
66
  type: :development
58
67
  version_requirements: !ruby/object:Gem::Requirement
59
68
  requirements:
69
+ - - "~>"
70
+ - !ruby/object:Gem::Version
71
+ version: '1.3'
60
72
  - - ">="
61
73
  - !ruby/object:Gem::Version
62
- version: '0'
74
+ version: 1.3.1
63
75
  - !ruby/object:Gem::Dependency
64
76
  requirement: !ruby/object:Gem::Requirement
65
77
  requirements:
66
- - - ">="
78
+ - - "~>"
67
79
  - !ruby/object:Gem::Version
68
- version: '0'
80
+ version: 0.4.0
69
81
  name: jar-dependencies
70
82
  prerelease: false
71
83
  type: :development
72
84
  version_requirements: !ruby/object:Gem::Requirement
73
85
  requirements:
74
- - - ">="
86
+ - - "~>"
75
87
  - !ruby/object:Gem::Version
76
- version: '0'
88
+ version: 0.4.0
77
89
  - !ruby/object:Gem::Dependency
78
90
  requirement: !ruby/object:Gem::Requirement
79
91
  requirements:
@@ -105,45 +117,57 @@ dependencies:
105
117
  - !ruby/object:Gem::Dependency
106
118
  requirement: !ruby/object:Gem::Requirement
107
119
  requirements:
120
+ - - "~>"
121
+ - !ruby/object:Gem::Version
122
+ version: '3.0'
108
123
  - - ">="
109
124
  - !ruby/object:Gem::Version
110
- version: '0'
125
+ version: 3.0.6
111
126
  name: logstash-input-generator
112
127
  prerelease: false
113
128
  type: :development
114
129
  version_requirements: !ruby/object:Gem::Requirement
115
130
  requirements:
131
+ - - "~>"
132
+ - !ruby/object:Gem::Version
133
+ version: '3.0'
116
134
  - - ">="
117
135
  - !ruby/object:Gem::Version
118
- version: '0'
136
+ version: 3.0.6
119
137
  - !ruby/object:Gem::Dependency
120
138
  requirement: !ruby/object:Gem::Requirement
121
139
  requirements:
140
+ - - "~>"
141
+ - !ruby/object:Gem::Version
142
+ version: '3.0'
122
143
  - - ">="
123
144
  - !ruby/object:Gem::Version
124
- version: '0'
145
+ version: 3.0.5
125
146
  name: logstash-codec-json
126
147
  prerelease: false
127
148
  type: :development
128
149
  version_requirements: !ruby/object:Gem::Requirement
129
150
  requirements:
151
+ - - "~>"
152
+ - !ruby/object:Gem::Version
153
+ version: '3.0'
130
154
  - - ">="
131
155
  - !ruby/object:Gem::Version
132
- version: '0'
156
+ version: 3.0.5
133
157
  - !ruby/object:Gem::Dependency
134
158
  requirement: !ruby/object:Gem::Requirement
135
159
  requirements:
136
- - - ">="
160
+ - - "~>"
137
161
  - !ruby/object:Gem::Version
138
- version: '0'
162
+ version: '1.0'
139
163
  name: insist
140
164
  prerelease: false
141
165
  type: :development
142
166
  version_requirements: !ruby/object:Gem::Requirement
143
167
  requirements:
144
- - - ">="
168
+ - - "~>"
145
169
  - !ruby/object:Gem::Version
146
- version: '0'
170
+ version: '1.0'
147
171
  description: This gem is a logstash plugin required to be installed on top of the
148
172
  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install 'logstash-output-charrington'.
149
173
  This gem is not a stand-alone program
@@ -158,28 +182,23 @@ files:
158
182
  - README.md
159
183
  - THANKS.md
160
184
  - lib/com/zaxxer/HikariCP/2.7.2/HikariCP-2.7.2.jar
161
- - lib/commons-io/commons-io/2.4/commons-io-2.4.jar
162
- - lib/de/flapdoodle/embed/de.flapdoodle.embed.process/2.0.2/de.flapdoodle.embed.process-2.0.2.jar
163
185
  - lib/logstash-output-charrington_jars.rb
164
186
  - lib/logstash/outputs/charrington.rb
165
- - lib/logstash/outputs/charrington/alter_table.rb
166
- - lib/logstash/outputs/charrington/create_table.rb
187
+ - lib/logstash/outputs/charrington/alter_postgres_table.rb
188
+ - lib/logstash/outputs/charrington/alter_redshift_table.rb
189
+ - lib/logstash/outputs/charrington/create_postgres_table.rb
190
+ - lib/logstash/outputs/charrington/create_redshift_table.rb
167
191
  - lib/logstash/outputs/charrington/insert.rb
168
192
  - lib/logstash/outputs/charrington/process.rb
169
193
  - lib/logstash/outputs/charrington/service.rb
170
- - lib/logstash/outputs/charrington/transform.rb
171
- - lib/net/java/dev/jna/jna-platform/4.0.0/jna-platform-4.0.0.jar
172
- - lib/net/java/dev/jna/jna/4.0.0/jna-4.0.0.jar
173
- - lib/org/apache/commons/commons-compress/1.10/commons-compress-1.10.jar
174
- - lib/org/apache/commons/commons-lang3/3.1/commons-lang3-3.1.jar
194
+ - lib/logstash/outputs/charrington/transform_postgres.rb
195
+ - lib/logstash/outputs/charrington/transform_redshift.rb
175
196
  - lib/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar
176
197
  - lib/org/apache/logging/log4j/log4j-slf4j-impl/2.6.2/log4j-slf4j-impl-2.6.2.jar
177
- - lib/org/postgresql/postgresql/42.2.5/postgresql-42.2.5.jar
178
198
  - lib/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar
179
- - lib/ru/yandex/qatools/embed/postgresql-embedded/2.10/postgresql-embedded-2.10.jar
180
- - lib/ru/yandex/qatools/embed/postgresql-embedded/2.8/postgresql-embedded-2.8.jar
181
199
  - logstash-output-charrington.gemspec
182
200
  - spec/charrington_spec_helper.rb
201
+ - spec/logstash-output-charrington_test_jars.rb
183
202
  - spec/outputs/charrington_mysql_spec.rb
184
203
  - spec/outputs/charrington_postgres_spec.rb
185
204
  - spec/outputs/charrington_spec.rb
@@ -187,11 +206,9 @@ files:
187
206
  - vendor/jar-dependencies/runtime-jars/log4j-api-2.6.2.jar
188
207
  - vendor/jar-dependencies/runtime-jars/log4j-slf4j-impl-2.6.2.jar
189
208
  - vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.25.jar
190
- - vendor/postgresql-42.2.5.jar
191
- - vendor/redshift.jar
192
- homepage:
209
+ homepage: https://gitlab.podium.com/engineering/analytics/logstash-output-charrington
193
210
  licenses:
194
- - Apache License (2.0)
211
+ - Apache-2.0
195
212
  metadata:
196
213
  logstash_plugin: 'true'
197
214
  logstash_group: output
@@ -219,6 +236,7 @@ specification_version: 4
219
236
  summary: This plugin allows you to output to SQL, via JDBC
220
237
  test_files:
221
238
  - spec/charrington_spec_helper.rb
239
+ - spec/logstash-output-charrington_test_jars.rb
222
240
  - spec/outputs/charrington_mysql_spec.rb
223
241
  - spec/outputs/charrington_postgres_spec.rb
224
242
  - spec/outputs/charrington_spec.rb
data/vendor/postgresql-42.2.5.jar DELETED
Binary file
data/vendor/redshift.jar DELETED
Binary file