logstash-output-charrington 0.2.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +38 -1
  3. data/lib/logstash/outputs/charrington.rb +32 -5
  4. data/lib/logstash/outputs/charrington/{alter_table.rb → alter_postgres_table.rb} +6 -5
  5. data/lib/logstash/outputs/charrington/alter_redshift_table.rb +109 -0
  6. data/lib/logstash/outputs/charrington/{create_table.rb → create_postgres_table.rb} +5 -4
  7. data/lib/logstash/outputs/charrington/create_redshift_table.rb +88 -0
  8. data/lib/logstash/outputs/charrington/insert.rb +27 -9
  9. data/lib/logstash/outputs/charrington/process.rb +8 -2
  10. data/lib/logstash/outputs/charrington/{transform.rb → transform_postgres.rb} +1 -1
  11. data/lib/logstash/outputs/charrington/transform_redshift.rb +102 -0
  12. data/logstash-output-charrington.gemspec +11 -9
  13. data/spec/charrington_spec_helper.rb +75 -188
  14. data/spec/logstash-output-charrington_test_jars.rb +5 -0
  15. data/spec/outputs/charrington_spec.rb +118 -39
  16. metadata +52 -34
  17. data/lib/commons-io/commons-io/2.4/commons-io-2.4.jar +0 -0
  18. data/lib/de/flapdoodle/embed/de.flapdoodle.embed.process/2.0.2/de.flapdoodle.embed.process-2.0.2.jar +0 -0
  19. data/lib/net/java/dev/jna/jna-platform/4.0.0/jna-platform-4.0.0.jar +0 -0
  20. data/lib/net/java/dev/jna/jna/4.0.0/jna-4.0.0.jar +0 -0
  21. data/lib/org/apache/commons/commons-compress/1.10/commons-compress-1.10.jar +0 -0
  22. data/lib/org/apache/commons/commons-lang3/3.1/commons-lang3-3.1.jar +0 -0
  23. data/lib/org/postgresql/postgresql/42.2.5/postgresql-42.2.5.jar +0 -0
  24. data/lib/ru/yandex/qatools/embed/postgresql-embedded/2.10/postgresql-embedded-2.10.jar +0 -0
  25. data/lib/ru/yandex/qatools/embed/postgresql-embedded/2.8/postgresql-embedded-2.8.jar +0 -0
  26. data/vendor/postgresql-42.2.5.jar +0 -0
  27. data/vendor/redshift.jar +0 -0
data/spec/logstash-output-charrington_test_jars.rb ADDED
@@ -0,0 +1,5 @@
1
+ # encoding: utf-8
2
+ require 'logstash/environment'
3
+
4
+ root_dir = File.expand_path(File.join(File.dirname(__FILE__), '..'))
5
+ LogStash::Environment.load_test_jars! File.join(root_dir, 'vendor')
data/spec/outputs/charrington_spec.rb CHANGED
@@ -1,44 +1,46 @@
1
1
  require_relative '../charrington_spec_helper'
2
- require 'insist'
3
2
 
4
3
  describe LogStash::Outputs::Charrington do
5
- describe 'when initializing' do
6
- it 'shouldn\'t register without a config' do
7
- expect do
8
- LogStash::Plugin.lookup('output', 'charrington').new
9
- end.to raise_error(LogStash::ConfigurationError)
4
+ include_context 'postgres'
5
+ include_context 'pipeline'
6
+
7
+ describe 'a new payload with one event' do
8
+ let(:config) do
9
+ <<-CONFIG
10
+ input {
11
+ generator {
12
+ message => '{"app_name": "Web App", "event": "From Agent", "meta": { "type": "XML" } }'
13
+ codec => 'json'
14
+ count => 1
15
+ }
16
+ }
17
+
18
+ output {
19
+ charrington {
20
+ connection_string => '#{@url}'
21
+ driver_jar_path => '#{driver_path}'
22
+ }
23
+ }
24
+ CONFIG
25
+ end
26
+
27
+ it 'creates a table and inserts a record' do
28
+ drop_table('from_agent')
29
+ run_pipeline
30
+ expect(query('SELECT * FROM from_agent')).to match_array([{id: "1", app_name: "Web App", event: "From Agent", inserted_at: a_kind_of(String), :meta_type => "XML"}])
31
+ expect(query('SELECT COUNT(1) FROM from_agent').first[:count]).to eq("1")
10
32
  end
11
33
  end
12
34
 
13
- # describe 'integration tests with agent' do
14
- # config <<-HEREDOC
15
- # input {
16
- # generator {
17
- # message => '{"app_name": "Web App", "event": "From Agent"}'
18
- # codec => 'json'
19
- # count => 1
20
- # }
21
- # }
22
-
23
- # output {
24
- # charrington {
25
- # connection_string => 'jdbc:postgresql://localhost:5432/winston?user=postgres&password=postgres'
26
- # driver_jar_path => '/projects/logstash-output-charrington/vendor/postgresql-42.2.5.jar'
27
- # schema => 'dea'
28
- # }
29
- # }
30
- # HEREDOC
31
-
32
- # agent do
33
- # puts "IT'S WORKING!!!!!"
34
- # end
35
- # end
36
-
37
- describe 'integration tests' do
38
- let(:config) do <<-CONFIG
35
+ describe '2 event payloads with different metadata' do
36
+ let(:config) do
37
+ <<-CONFIG
39
38
  input {
40
39
  generator {
41
- message => '{"app_name": "Web App", "event": "Hi - Dan"}'
40
+ lines => [
41
+ '{"app_name": "Web App", "event": "From Agent", "meta": { "type": "XML" } }',
42
+ '{"app_name": "Web App", "event": "From Agent", "meta": { "type": "XML", "file_name": "virus.pdf" } }'
43
+ ]
42
44
  codec => 'json'
43
45
  count => 1
44
46
  }
@@ -46,19 +48,96 @@ describe LogStash::Outputs::Charrington do
46
48
 
47
49
  output {
48
50
  charrington {
49
- connection_string => 'jdbc:postgresql://localhost:5432/winston?user=postgres&password=postgres'
50
- driver_jar_path => '/projects/logstash-output-charrington/vendor/postgresql-42.2.5.jar'
51
- schema => 'dea'
51
+ connection_string => '#{@url}'
52
+ driver_jar_path => '#{driver_path}'
52
53
  }
53
54
  }
54
55
  CONFIG
55
56
  end
56
57
 
57
- it("isn't effed") do
58
- pipeline = new_pipeline_from_string(config)
59
- pipeline.run
58
+ it 'creates a table and inserts the first record and alters the table for the second record' do
59
+ drop_table('from_agent')
60
+ run_pipeline
61
+ expect(query('SELECT * FROM from_agent')).to match_array([
62
+ {app_name: 'Web App', event: 'From Agent', id: '1', inserted_at: a_kind_of(String), meta_file_name: nil, meta_type: 'XML'},
63
+ {app_name: 'Web App', event: 'From Agent', id: '2', inserted_at: a_kind_of(String), meta_file_name: 'virus.pdf', meta_type: 'XML'}
64
+ ])
65
+ expect(query('SELECT COUNT(1) FROM from_agent').first[:count]).to eq("2")
60
66
  end
61
67
  end
68
+
69
+ describe 'event payload for an existing table with existing data' do
70
+ let(:config) do
71
+ <<-CONFIG
72
+ input {
73
+ generator {
74
+ message => '{"app_name": "Web App", "event": "From Agent", "meta": { "type": "XML" } }'
75
+ codec => 'json'
76
+ count => 1
77
+ }
78
+ }
79
+
80
+ output {
81
+ charrington {
82
+ connection_string => '#{@url}'
83
+ driver_jar_path => '#{driver_path}'
84
+ }
85
+ }
86
+ CONFIG
87
+ end
88
+
89
+ it 'can insert into an existing table' do
90
+ # setup pre-existing data
91
+ drop_table('from_agent')
92
+ create_table('CREATE TABLE from_agent (id SERIAL PRIMARY KEY, inserted_at TIMESTAMP DEFAULT NOW(), app_name VARCHAR(255))')
93
+ insert("INSERT INTO from_agent (app_name) VALUES ('Not Agent')")
94
+ expect(query('SELECT * FROM from_agent')).to match_array([{id: "1", inserted_at: a_kind_of(String), app_name: "Not Agent"}])
95
+
96
+ run_pipeline
97
+ expect(query('SELECT * FROM from_agent')).to match_array([
98
+ {id: '1', app_name: 'Not Agent', event: nil, inserted_at: a_kind_of(String), meta_type: nil},
99
+ {id: '2', app_name: 'Web App', event: 'From Agent', inserted_at: a_kind_of(String), meta_type: 'XML'}
100
+ ])
101
+ end
102
+ end
103
+
104
+ describe '2 event payloads with different metadata and different schema' do
105
+ let(:schema) { "dea_test" }
106
+ let(:config) do
107
+ <<-CONFIG
108
+ input {
109
+ generator {
110
+ lines => [
111
+ '{"app_name": "Web App", "event": "From Agent", "meta": { "type": "XML" } }',
112
+ '{"app_name": "Web App", "event": "From Agent", "meta": { "type": "XML", "file_name": "virus.pdf" } }'
113
+ ]
114
+ codec => 'json'
115
+ count => 1
116
+ }
117
+ }
118
+
119
+ output {
120
+ charrington {
121
+ connection_string => '#{@url}'
122
+ driver_jar_path => '#{driver_path}'
123
+ schema => '#{schema}'
124
+ }
125
+ }
126
+ CONFIG
127
+ end
128
+
129
+ it 'creates a table and inserts the first record and alters the table for the second record' do
130
+ create("CREATE SCHEMA IF NOT EXISTS #{schema}")
131
+ drop_table("#{schema}.from_agent")
132
+ run_pipeline
133
+ expect(query("SELECT * FROM #{schema}.from_agent")).to match_array([
134
+ {app_name: 'Web App', event: 'From Agent', id: '1', inserted_at: a_kind_of(String), meta_file_name: nil, meta_type: 'XML'},
135
+ {app_name: 'Web App', event: 'From Agent', id: '2', inserted_at: a_kind_of(String), meta_file_name: 'virus.pdf', meta_type: 'XML'}
136
+ ])
137
+ expect(query("SELECT COUNT(1) FROM #{schema}.from_agent").first[:count]).to eq("2")
138
+ end
139
+ end
140
+
62
141
  end
63
142
 
64
143
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-charrington
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.2
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - dconger
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2019-06-19 00:00:00.000000000 Z
13
+ date: 2019-07-27 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  requirement: !ruby/object:Gem::Requirement
@@ -35,45 +35,57 @@ dependencies:
35
35
  - !ruby/object:Gem::Dependency
36
36
  requirement: !ruby/object:Gem::Requirement
37
37
  requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '3.0'
38
41
  - - ">="
39
42
  - !ruby/object:Gem::Version
40
- version: '0'
43
+ version: 3.0.6
41
44
  name: logstash-codec-plain
42
45
  prerelease: false
43
46
  type: :runtime
44
47
  version_requirements: !ruby/object:Gem::Requirement
45
48
  requirements:
49
+ - - "~>"
50
+ - !ruby/object:Gem::Version
51
+ version: '3.0'
46
52
  - - ">="
47
53
  - !ruby/object:Gem::Version
48
- version: '0'
54
+ version: 3.0.6
49
55
  - !ruby/object:Gem::Dependency
50
56
  requirement: !ruby/object:Gem::Requirement
51
57
  requirements:
58
+ - - "~>"
59
+ - !ruby/object:Gem::Version
60
+ version: '1.3'
52
61
  - - ">="
53
62
  - !ruby/object:Gem::Version
54
- version: '0'
63
+ version: 1.3.1
55
64
  name: logstash-devutils
56
65
  prerelease: false
57
66
  type: :development
58
67
  version_requirements: !ruby/object:Gem::Requirement
59
68
  requirements:
69
+ - - "~>"
70
+ - !ruby/object:Gem::Version
71
+ version: '1.3'
60
72
  - - ">="
61
73
  - !ruby/object:Gem::Version
62
- version: '0'
74
+ version: 1.3.1
63
75
  - !ruby/object:Gem::Dependency
64
76
  requirement: !ruby/object:Gem::Requirement
65
77
  requirements:
66
- - - ">="
78
+ - - "~>"
67
79
  - !ruby/object:Gem::Version
68
- version: '0'
80
+ version: 0.4.0
69
81
  name: jar-dependencies
70
82
  prerelease: false
71
83
  type: :development
72
84
  version_requirements: !ruby/object:Gem::Requirement
73
85
  requirements:
74
- - - ">="
86
+ - - "~>"
75
87
  - !ruby/object:Gem::Version
76
- version: '0'
88
+ version: 0.4.0
77
89
  - !ruby/object:Gem::Dependency
78
90
  requirement: !ruby/object:Gem::Requirement
79
91
  requirements:
@@ -105,45 +117,57 @@ dependencies:
105
117
  - !ruby/object:Gem::Dependency
106
118
  requirement: !ruby/object:Gem::Requirement
107
119
  requirements:
120
+ - - "~>"
121
+ - !ruby/object:Gem::Version
122
+ version: '3.0'
108
123
  - - ">="
109
124
  - !ruby/object:Gem::Version
110
- version: '0'
125
+ version: 3.0.6
111
126
  name: logstash-input-generator
112
127
  prerelease: false
113
128
  type: :development
114
129
  version_requirements: !ruby/object:Gem::Requirement
115
130
  requirements:
131
+ - - "~>"
132
+ - !ruby/object:Gem::Version
133
+ version: '3.0'
116
134
  - - ">="
117
135
  - !ruby/object:Gem::Version
118
- version: '0'
136
+ version: 3.0.6
119
137
  - !ruby/object:Gem::Dependency
120
138
  requirement: !ruby/object:Gem::Requirement
121
139
  requirements:
140
+ - - "~>"
141
+ - !ruby/object:Gem::Version
142
+ version: '3.0'
122
143
  - - ">="
123
144
  - !ruby/object:Gem::Version
124
- version: '0'
145
+ version: 3.0.5
125
146
  name: logstash-codec-json
126
147
  prerelease: false
127
148
  type: :development
128
149
  version_requirements: !ruby/object:Gem::Requirement
129
150
  requirements:
151
+ - - "~>"
152
+ - !ruby/object:Gem::Version
153
+ version: '3.0'
130
154
  - - ">="
131
155
  - !ruby/object:Gem::Version
132
- version: '0'
156
+ version: 3.0.5
133
157
  - !ruby/object:Gem::Dependency
134
158
  requirement: !ruby/object:Gem::Requirement
135
159
  requirements:
136
- - - ">="
160
+ - - "~>"
137
161
  - !ruby/object:Gem::Version
138
- version: '0'
162
+ version: '1.0'
139
163
  name: insist
140
164
  prerelease: false
141
165
  type: :development
142
166
  version_requirements: !ruby/object:Gem::Requirement
143
167
  requirements:
144
- - - ">="
168
+ - - "~>"
145
169
  - !ruby/object:Gem::Version
146
- version: '0'
170
+ version: '1.0'
147
171
  description: This gem is a logstash plugin required to be installed on top of the
148
172
  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install 'logstash-output-charrington'.
149
173
  This gem is not a stand-alone program
@@ -158,28 +182,23 @@ files:
158
182
  - README.md
159
183
  - THANKS.md
160
184
  - lib/com/zaxxer/HikariCP/2.7.2/HikariCP-2.7.2.jar
161
- - lib/commons-io/commons-io/2.4/commons-io-2.4.jar
162
- - lib/de/flapdoodle/embed/de.flapdoodle.embed.process/2.0.2/de.flapdoodle.embed.process-2.0.2.jar
163
185
  - lib/logstash-output-charrington_jars.rb
164
186
  - lib/logstash/outputs/charrington.rb
165
- - lib/logstash/outputs/charrington/alter_table.rb
166
- - lib/logstash/outputs/charrington/create_table.rb
187
+ - lib/logstash/outputs/charrington/alter_postgres_table.rb
188
+ - lib/logstash/outputs/charrington/alter_redshift_table.rb
189
+ - lib/logstash/outputs/charrington/create_postgres_table.rb
190
+ - lib/logstash/outputs/charrington/create_redshift_table.rb
167
191
  - lib/logstash/outputs/charrington/insert.rb
168
192
  - lib/logstash/outputs/charrington/process.rb
169
193
  - lib/logstash/outputs/charrington/service.rb
170
- - lib/logstash/outputs/charrington/transform.rb
171
- - lib/net/java/dev/jna/jna-platform/4.0.0/jna-platform-4.0.0.jar
172
- - lib/net/java/dev/jna/jna/4.0.0/jna-4.0.0.jar
173
- - lib/org/apache/commons/commons-compress/1.10/commons-compress-1.10.jar
174
- - lib/org/apache/commons/commons-lang3/3.1/commons-lang3-3.1.jar
194
+ - lib/logstash/outputs/charrington/transform_postgres.rb
195
+ - lib/logstash/outputs/charrington/transform_redshift.rb
175
196
  - lib/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar
176
197
  - lib/org/apache/logging/log4j/log4j-slf4j-impl/2.6.2/log4j-slf4j-impl-2.6.2.jar
177
- - lib/org/postgresql/postgresql/42.2.5/postgresql-42.2.5.jar
178
198
  - lib/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar
179
- - lib/ru/yandex/qatools/embed/postgresql-embedded/2.10/postgresql-embedded-2.10.jar
180
- - lib/ru/yandex/qatools/embed/postgresql-embedded/2.8/postgresql-embedded-2.8.jar
181
199
  - logstash-output-charrington.gemspec
182
200
  - spec/charrington_spec_helper.rb
201
+ - spec/logstash-output-charrington_test_jars.rb
183
202
  - spec/outputs/charrington_mysql_spec.rb
184
203
  - spec/outputs/charrington_postgres_spec.rb
185
204
  - spec/outputs/charrington_spec.rb
@@ -187,11 +206,9 @@ files:
187
206
  - vendor/jar-dependencies/runtime-jars/log4j-api-2.6.2.jar
188
207
  - vendor/jar-dependencies/runtime-jars/log4j-slf4j-impl-2.6.2.jar
189
208
  - vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.25.jar
190
- - vendor/postgresql-42.2.5.jar
191
- - vendor/redshift.jar
192
- homepage:
209
+ homepage: https://gitlab.podium.com/engineering/analytics/logstash-output-charrington
193
210
  licenses:
194
- - Apache License (2.0)
211
+ - Apache-2.0
195
212
  metadata:
196
213
  logstash_plugin: 'true'
197
214
  logstash_group: output
@@ -219,6 +236,7 @@ specification_version: 4
219
236
  summary: This plugin allows you to output to SQL, via JDBC
220
237
  test_files:
221
238
  - spec/charrington_spec_helper.rb
239
+ - spec/logstash-output-charrington_test_jars.rb
222
240
  - spec/outputs/charrington_mysql_spec.rb
223
241
  - spec/outputs/charrington_postgres_spec.rb
224
242
  - spec/outputs/charrington_spec.rb
data/vendor/postgresql-42.2.5.jar DELETED
Binary file
data/vendor/redshift.jar DELETED
Binary file