logstash-output-charrington 0.3.20 → 0.3.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. checksums.yaml +4 -4
  2. data/lib/logstash/outputs/charrington/insert.rb +1 -1
  3. data/lib/org/apache/logging/log4j/log4j-api/2.16.0/log4j-api-2.16.0.jar +0 -0
  4. data/lib/org/apache/logging/log4j/log4j-core/2.16.0/log4j-core-2.16.0.jar +0 -0
  5. data/lib/org/apache/logging/log4j/log4j-slf4j-impl/2.16.0/log4j-slf4j-impl-2.16.0.jar +0 -0
  6. data/logstash-output-charrington.gemspec +5 -5
  7. data/spec/charrington_spec_helper.rb +1 -0
  8. data/spec/outputs/charrington_spec.rb +3 -3
  9. data/vendor/jar-dependencies/runtime-jars/log4j-api-2.16.0.jar +0 -0
  10. data/vendor/jar-dependencies/runtime-jars/log4j-core-2.16.0.jar +0 -0
  11. data/vendor/jar-dependencies/runtime-jars/log4j-slf4j-impl-2.16.0.jar +0 -0
  12. metadata +15 -32
  13. data/CHANGELOG.md +0 -64
  14. data/LICENSE.txt +0 -21
  15. data/README.md +0 -130
  16. data/THANKS.md +0 -18
  17. data/lib/commons-io/commons-io/2.4/commons-io-2.4.jar +0 -0
  18. data/lib/de/flapdoodle/embed/de.flapdoodle.embed.process/2.0.2/de.flapdoodle.embed.process-2.0.2.jar +0 -0
  19. data/lib/net/java/dev/jna/jna/4.0.0/jna-4.0.0.jar +0 -0
  20. data/lib/net/java/dev/jna/jna-platform/4.0.0/jna-platform-4.0.0.jar +0 -0
  21. data/lib/org/apache/commons/commons-compress/1.10/commons-compress-1.10.jar +0 -0
  22. data/lib/org/apache/commons/commons-lang3/3.1/commons-lang3-3.1.jar +0 -0
  23. data/lib/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar +0 -0
  24. data/lib/org/apache/logging/log4j/log4j-slf4j-impl/2.6.2/log4j-slf4j-impl-2.6.2.jar +0 -0
  25. data/lib/org/postgresql/postgresql/42.2.5/postgresql-42.2.5.jar +0 -0
  26. data/lib/ru/yandex/qatools/embed/postgresql-embedded/2.10/postgresql-embedded-2.10.jar +0 -0
  27. data/lib/ru/yandex/qatools/embed/postgresql-embedded/2.8/postgresql-embedded-2.8.jar +0 -0
  28. data/vendor/jar-dependencies/runtime-jars/log4j-api-2.6.2.jar +0 -0
  29. data/vendor/jar-dependencies/runtime-jars/log4j-slf4j-impl-2.6.2.jar +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 2ae644e2b642cfbf37d315289ff166d7b03e19865dd50320126e71f50954965c
4
- data.tar.gz: 5af164f7c8a73ed16001ef89922095bb0d37699fbc7c144c06eff4ab23e0481a
3
+ metadata.gz: da89e1fb9c647c2c2e05371870f6960c50fbb0a31a7207cebf404161f5ea0ac8
4
+ data.tar.gz: 1932449a46a3d1c1c8c26fd80f8150a49fc9e5535fa6b500487a9ebb284f5492
5
5
  SHA512:
6
- metadata.gz: 5f6583538a0632625e50bb48e6b48864455c2edb0987d434bda4f31edd0479b4805e72974feb0213496f6db77bb6e133661e09d448ac4423cd69bce5ddf07c9e
7
- data.tar.gz: 7f3cf086ba6be74f724beed922eaf3fc8d8dc8dcabf24a599cce0ac5acbc1cc8ae927d0d10f0da8e287c00145cdbaa770141d7608e79966dcfb521bdde91c76e
6
+ metadata.gz: fe8fc4626d86739e54e8c4bc951417dcf5111ad653b7f5ee11b268566284f95ce537b6ae764b520d18cda43368f96b2ee973e6bc96934ee0e885320b854398bb
7
+ data.tar.gz: '01392652632ebff7ea339b3a0da68df0997ae90d1766667f8cbed0ece191787101257adc4d4e4ad624c737c97c84dc9dffb02b6eec5c4c368e4ea84523f979ef'
@@ -16,7 +16,7 @@ module Charrington
16
16
  attr_reader :connection, :schema, :table_name, :columns, :driver, :opts, :transformer, :tracks
17
17
  attr_reader :event_as_json_keyword, :enable_event_as_json_keyword
18
18
  # TODO create a current_table_columns (alter_postgres_table.rb) query on the tracks table to get the current columns
19
- @@redshift_tracks_columns = %w(id app_name received_at uuid uuid_ts anonymous_id context_ip context_library_name context_library_version context_page_path context_page_referrer context_page_title context_page_url context_user_agent event event_text original_timestamp sent_at timestamp user_id user_uid context_campaign_medium context_campaign_name context_page_search context_campaign_source segment_dedupe_id context_campaign_content)
19
+ @@redshift_tracks_columns = %w(id action app_name received_at uuid uuid_ts anonymous_id context_ip context_library_name context_library_version context_page_path context_page_referrer context_page_title context_page_url context_user_agent event event_text original_timestamp sent_at timestamp user_id user_uid context_campaign_medium context_campaign_name context_page_search context_campaign_source segment_dedupe_id context_campaign_content)
20
20
  @@postgres_tracks_columns = %w(anonymous_user app_name event published_at session_ip session_library_name session_library_version session_page_path session_page_referrer session_page_search session_page_title session_page_url session_user_agent user_id user_uid)
21
21
  @@timestamp_columns = %w(published_at sent_at original_timestamp received_at timestamp)
22
22
 
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-charrington'
3
- s.version = '0.3.20'
3
+ s.version = '0.3.24'
4
4
 
5
5
  s.licenses = ['Apache-2.0']
6
6
  s.homepage = 'https://gitlab.podium.com/engineering/analytics/logstash-output-charrington'
@@ -24,11 +24,11 @@ Gem::Specification.new do |s|
24
24
  # The 'install_jars' rake task will download these jars from Maven and put them into the vendor directory
25
25
  # See jar-dependencies gem's wiki - https://github.com/mkristian/jar-dependencies/wiki/declare-jars-inside-gemspec
26
26
  s.requirements << "jar 'com.zaxxer:HikariCP', '2.7.2'"
27
- s.requirements << "jar 'org.apache.logging.log4j:log4j-slf4j-impl', '2.6.2'"
27
+ s.requirements << "jar 'org.apache.logging.log4j:log4j-slf4j-impl', '2.16.0'"
28
28
 
29
- s.add_development_dependency 'logstash-devutils', '~> 1.3', '>= 1.3.1'
30
- s.add_development_dependency 'jar-dependencies', '~> 0.4.0'
31
- s.add_development_dependency 'ruby-maven', '~> 3.3'
29
+ s.add_development_dependency 'logstash-devutils'
30
+ s.add_development_dependency 'jar-dependencies', '~> 0.4.1'
31
+ s.add_development_dependency 'ruby-maven', '~> 3.3.12'
32
32
  s.add_development_dependency 'rubocop', '0.41.2'
33
33
  s.add_development_dependency 'logstash-input-generator', '~> 3.0', '>= 3.0.6'
34
34
  s.add_development_dependency 'logstash-codec-json', '~> 3.0', '>= 3.0.5'
@@ -93,6 +93,7 @@ RSpec.shared_context 'postgres' do
93
93
  CREATE TABLE IF NOT EXISTS #{ schema.empty? ? schema : schema + '.'}tracks
94
94
  (
95
95
  id VARCHAR(512) NOT NULL CONSTRAINT tracks_pkey PRIMARY KEY,
96
+ action VARCHAR(512),
96
97
  anonymous_id VARCHAR(512),
97
98
  app_name VARCHAR(512),
98
99
  context_campaign_content VARCHAR(512),
@@ -221,7 +221,7 @@ describe LogStash::Outputs::Charrington do
221
221
  <<-CONFIG
222
222
  input {
223
223
  generator {
224
- message => '{"app_name": "Web App", "event": "schemaless", "meta": { "type": "XML" }, "published_at": "2019-07-29T20:09:18Z", "user_id": "123", "user_uid": "456" }'
224
+ message => '{"app_name": "Web App", "action": "click", "event": "schemaless", "meta": { "type": "XML" }, "published_at": "2019-07-29T20:09:18Z", "user_id": "123", "user_uid": "456" }'
225
225
  codec => 'json'
226
226
  count => 1
227
227
  }
@@ -244,11 +244,11 @@ describe LogStash::Outputs::Charrington do
244
244
  drop_table('schemaless')
245
245
  run_pipeline
246
246
  expect(query('SELECT * FROM schemaless')).to match_array([
247
- {:anonymous_id=>"", :app_name=>"Web App", :event=>"schemaless", :event_text=>"schemaless", :id=>a_kind_of(String), :original_timestamp=>a_kind_of(String), :received_at=>a_kind_of(String), :sent_at=>"2019-07-29 20:09:18", :timestamp=>a_kind_of(String), :type=>"XML", :user_id=>"123", :user_uid=>"456", :uuid_ts=>a_kind_of(String)}
247
+ {:anonymous_id=>"", :action=>"click", :app_name=>"Web App", :event=>"schemaless", :event_text=>"schemaless", :id=>a_kind_of(String), :original_timestamp=>a_kind_of(String), :received_at=>a_kind_of(String), :sent_at=>"2019-07-29 20:09:18", :timestamp=>a_kind_of(String), :type=>"XML", :user_id=>"123", :user_uid=>"456", :uuid_ts=>a_kind_of(String)}
248
248
  ])
249
249
  expect(query('SELECT COUNT(1) FROM schemaless').first[:count]).to eq("1")
250
250
  expect(query("SELECT * FROM tracks")).to match_array([
251
- {:id=>a_kind_of(String), :anonymous_id=>"", :app_name=>"Web App", :context_campaign_content=>nil, :context_campaign_medium=>nil, :context_campaign_name=>nil, :context_campaign_source=>nil, :context_ip=>nil, :context_library_name=>nil, :context_library_version=>nil, :context_page_path=>nil, :context_page_referrer=>nil, :context_page_search=>nil, :context_page_title=>nil, :context_page_url=>nil, :context_user_agent=>nil, :event=>"schemaless", :event_text=>"schemaless", :original_timestamp=>"2019-07-29 20:09:18", :received_at=>"2019-07-29 20:09:18", :segment_dedupe_id=>nil, :sent_at=>"2019-07-29 20:09:18", :timestamp=>"2019-07-29 20:09:18", :user_id=>"123", :user_uid=>"456", :uuid=>nil, :uuid_ts=>a_kind_of(String)}
251
+ {:id=>a_kind_of(String), :action=>"click", :anonymous_id=>"", :app_name=>"Web App", :context_campaign_content=>nil, :context_campaign_medium=>nil, :context_campaign_name=>nil, :context_campaign_source=>nil, :context_ip=>nil, :context_library_name=>nil, :context_library_version=>nil, :context_page_path=>nil, :context_page_referrer=>nil, :context_page_search=>nil, :context_page_title=>nil, :context_page_url=>nil, :context_user_agent=>nil, :event=>"schemaless", :event_text=>"schemaless", :original_timestamp=>"2019-07-29 20:09:18", :received_at=>"2019-07-29 20:09:18", :segment_dedupe_id=>nil, :sent_at=>"2019-07-29 20:09:18", :timestamp=>"2019-07-29 20:09:18", :user_id=>"123", :user_uid=>"456", :uuid=>nil, :uuid_ts=>a_kind_of(String)}
252
252
  ])
253
253
  end
254
254
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-charrington
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.20
4
+ version: 0.3.24
5
5
  platform: ruby
6
6
  authors:
7
7
  - dconger
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2019-08-29 00:00:00.000000000 Z
13
+ date: 2021-12-15 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  requirement: !ruby/object:Gem::Requirement
@@ -55,29 +55,23 @@ dependencies:
55
55
  - !ruby/object:Gem::Dependency
56
56
  requirement: !ruby/object:Gem::Requirement
57
57
  requirements:
58
- - - "~>"
59
- - !ruby/object:Gem::Version
60
- version: '1.3'
61
58
  - - ">="
62
59
  - !ruby/object:Gem::Version
63
- version: 1.3.1
60
+ version: '0'
64
61
  name: logstash-devutils
65
62
  prerelease: false
66
63
  type: :development
67
64
  version_requirements: !ruby/object:Gem::Requirement
68
65
  requirements:
69
- - - "~>"
70
- - !ruby/object:Gem::Version
71
- version: '1.3'
72
66
  - - ">="
73
67
  - !ruby/object:Gem::Version
74
- version: 1.3.1
68
+ version: '0'
75
69
  - !ruby/object:Gem::Dependency
76
70
  requirement: !ruby/object:Gem::Requirement
77
71
  requirements:
78
72
  - - "~>"
79
73
  - !ruby/object:Gem::Version
80
- version: 0.4.0
74
+ version: 0.4.1
81
75
  name: jar-dependencies
82
76
  prerelease: false
83
77
  type: :development
@@ -85,13 +79,13 @@ dependencies:
85
79
  requirements:
86
80
  - - "~>"
87
81
  - !ruby/object:Gem::Version
88
- version: 0.4.0
82
+ version: 0.4.1
89
83
  - !ruby/object:Gem::Dependency
90
84
  requirement: !ruby/object:Gem::Requirement
91
85
  requirements:
92
86
  - - "~>"
93
87
  - !ruby/object:Gem::Version
94
- version: '3.3'
88
+ version: 3.3.12
95
89
  name: ruby-maven
96
90
  prerelease: false
97
91
  type: :development
@@ -99,7 +93,7 @@ dependencies:
99
93
  requirements:
100
94
  - - "~>"
101
95
  - !ruby/object:Gem::Version
102
- version: '3.3'
96
+ version: 3.3.12
103
97
  - !ruby/object:Gem::Dependency
104
98
  requirement: !ruby/object:Gem::Requirement
105
99
  requirements:
@@ -176,14 +170,8 @@ executables: []
176
170
  extensions: []
177
171
  extra_rdoc_files: []
178
172
  files:
179
- - CHANGELOG.md
180
173
  - Gemfile
181
- - LICENSE.txt
182
- - README.md
183
- - THANKS.md
184
174
  - lib/com/zaxxer/HikariCP/2.7.2/HikariCP-2.7.2.jar
185
- - lib/commons-io/commons-io/2.4/commons-io-2.4.jar
186
- - lib/de/flapdoodle/embed/de.flapdoodle.embed.process/2.0.2/de.flapdoodle.embed.process-2.0.2.jar
187
175
  - lib/logstash-output-charrington_jars.rb
188
176
  - lib/logstash/outputs/charrington.rb
189
177
  - lib/logstash/outputs/charrington/alter_postgres_table.rb
@@ -195,16 +183,10 @@ files:
195
183
  - lib/logstash/outputs/charrington/service.rb
196
184
  - lib/logstash/outputs/charrington/transform_postgres.rb
197
185
  - lib/logstash/outputs/charrington/transform_redshift.rb
198
- - lib/net/java/dev/jna/jna-platform/4.0.0/jna-platform-4.0.0.jar
199
- - lib/net/java/dev/jna/jna/4.0.0/jna-4.0.0.jar
200
- - lib/org/apache/commons/commons-compress/1.10/commons-compress-1.10.jar
201
- - lib/org/apache/commons/commons-lang3/3.1/commons-lang3-3.1.jar
202
- - lib/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar
203
- - lib/org/apache/logging/log4j/log4j-slf4j-impl/2.6.2/log4j-slf4j-impl-2.6.2.jar
204
- - lib/org/postgresql/postgresql/42.2.5/postgresql-42.2.5.jar
186
+ - lib/org/apache/logging/log4j/log4j-api/2.16.0/log4j-api-2.16.0.jar
187
+ - lib/org/apache/logging/log4j/log4j-core/2.16.0/log4j-core-2.16.0.jar
188
+ - lib/org/apache/logging/log4j/log4j-slf4j-impl/2.16.0/log4j-slf4j-impl-2.16.0.jar
205
189
  - lib/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar
206
- - lib/ru/yandex/qatools/embed/postgresql-embedded/2.10/postgresql-embedded-2.10.jar
207
- - lib/ru/yandex/qatools/embed/postgresql-embedded/2.8/postgresql-embedded-2.8.jar
208
190
  - logstash-output-charrington.gemspec
209
191
  - spec/charrington_spec_helper.rb
210
192
  - spec/logstash-output-charrington_test_jars.rb
@@ -212,8 +194,9 @@ files:
212
194
  - spec/outputs/charrington_postgres_spec.rb
213
195
  - spec/outputs/charrington_spec.rb
214
196
  - vendor/jar-dependencies/runtime-jars/HikariCP-2.7.2.jar
215
- - vendor/jar-dependencies/runtime-jars/log4j-api-2.6.2.jar
216
- - vendor/jar-dependencies/runtime-jars/log4j-slf4j-impl-2.6.2.jar
197
+ - vendor/jar-dependencies/runtime-jars/log4j-api-2.16.0.jar
198
+ - vendor/jar-dependencies/runtime-jars/log4j-core-2.16.0.jar
199
+ - vendor/jar-dependencies/runtime-jars/log4j-slf4j-impl-2.16.0.jar
217
200
  - vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.25.jar
218
201
  homepage: https://gitlab.podium.com/engineering/analytics/logstash-output-charrington
219
202
  licenses:
@@ -237,7 +220,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
237
220
  version: '0'
238
221
  requirements:
239
222
  - jar 'com.zaxxer:HikariCP', '2.7.2'
240
- - jar 'org.apache.logging.log4j:log4j-slf4j-impl', '2.6.2'
223
+ - jar 'org.apache.logging.log4j:log4j-slf4j-impl', '2.16.0'
241
224
  rubyforge_project:
242
225
  rubygems_version: 2.7.6
243
226
  signing_key:
data/CHANGELOG.md DELETED
@@ -1,64 +0,0 @@
1
- # Change Log
2
- All notable changes to this project will be documented in this file, from 0.2.0.
3
-
4
- ## [5.3.0] - 2017-11-08
5
- - Adds configuration options `enable_event_as_json_keyword` and `event_as_json_keyword`
6
- - Adds BigDecimal support
7
- - Adds additional logging for debugging purposes (with thanks to @mlkmhd's work)
8
-
9
- ## [5.2.1] - 2017-04-09
10
- - Adds Array and Hash to_json support for non-sprintf syntax
11
-
12
- ## [5.2.0] - 2017-04-01
13
- - Upgrades HikariCP to latest
14
- - Fixes HikariCP logging integration issues
15
-
16
- ## [5.1.0] - 2016-12-17
17
- - phoenix-thin fixes for issue #60
18
-
19
- ## [5.0.0] - 2016-11-03
20
- - logstash v5 support
21
-
22
- ## [0.3.1] - 2016-08-28
23
- - Adds connection_test configuration option, to prevent the connection test from occuring, allowing the error to be suppressed.
24
- Useful for cockroachdb deployments. https://github.com/theangryangel/logstash-output-jdbc/issues/53
25
-
26
- ## [0.3.0] - 2016-07-24
27
- - Brings tests from v5 branch, providing greater coverage
28
- - Removes bulk update support, due to inconsistent behaviour
29
- - Plugin now marked as threadsafe, meaning only 1 instance per-Logstash
30
- - Raises default max_pool_size to match the default number of workers (1 connection per worker)
31
-
32
- ## [0.2.10] - 2016-07-07
33
- - Support non-string entries in statement array
34
- - Adds backtrace to exception logging
35
-
36
- ## [0.2.9] - 2016-06-29
37
- - Fix NameError exception.
38
- - Moved log_jdbc_exception calls
39
-
40
- ## [0.2.7] - 2016-05-29
41
- - Backport retry exception logic from v5 branch
42
- - Backport improved timestamp compatibility from v5 branch
43
-
44
- ## [0.2.6] - 2016-05-02
45
- - Fix for exception infinite loop
46
-
47
- ## [0.2.5] - 2016-04-11
48
- ### Added
49
- - Basic tests running against DerbyDB
50
- - Fix for converting Logstash::Timestamp to iso8601 from @hordijk
51
-
52
- ## [0.2.4] - 2016-04-07
53
- - Documentation fixes from @hordijk
54
-
55
- ## [0.2.3] - 2016-02-16
56
- - Bug fixes
57
-
58
- ## [0.2.2] - 2015-12-30
59
- - Bug fixes
60
-
61
- ## [0.2.1] - 2015-12-22
62
- - Support for connection pooling support added through HikariCP
63
- - Support for unsafe statement handling (allowing dynamic queries)
64
- - Altered exception handling to now count sequential flushes with exceptions thrown
data/LICENSE.txt DELETED
@@ -1,21 +0,0 @@
1
- The MIT License (MIT)
2
-
3
- Copyright (c) 2014
4
-
5
- Permission is hereby granted, free of charge, to any person obtaining a copy
6
- of this software and associated documentation files (the "Software"), to deal
7
- in the Software without restriction, including without limitation the rights
8
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
- copies of the Software, and to permit persons to whom the Software is
10
- furnished to do so, subject to the following conditions:
11
-
12
- The above copyright notice and this permission notice shall be included in all
13
- copies or substantial portions of the Software.
14
-
15
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
- SOFTWARE.
data/README.md DELETED
@@ -1,130 +0,0 @@
1
- # Logstash Output Charrington
2
-
3
- Logstash Output Plugin to handle batching messages and dynamically create and alter tables.
4
-
5
- Forked From [logstash-output-jdbc](https://github.com/theangryangel/logstash-output-jdbc)
6
- ## Configure
7
- Example Logstash Config
8
- ```bash
9
- input {
10
- kafka {
11
- auto_offset_reset => earliest
12
- bootstrap_servers => "${LS_KAFKA_BOOTSTRAP_SERVERS:kafka:9092}"
13
- codec => json
14
- group_id => "winston_ls_pg_sinks_cg"
15
- topics => [
16
- "orwell.analytics.v1.json"
17
- ]
18
- }
19
- }
20
-
21
- filter {
22
- if "_jsonparsefailure" in [tags] {
23
- drop { }
24
- }
25
-
26
- ruby {
27
- code => "
28
- hash = event.to_hash
29
- hash.each do |k,v|
30
- if v == nil
31
- event.remove(k)
32
- end
33
- end
34
- "
35
- }
36
- }
37
-
38
- output {
39
- charrington {
40
- driver_jar_path => "${DRIVER_PATH:/usr/share/logstash/vendor/jar/jdbc/postgresql-42.2.5.jar}"
41
- connection_string => "jdbc:postgresql://${WINSTON_DATABASE_HOST}:5432/${WINSTON_DATABASE}?user=${WINSTON_DATABASE_USERNAME}&password=${WINSTON_DATABASE_PASSWORD}"
42
- schema => "dea"
43
- }
44
-
45
- if "${LS_SHOW_DEBUG_OUTPUT}" == "true" {
46
- stdout { codec => rubydebug }
47
- }
48
- }
49
- ```
50
-
51
- ## Build & Publish
52
-
53
- **TLDR**
54
- ```bash
55
- bundle exec rake build # build gem
56
- gem push logstash-output-charrington-x.x.x.gem # publish
57
- ```
58
-
59
- Everything in the [vendor](./vendor) directory (including `test-jars` if it exists) will be packaged with the generated gem.
60
- Avoid including unnecessary dependencies, by removing the vendor directory and regenerating it with the rake task.
61
- See the first command below:
62
-
63
- ```bash
64
- rm -r vendor && bundle exec rake install_jars # clean vendor directory and re-install runtime jar dependencies
65
- gem build logstash-output-charrington.gemspec # build
66
- gem push logstash-output-charrington-x.x.x.gem # publish
67
- gem owner --add <email> logstash-output-charrington # add another authorized publisher
68
- ```
69
-
70
- ## Testing
71
- **NOTE:** Downloading test_jars requires maven and Java. You can set them up with asdf: [asdf-java](https://github.com/skotchpine/asdf-java), [asdf-maven](https://github.com/skotchpine/asdf-maven)
72
- * There is currently an [open bug](https://github.com/skotchpine/asdf-maven/pull/17) with how `asdf-maven` sets `$JAVA_HOME` on a mac. Edit your `~/.asdf/plugins/maven/bin/exec-env` file to fix the bug
73
-
74
- ```bash
75
- bundle exec rake install_test_jars # install test runtime jar dependencies
76
- bundle exec rspec
77
- ```
78
-
79
- #### Dangling Embedded Postgres
80
- **TLDR**
81
- ```bash
82
- lsof -PiTCP -sTCP:LISTEN | grep postgres | awk '{print $2}' | xargs kill
83
- ```
84
- Sometimes the embedded postgres JAR can leave dangling open processes. This is obvious when you have `export TEST_DEBUG=true`
85
- and the test output contains a stacktrace that starts with `[2019-06-21T15:18:06,734][ERROR][ru.yandex.qatools.embed.postgresql.PostgresProcess] Failed to read PID file (File '/var/folders/...`.
86
- To resolve this issue, check for dangling processes using `lsof -PiTCP -sTCP:LISTEN | grep postgres`, then you can kill the process ids
87
-
88
- #### Tracks Table
89
- **Create Statment for Driver is Postgres and Transform is Redshift**
90
- * See `@@redshift_tracks_columns` in [insert.rb](./lib/logstash/outputs/charrington/insert.rb) for column list
91
- ```sql
92
- create table <schema>.tracks
93
- (
94
- id varchar(512) not null constraint tracks_pkey primary key,
95
- anonymous_id varchar(512),
96
- app_name varchar(512),
97
- context_campaign_content varchar(512)
98
- context_campaign_medium varchar(512),
99
- context_campaign_name varchar(512),
100
- context_campaign_source varchar(512),
101
- context_ip varchar(512),
102
- context_library_name varchar(512),
103
- context_library_version varchar(512),
104
- context_page_path varchar(512),
105
- context_page_referrer varchar(512),
106
- context_page_search varchar(512),
107
- context_page_title varchar(512),
108
- context_page_url varchar(512),
109
- context_user_agent varchar(512),
110
- event varchar(512),
111
- event_text varchar(512),
112
- original_timestamp timestamp,
113
- received_at timestamp,
114
- segment_dedupe_id varchar(512),
115
- sent_at timestamp,
116
- timestamp timestamp,
117
- user_id varchar(512),
118
- user_uid varchar(512),
119
- uuid bigint,
120
- uuid_ts timestamp default ('now'::text)::timestamp without time zone,
121
- )
122
-
123
- ```
124
-
125
- #### Resources
126
- * [logstash-devutils](https://rubygems.org/gems/logstash-devutils) gem with logstash helper methods and tools
127
- * [spec_helper.rb source](https://github.com/elastic/logstash-devutils/blob/master/lib/logstash/devutils/rspec/spec_helper.rb)
128
- * [logstash_helpers.rb source](https://github.com/elastic/logstash-devutils/blob/master/lib/logstash/devutils/rspec/logstash_helpers.rb)
129
- * [logstash-output-file spec test](https://github.com/logstash-plugins/logstash-output-file/blob/master/spec/outputs/file_spec.rb)
130
-
data/THANKS.md DELETED
@@ -1,18 +0,0 @@
1
- logstash-output-jdbc is a project originally created by Karl Southern
2
- (the_angry_angel), but there are a number of people that have contributed
3
- or implemented key features over time. We do our best to keep this list
4
- up-to-date, but you can also have a look at the nice contributor graphs
5
- produced by GitHub: https://github.com/theangryangel/logstash-output-jdbc/graphs/contributors
6
-
7
- * [hordijk](https://github.com/hordijk)
8
- * [dmitryakadiamond](https://github.com/dmitryakadiamond)
9
- * [MassimoSporchia](https://github.com/MassimoSporchia)
10
- * [ebuildy](https://github.com/ebuildy)
11
- * [kushtrimjunuzi](https://github.com/kushtrimjunuzi)
12
- * [josemazo](https://github.com/josemazo)
13
- * [aceoliver](https://github.com/aceoliver)
14
- * [roflmao](https://github.com/roflmao)
15
- * [onesuper](https://github.com/onesuper)
16
- * [phr0gz](https://github.com/phr0gz)
17
- * [jMonsinjon](https://github.com/jMonsinjon)
18
- * [mlkmhd](https://github.com/mlkmhd)