logstash-output-charrington 0.3.20 → 0.3.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/logstash/outputs/charrington/insert.rb +1 -1
- data/lib/org/apache/logging/log4j/log4j-api/2.16.0/log4j-api-2.16.0.jar +0 -0
- data/lib/org/apache/logging/log4j/log4j-core/2.16.0/log4j-core-2.16.0.jar +0 -0
- data/lib/org/apache/logging/log4j/log4j-slf4j-impl/2.16.0/log4j-slf4j-impl-2.16.0.jar +0 -0
- data/logstash-output-charrington.gemspec +5 -5
- data/spec/charrington_spec_helper.rb +1 -0
- data/spec/outputs/charrington_spec.rb +3 -3
- data/vendor/jar-dependencies/runtime-jars/log4j-api-2.16.0.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/log4j-core-2.16.0.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/log4j-slf4j-impl-2.16.0.jar +0 -0
- metadata +15 -32
- data/CHANGELOG.md +0 -64
- data/LICENSE.txt +0 -21
- data/README.md +0 -130
- data/THANKS.md +0 -18
- data/lib/commons-io/commons-io/2.4/commons-io-2.4.jar +0 -0
- data/lib/de/flapdoodle/embed/de.flapdoodle.embed.process/2.0.2/de.flapdoodle.embed.process-2.0.2.jar +0 -0
- data/lib/net/java/dev/jna/jna/4.0.0/jna-4.0.0.jar +0 -0
- data/lib/net/java/dev/jna/jna-platform/4.0.0/jna-platform-4.0.0.jar +0 -0
- data/lib/org/apache/commons/commons-compress/1.10/commons-compress-1.10.jar +0 -0
- data/lib/org/apache/commons/commons-lang3/3.1/commons-lang3-3.1.jar +0 -0
- data/lib/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar +0 -0
- data/lib/org/apache/logging/log4j/log4j-slf4j-impl/2.6.2/log4j-slf4j-impl-2.6.2.jar +0 -0
- data/lib/org/postgresql/postgresql/42.2.5/postgresql-42.2.5.jar +0 -0
- data/lib/ru/yandex/qatools/embed/postgresql-embedded/2.10/postgresql-embedded-2.10.jar +0 -0
- data/lib/ru/yandex/qatools/embed/postgresql-embedded/2.8/postgresql-embedded-2.8.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/log4j-api-2.6.2.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/log4j-slf4j-impl-2.6.2.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: da89e1fb9c647c2c2e05371870f6960c50fbb0a31a7207cebf404161f5ea0ac8
|
4
|
+
data.tar.gz: 1932449a46a3d1c1c8c26fd80f8150a49fc9e5535fa6b500487a9ebb284f5492
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fe8fc4626d86739e54e8c4bc951417dcf5111ad653b7f5ee11b268566284f95ce537b6ae764b520d18cda43368f96b2ee973e6bc96934ee0e885320b854398bb
|
7
|
+
data.tar.gz: '01392652632ebff7ea339b3a0da68df0997ae90d1766667f8cbed0ece191787101257adc4d4e4ad624c737c97c84dc9dffb02b6eec5c4c368e4ea84523f979ef'
|
@@ -16,7 +16,7 @@ module Charrington
|
|
16
16
|
attr_reader :connection, :schema, :table_name, :columns, :driver, :opts, :transformer, :tracks
|
17
17
|
attr_reader :event_as_json_keyword, :enable_event_as_json_keyword
|
18
18
|
# TODO create a current_table_columns (alter_postgres_table.rb) query on the tracks table to get the current columns
|
19
|
-
@@redshift_tracks_columns = %w(id app_name received_at uuid uuid_ts anonymous_id context_ip context_library_name context_library_version context_page_path context_page_referrer context_page_title context_page_url context_user_agent event event_text original_timestamp sent_at timestamp user_id user_uid context_campaign_medium context_campaign_name context_page_search context_campaign_source segment_dedupe_id context_campaign_content)
|
19
|
+
@@redshift_tracks_columns = %w(id action app_name received_at uuid uuid_ts anonymous_id context_ip context_library_name context_library_version context_page_path context_page_referrer context_page_title context_page_url context_user_agent event event_text original_timestamp sent_at timestamp user_id user_uid context_campaign_medium context_campaign_name context_page_search context_campaign_source segment_dedupe_id context_campaign_content)
|
20
20
|
@@postgres_tracks_columns = %w(anonymous_user app_name event published_at session_ip session_library_name session_library_version session_page_path session_page_referrer session_page_search session_page_title session_page_url session_user_agent user_id user_uid)
|
21
21
|
@@timestamp_columns = %w(published_at sent_at original_timestamp received_at timestamp)
|
22
22
|
|
Binary file
|
Binary file
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-charrington'
|
3
|
-
s.version = '0.3.
|
3
|
+
s.version = '0.3.24'
|
4
4
|
|
5
5
|
s.licenses = ['Apache-2.0']
|
6
6
|
s.homepage = 'https://gitlab.podium.com/engineering/analytics/logstash-output-charrington'
|
@@ -24,11 +24,11 @@ Gem::Specification.new do |s|
|
|
24
24
|
# The 'install_jars' rake task will download these jars from Maven and put them into the vendor directory
|
25
25
|
# See jar-dependencies gem's wiki - https://github.com/mkristian/jar-dependencies/wiki/declare-jars-inside-gemspec
|
26
26
|
s.requirements << "jar 'com.zaxxer:HikariCP', '2.7.2'"
|
27
|
-
s.requirements << "jar 'org.apache.logging.log4j:log4j-slf4j-impl', '2.
|
27
|
+
s.requirements << "jar 'org.apache.logging.log4j:log4j-slf4j-impl', '2.16.0'"
|
28
28
|
|
29
|
-
s.add_development_dependency 'logstash-devutils'
|
30
|
-
s.add_development_dependency 'jar-dependencies', '~> 0.4.
|
31
|
-
s.add_development_dependency 'ruby-maven', '~> 3.3'
|
29
|
+
s.add_development_dependency 'logstash-devutils'
|
30
|
+
s.add_development_dependency 'jar-dependencies', '~> 0.4.1'
|
31
|
+
s.add_development_dependency 'ruby-maven', '~> 3.3.12'
|
32
32
|
s.add_development_dependency 'rubocop', '0.41.2'
|
33
33
|
s.add_development_dependency 'logstash-input-generator', '~> 3.0', '>= 3.0.6'
|
34
34
|
s.add_development_dependency 'logstash-codec-json', '~> 3.0', '>= 3.0.5'
|
@@ -93,6 +93,7 @@ RSpec.shared_context 'postgres' do
|
|
93
93
|
CREATE TABLE IF NOT EXISTS #{ schema.empty? ? schema : schema + '.'}tracks
|
94
94
|
(
|
95
95
|
id VARCHAR(512) NOT NULL CONSTRAINT tracks_pkey PRIMARY KEY,
|
96
|
+
action VARCHAR(512),
|
96
97
|
anonymous_id VARCHAR(512),
|
97
98
|
app_name VARCHAR(512),
|
98
99
|
context_campaign_content VARCHAR(512),
|
@@ -221,7 +221,7 @@ describe LogStash::Outputs::Charrington do
|
|
221
221
|
<<-CONFIG
|
222
222
|
input {
|
223
223
|
generator {
|
224
|
-
message => '{"app_name": "Web App", "event": "schemaless", "meta": { "type": "XML" }, "published_at": "2019-07-29T20:09:18Z", "user_id": "123", "user_uid": "456" }'
|
224
|
+
message => '{"app_name": "Web App", "action": "click", "event": "schemaless", "meta": { "type": "XML" }, "published_at": "2019-07-29T20:09:18Z", "user_id": "123", "user_uid": "456" }'
|
225
225
|
codec => 'json'
|
226
226
|
count => 1
|
227
227
|
}
|
@@ -244,11 +244,11 @@ describe LogStash::Outputs::Charrington do
|
|
244
244
|
drop_table('schemaless')
|
245
245
|
run_pipeline
|
246
246
|
expect(query('SELECT * FROM schemaless')).to match_array([
|
247
|
-
{:anonymous_id=>"", :app_name=>"Web App", :event=>"schemaless", :event_text=>"schemaless", :id=>a_kind_of(String), :original_timestamp=>a_kind_of(String), :received_at=>a_kind_of(String), :sent_at=>"2019-07-29 20:09:18", :timestamp=>a_kind_of(String), :type=>"XML", :user_id=>"123", :user_uid=>"456", :uuid_ts=>a_kind_of(String)}
|
247
|
+
{:anonymous_id=>"", :action=>"click", :app_name=>"Web App", :event=>"schemaless", :event_text=>"schemaless", :id=>a_kind_of(String), :original_timestamp=>a_kind_of(String), :received_at=>a_kind_of(String), :sent_at=>"2019-07-29 20:09:18", :timestamp=>a_kind_of(String), :type=>"XML", :user_id=>"123", :user_uid=>"456", :uuid_ts=>a_kind_of(String)}
|
248
248
|
])
|
249
249
|
expect(query('SELECT COUNT(1) FROM schemaless').first[:count]).to eq("1")
|
250
250
|
expect(query("SELECT * FROM tracks")).to match_array([
|
251
|
-
{:id=>a_kind_of(String), :anonymous_id=>"", :app_name=>"Web App", :context_campaign_content=>nil, :context_campaign_medium=>nil, :context_campaign_name=>nil, :context_campaign_source=>nil, :context_ip=>nil, :context_library_name=>nil, :context_library_version=>nil, :context_page_path=>nil, :context_page_referrer=>nil, :context_page_search=>nil, :context_page_title=>nil, :context_page_url=>nil, :context_user_agent=>nil, :event=>"schemaless", :event_text=>"schemaless", :original_timestamp=>"2019-07-29 20:09:18", :received_at=>"2019-07-29 20:09:18", :segment_dedupe_id=>nil, :sent_at=>"2019-07-29 20:09:18", :timestamp=>"2019-07-29 20:09:18", :user_id=>"123", :user_uid=>"456", :uuid=>nil, :uuid_ts=>a_kind_of(String)}
|
251
|
+
{:id=>a_kind_of(String), :action=>"click", :anonymous_id=>"", :app_name=>"Web App", :context_campaign_content=>nil, :context_campaign_medium=>nil, :context_campaign_name=>nil, :context_campaign_source=>nil, :context_ip=>nil, :context_library_name=>nil, :context_library_version=>nil, :context_page_path=>nil, :context_page_referrer=>nil, :context_page_search=>nil, :context_page_title=>nil, :context_page_url=>nil, :context_user_agent=>nil, :event=>"schemaless", :event_text=>"schemaless", :original_timestamp=>"2019-07-29 20:09:18", :received_at=>"2019-07-29 20:09:18", :segment_dedupe_id=>nil, :sent_at=>"2019-07-29 20:09:18", :timestamp=>"2019-07-29 20:09:18", :user_id=>"123", :user_uid=>"456", :uuid=>nil, :uuid_ts=>a_kind_of(String)}
|
252
252
|
])
|
253
253
|
end
|
254
254
|
end
|
Binary file
|
Binary file
|
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-charrington
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.24
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- dconger
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2021-12-15 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
requirement: !ruby/object:Gem::Requirement
|
@@ -55,29 +55,23 @@ dependencies:
|
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
requirement: !ruby/object:Gem::Requirement
|
57
57
|
requirements:
|
58
|
-
- - "~>"
|
59
|
-
- !ruby/object:Gem::Version
|
60
|
-
version: '1.3'
|
61
58
|
- - ">="
|
62
59
|
- !ruby/object:Gem::Version
|
63
|
-
version:
|
60
|
+
version: '0'
|
64
61
|
name: logstash-devutils
|
65
62
|
prerelease: false
|
66
63
|
type: :development
|
67
64
|
version_requirements: !ruby/object:Gem::Requirement
|
68
65
|
requirements:
|
69
|
-
- - "~>"
|
70
|
-
- !ruby/object:Gem::Version
|
71
|
-
version: '1.3'
|
72
66
|
- - ">="
|
73
67
|
- !ruby/object:Gem::Version
|
74
|
-
version:
|
68
|
+
version: '0'
|
75
69
|
- !ruby/object:Gem::Dependency
|
76
70
|
requirement: !ruby/object:Gem::Requirement
|
77
71
|
requirements:
|
78
72
|
- - "~>"
|
79
73
|
- !ruby/object:Gem::Version
|
80
|
-
version: 0.4.
|
74
|
+
version: 0.4.1
|
81
75
|
name: jar-dependencies
|
82
76
|
prerelease: false
|
83
77
|
type: :development
|
@@ -85,13 +79,13 @@ dependencies:
|
|
85
79
|
requirements:
|
86
80
|
- - "~>"
|
87
81
|
- !ruby/object:Gem::Version
|
88
|
-
version: 0.4.
|
82
|
+
version: 0.4.1
|
89
83
|
- !ruby/object:Gem::Dependency
|
90
84
|
requirement: !ruby/object:Gem::Requirement
|
91
85
|
requirements:
|
92
86
|
- - "~>"
|
93
87
|
- !ruby/object:Gem::Version
|
94
|
-
version:
|
88
|
+
version: 3.3.12
|
95
89
|
name: ruby-maven
|
96
90
|
prerelease: false
|
97
91
|
type: :development
|
@@ -99,7 +93,7 @@ dependencies:
|
|
99
93
|
requirements:
|
100
94
|
- - "~>"
|
101
95
|
- !ruby/object:Gem::Version
|
102
|
-
version:
|
96
|
+
version: 3.3.12
|
103
97
|
- !ruby/object:Gem::Dependency
|
104
98
|
requirement: !ruby/object:Gem::Requirement
|
105
99
|
requirements:
|
@@ -176,14 +170,8 @@ executables: []
|
|
176
170
|
extensions: []
|
177
171
|
extra_rdoc_files: []
|
178
172
|
files:
|
179
|
-
- CHANGELOG.md
|
180
173
|
- Gemfile
|
181
|
-
- LICENSE.txt
|
182
|
-
- README.md
|
183
|
-
- THANKS.md
|
184
174
|
- lib/com/zaxxer/HikariCP/2.7.2/HikariCP-2.7.2.jar
|
185
|
-
- lib/commons-io/commons-io/2.4/commons-io-2.4.jar
|
186
|
-
- lib/de/flapdoodle/embed/de.flapdoodle.embed.process/2.0.2/de.flapdoodle.embed.process-2.0.2.jar
|
187
175
|
- lib/logstash-output-charrington_jars.rb
|
188
176
|
- lib/logstash/outputs/charrington.rb
|
189
177
|
- lib/logstash/outputs/charrington/alter_postgres_table.rb
|
@@ -195,16 +183,10 @@ files:
|
|
195
183
|
- lib/logstash/outputs/charrington/service.rb
|
196
184
|
- lib/logstash/outputs/charrington/transform_postgres.rb
|
197
185
|
- lib/logstash/outputs/charrington/transform_redshift.rb
|
198
|
-
- lib/
|
199
|
-
- lib/
|
200
|
-
- lib/org/apache/
|
201
|
-
- lib/org/apache/commons/commons-lang3/3.1/commons-lang3-3.1.jar
|
202
|
-
- lib/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar
|
203
|
-
- lib/org/apache/logging/log4j/log4j-slf4j-impl/2.6.2/log4j-slf4j-impl-2.6.2.jar
|
204
|
-
- lib/org/postgresql/postgresql/42.2.5/postgresql-42.2.5.jar
|
186
|
+
- lib/org/apache/logging/log4j/log4j-api/2.16.0/log4j-api-2.16.0.jar
|
187
|
+
- lib/org/apache/logging/log4j/log4j-core/2.16.0/log4j-core-2.16.0.jar
|
188
|
+
- lib/org/apache/logging/log4j/log4j-slf4j-impl/2.16.0/log4j-slf4j-impl-2.16.0.jar
|
205
189
|
- lib/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar
|
206
|
-
- lib/ru/yandex/qatools/embed/postgresql-embedded/2.10/postgresql-embedded-2.10.jar
|
207
|
-
- lib/ru/yandex/qatools/embed/postgresql-embedded/2.8/postgresql-embedded-2.8.jar
|
208
190
|
- logstash-output-charrington.gemspec
|
209
191
|
- spec/charrington_spec_helper.rb
|
210
192
|
- spec/logstash-output-charrington_test_jars.rb
|
@@ -212,8 +194,9 @@ files:
|
|
212
194
|
- spec/outputs/charrington_postgres_spec.rb
|
213
195
|
- spec/outputs/charrington_spec.rb
|
214
196
|
- vendor/jar-dependencies/runtime-jars/HikariCP-2.7.2.jar
|
215
|
-
- vendor/jar-dependencies/runtime-jars/log4j-api-2.
|
216
|
-
- vendor/jar-dependencies/runtime-jars/log4j-
|
197
|
+
- vendor/jar-dependencies/runtime-jars/log4j-api-2.16.0.jar
|
198
|
+
- vendor/jar-dependencies/runtime-jars/log4j-core-2.16.0.jar
|
199
|
+
- vendor/jar-dependencies/runtime-jars/log4j-slf4j-impl-2.16.0.jar
|
217
200
|
- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.25.jar
|
218
201
|
homepage: https://gitlab.podium.com/engineering/analytics/logstash-output-charrington
|
219
202
|
licenses:
|
@@ -237,7 +220,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
237
220
|
version: '0'
|
238
221
|
requirements:
|
239
222
|
- jar 'com.zaxxer:HikariCP', '2.7.2'
|
240
|
-
- jar 'org.apache.logging.log4j:log4j-slf4j-impl', '2.
|
223
|
+
- jar 'org.apache.logging.log4j:log4j-slf4j-impl', '2.16.0'
|
241
224
|
rubyforge_project:
|
242
225
|
rubygems_version: 2.7.6
|
243
226
|
signing_key:
|
data/CHANGELOG.md
DELETED
@@ -1,64 +0,0 @@
|
|
1
|
-
# Change Log
|
2
|
-
All notable changes to this project will be documented in this file, from 0.2.0.
|
3
|
-
|
4
|
-
## [5.3.0] - 2017-11-08
|
5
|
-
- Adds configuration options `enable_event_as_json_keyword` and `event_as_json_keyword`
|
6
|
-
- Adds BigDecimal support
|
7
|
-
- Adds additional logging for debugging purposes (with thanks to @mlkmhd's work)
|
8
|
-
|
9
|
-
## [5.2.1] - 2017-04-09
|
10
|
-
- Adds Array and Hash to_json support for non-sprintf syntax
|
11
|
-
|
12
|
-
## [5.2.0] - 2017-04-01
|
13
|
-
- Upgrades HikariCP to latest
|
14
|
-
- Fixes HikariCP logging integration issues
|
15
|
-
|
16
|
-
## [5.1.0] - 2016-12-17
|
17
|
-
- phoenix-thin fixes for issue #60
|
18
|
-
|
19
|
-
## [5.0.0] - 2016-11-03
|
20
|
-
- logstash v5 support
|
21
|
-
|
22
|
-
## [0.3.1] - 2016-08-28
|
23
|
-
- Adds connection_test configuration option, to prevent the connection test from occuring, allowing the error to be suppressed.
|
24
|
-
Useful for cockroachdb deployments. https://github.com/theangryangel/logstash-output-jdbc/issues/53
|
25
|
-
|
26
|
-
## [0.3.0] - 2016-07-24
|
27
|
-
- Brings tests from v5 branch, providing greater coverage
|
28
|
-
- Removes bulk update support, due to inconsistent behaviour
|
29
|
-
- Plugin now marked as threadsafe, meaning only 1 instance per-Logstash
|
30
|
-
- Raises default max_pool_size to match the default number of workers (1 connection per worker)
|
31
|
-
|
32
|
-
## [0.2.10] - 2016-07-07
|
33
|
-
- Support non-string entries in statement array
|
34
|
-
- Adds backtrace to exception logging
|
35
|
-
|
36
|
-
## [0.2.9] - 2016-06-29
|
37
|
-
- Fix NameError exception.
|
38
|
-
- Moved log_jdbc_exception calls
|
39
|
-
|
40
|
-
## [0.2.7] - 2016-05-29
|
41
|
-
- Backport retry exception logic from v5 branch
|
42
|
-
- Backport improved timestamp compatibility from v5 branch
|
43
|
-
|
44
|
-
## [0.2.6] - 2016-05-02
|
45
|
-
- Fix for exception infinite loop
|
46
|
-
|
47
|
-
## [0.2.5] - 2016-04-11
|
48
|
-
### Added
|
49
|
-
- Basic tests running against DerbyDB
|
50
|
-
- Fix for converting Logstash::Timestamp to iso8601 from @hordijk
|
51
|
-
|
52
|
-
## [0.2.4] - 2016-04-07
|
53
|
-
- Documentation fixes from @hordijk
|
54
|
-
|
55
|
-
## [0.2.3] - 2016-02-16
|
56
|
-
- Bug fixes
|
57
|
-
|
58
|
-
## [0.2.2] - 2015-12-30
|
59
|
-
- Bug fixes
|
60
|
-
|
61
|
-
## [0.2.1] - 2015-12-22
|
62
|
-
- Support for connection pooling support added through HikariCP
|
63
|
-
- Support for unsafe statement handling (allowing dynamic queries)
|
64
|
-
- Altered exception handling to now count sequential flushes with exceptions thrown
|
data/LICENSE.txt
DELETED
@@ -1,21 +0,0 @@
|
|
1
|
-
The MIT License (MIT)
|
2
|
-
|
3
|
-
Copyright (c) 2014
|
4
|
-
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
-
of this software and associated documentation files (the "Software"), to deal
|
7
|
-
in the Software without restriction, including without limitation the rights
|
8
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
-
copies of the Software, and to permit persons to whom the Software is
|
10
|
-
furnished to do so, subject to the following conditions:
|
11
|
-
|
12
|
-
The above copyright notice and this permission notice shall be included in all
|
13
|
-
copies or substantial portions of the Software.
|
14
|
-
|
15
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
-
SOFTWARE.
|
data/README.md
DELETED
@@ -1,130 +0,0 @@
|
|
1
|
-
# Logstash Output Charrington
|
2
|
-
|
3
|
-
Logstash Output Plugin to handle batching messages and dynamically create and alter tables.
|
4
|
-
|
5
|
-
Forked From [logstash-output-jdbc](https://github.com/theangryangel/logstash-output-jdbc)
|
6
|
-
## Configure
|
7
|
-
Example Logstash Config
|
8
|
-
```bash
|
9
|
-
input {
|
10
|
-
kafka {
|
11
|
-
auto_offset_reset => earliest
|
12
|
-
bootstrap_servers => "${LS_KAFKA_BOOTSTRAP_SERVERS:kafka:9092}"
|
13
|
-
codec => json
|
14
|
-
group_id => "winston_ls_pg_sinks_cg"
|
15
|
-
topics => [
|
16
|
-
"orwell.analytics.v1.json"
|
17
|
-
]
|
18
|
-
}
|
19
|
-
}
|
20
|
-
|
21
|
-
filter {
|
22
|
-
if "_jsonparsefailure" in [tags] {
|
23
|
-
drop { }
|
24
|
-
}
|
25
|
-
|
26
|
-
ruby {
|
27
|
-
code => "
|
28
|
-
hash = event.to_hash
|
29
|
-
hash.each do |k,v|
|
30
|
-
if v == nil
|
31
|
-
event.remove(k)
|
32
|
-
end
|
33
|
-
end
|
34
|
-
"
|
35
|
-
}
|
36
|
-
}
|
37
|
-
|
38
|
-
output {
|
39
|
-
charrington {
|
40
|
-
driver_jar_path => "${DRIVER_PATH:/usr/share/logstash/vendor/jar/jdbc/postgresql-42.2.5.jar}"
|
41
|
-
connection_string => "jdbc:postgresql://${WINSTON_DATABASE_HOST}:5432/${WINSTON_DATABASE}?user=${WINSTON_DATABASE_USERNAME}&password=${WINSTON_DATABASE_PASSWORD}"
|
42
|
-
schema => "dea"
|
43
|
-
}
|
44
|
-
|
45
|
-
if "${LS_SHOW_DEBUG_OUTPUT}" == "true" {
|
46
|
-
stdout { codec => rubydebug }
|
47
|
-
}
|
48
|
-
}
|
49
|
-
```
|
50
|
-
|
51
|
-
## Build & Publish
|
52
|
-
|
53
|
-
**TLDR**
|
54
|
-
```bash
|
55
|
-
bundle exec rake build # build gem
|
56
|
-
gem push logstash-output-charrington-x.x.x.gem # publish
|
57
|
-
```
|
58
|
-
|
59
|
-
Everything in the [vendor](./vendor) directory (including `test-jars` if it exists) will be packaged with the generated gem.
|
60
|
-
Avoid including unnecessary dependencies, by removing the vendor directory and regenerating it with the rake task.
|
61
|
-
See the first command below:
|
62
|
-
|
63
|
-
```bash
|
64
|
-
rm -r vendor && bundle exec rake install_jars # clean vendor directory and re-install runtime jar dependencies
|
65
|
-
gem build logstash-output-charrington.gemspec # build
|
66
|
-
gem push logstash-output-charrington-x.x.x.gem # publish
|
67
|
-
gem owner --add <email> logstash-output-charrington # add another authorized publisher
|
68
|
-
```
|
69
|
-
|
70
|
-
## Testing
|
71
|
-
**NOTE:** Downloading test_jars requires maven and Java. You can set them up with asdf: [asdf-java](https://github.com/skotchpine/asdf-java), [asdf-maven](https://github.com/skotchpine/asdf-maven)
|
72
|
-
* There is currently an [open bug](https://github.com/skotchpine/asdf-maven/pull/17) with how `asdf-maven` sets `$JAVA_HOME` on a mac. Edit your `~/.asdf/plugins/maven/bin/exec-env` file to fix the bug
|
73
|
-
|
74
|
-
```bash
|
75
|
-
bundle exec rake install_test_jars # install test runtime jar dependencies
|
76
|
-
bundle exec rspec
|
77
|
-
```
|
78
|
-
|
79
|
-
#### Dangling Embedded Postgres
|
80
|
-
**TLDR**
|
81
|
-
```bash
|
82
|
-
lsof -PiTCP -sTCP:LISTEN | grep postgres | awk '{print $2}' | xargs kill
|
83
|
-
```
|
84
|
-
Sometimes the embedded postgres JAR can leave dangling open processes. This is obvious when you have `export TEST_DEBUG=true`
|
85
|
-
and the test output contains a stacktrace that starts with `[2019-06-21T15:18:06,734][ERROR][ru.yandex.qatools.embed.postgresql.PostgresProcess] Failed to read PID file (File '/var/folders/...`.
|
86
|
-
To resolve this issue, check for dangling processes using `lsof -PiTCP -sTCP:LISTEN | grep postgres`, then you can kill the process ids
|
87
|
-
|
88
|
-
#### Tracks Table
|
89
|
-
**Create Statment for Driver is Postgres and Transform is Redshift**
|
90
|
-
* See `@@redshift_tracks_columns` in [insert.rb](./lib/logstash/outputs/charrington/insert.rb) for column list
|
91
|
-
```sql
|
92
|
-
create table <schema>.tracks
|
93
|
-
(
|
94
|
-
id varchar(512) not null constraint tracks_pkey primary key,
|
95
|
-
anonymous_id varchar(512),
|
96
|
-
app_name varchar(512),
|
97
|
-
context_campaign_content varchar(512)
|
98
|
-
context_campaign_medium varchar(512),
|
99
|
-
context_campaign_name varchar(512),
|
100
|
-
context_campaign_source varchar(512),
|
101
|
-
context_ip varchar(512),
|
102
|
-
context_library_name varchar(512),
|
103
|
-
context_library_version varchar(512),
|
104
|
-
context_page_path varchar(512),
|
105
|
-
context_page_referrer varchar(512),
|
106
|
-
context_page_search varchar(512),
|
107
|
-
context_page_title varchar(512),
|
108
|
-
context_page_url varchar(512),
|
109
|
-
context_user_agent varchar(512),
|
110
|
-
event varchar(512),
|
111
|
-
event_text varchar(512),
|
112
|
-
original_timestamp timestamp,
|
113
|
-
received_at timestamp,
|
114
|
-
segment_dedupe_id varchar(512),
|
115
|
-
sent_at timestamp,
|
116
|
-
timestamp timestamp,
|
117
|
-
user_id varchar(512),
|
118
|
-
user_uid varchar(512),
|
119
|
-
uuid bigint,
|
120
|
-
uuid_ts timestamp default ('now'::text)::timestamp without time zone,
|
121
|
-
)
|
122
|
-
|
123
|
-
```
|
124
|
-
|
125
|
-
#### Resources
|
126
|
-
* [logstash-devutils](https://rubygems.org/gems/logstash-devutils) gem with logstash helper methods and tools
|
127
|
-
* [spec_helper.rb source](https://github.com/elastic/logstash-devutils/blob/master/lib/logstash/devutils/rspec/spec_helper.rb)
|
128
|
-
* [logstash_helpers.rb source](https://github.com/elastic/logstash-devutils/blob/master/lib/logstash/devutils/rspec/logstash_helpers.rb)
|
129
|
-
* [logstash-output-file spec test](https://github.com/logstash-plugins/logstash-output-file/blob/master/spec/outputs/file_spec.rb)
|
130
|
-
|
data/THANKS.md
DELETED
@@ -1,18 +0,0 @@
|
|
1
|
-
logstash-output-jdbc is a project originally created by Karl Southern
|
2
|
-
(the_angry_angel), but there are a number of people that have contributed
|
3
|
-
or implemented key features over time. We do our best to keep this list
|
4
|
-
up-to-date, but you can also have a look at the nice contributor graphs
|
5
|
-
produced by GitHub: https://github.com/theangryangel/logstash-output-jdbc/graphs/contributors
|
6
|
-
|
7
|
-
* [hordijk](https://github.com/hordijk)
|
8
|
-
* [dmitryakadiamond](https://github.com/dmitryakadiamond)
|
9
|
-
* [MassimoSporchia](https://github.com/MassimoSporchia)
|
10
|
-
* [ebuildy](https://github.com/ebuildy)
|
11
|
-
* [kushtrimjunuzi](https://github.com/kushtrimjunuzi)
|
12
|
-
* [josemazo](https://github.com/josemazo)
|
13
|
-
* [aceoliver](https://github.com/aceoliver)
|
14
|
-
* [roflmao](https://github.com/roflmao)
|
15
|
-
* [onesuper](https://github.com/onesuper)
|
16
|
-
* [phr0gz](https://github.com/phr0gz)
|
17
|
-
* [jMonsinjon](https://github.com/jMonsinjon)
|
18
|
-
* [mlkmhd](https://github.com/mlkmhd)
|
Binary file
|
data/lib/de/flapdoodle/embed/de.flapdoodle.embed.process/2.0.2/de.flapdoodle.embed.process-2.0.2.jar
DELETED
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|