logstash-output-charrington 0.3.13 → 0.3.14
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +37 -0
- data/lib/commons-io/commons-io/2.4/commons-io-2.4.jar +0 -0
- data/lib/de/flapdoodle/embed/de.flapdoodle.embed.process/2.0.2/de.flapdoodle.embed.process-2.0.2.jar +0 -0
- data/lib/logstash/outputs/charrington/alter_postgres_table.rb +1 -3
- data/lib/logstash/outputs/charrington/alter_redshift_table.rb +3 -5
- data/lib/logstash/outputs/charrington/create_postgres_table.rb +1 -2
- data/lib/logstash/outputs/charrington/create_redshift_table.rb +0 -1
- data/lib/logstash/outputs/charrington/insert.rb +81 -27
- data/lib/net/java/dev/jna/jna-platform/4.0.0/jna-platform-4.0.0.jar +0 -0
- data/lib/net/java/dev/jna/jna/4.0.0/jna-4.0.0.jar +0 -0
- data/lib/org/apache/commons/commons-compress/1.10/commons-compress-1.10.jar +0 -0
- data/lib/org/apache/commons/commons-lang3/3.1/commons-lang3-3.1.jar +0 -0
- data/lib/org/postgresql/postgresql/42.2.5/postgresql-42.2.5.jar +0 -0
- data/lib/ru/yandex/qatools/embed/postgresql-embedded/2.10/postgresql-embedded-2.10.jar +0 -0
- data/lib/ru/yandex/qatools/embed/postgresql-embedded/2.8/postgresql-embedded-2.8.jar +0 -0
- data/logstash-output-charrington.gemspec +1 -1
- data/spec/charrington_spec_helper.rb +36 -0
- data/spec/outputs/charrington_spec.rb +15 -2
- metadata +10 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7bfd649565a175e374554e824e34e48bc7d3a343a6bf4c3dd3343153ef783c9e
|
4
|
+
data.tar.gz: 6e57f8174acea5086848184d69b24a98b7416f694ab4d44a8fab18077aa912cb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 102b9c2e207472fe275ec8528266779f33962f18091a6a72aa71d149c5b08ec6ea58546872be37b45b06ac110cbcd12501edccdd0c9c399349cabd679d6dda35
|
7
|
+
data.tar.gz: bd2a4119733e409a0ca980acb5662688ecdc2b6e6255f101f263caea39ae0429fcc7224cd449af390f918a2ec483dcca421e7d227cafdf5c804ae6a278b38b9a
|
data/README.md
CHANGED
@@ -85,6 +85,43 @@ Sometimes the embedded postgres JAR can leave dangling open processes. This is o
|
|
85
85
|
and the test output contains a stacktrace that starts with `[2019-06-21T15:18:06,734][ERROR][ru.yandex.qatools.embed.postgresql.PostgresProcess] Failed to read PID file (File '/var/folders/...`.
|
86
86
|
To resolve this issue, check for dangling processes using `lsof -PiTCP -sTCP:LISTEN | grep postgres`, then you can kill the process ids
|
87
87
|
|
88
|
+
#### Tracks Table
|
89
|
+
**Create Statment for Driver is Postgres and Transform is Redshift**
|
90
|
+
* See `@@redshift_tracks_columns` in [insert.rb](./lib/logstash/outputs/charrington/insert.rb) for column list
|
91
|
+
```sql
|
92
|
+
create table <schema>.tracks
|
93
|
+
(
|
94
|
+
id varchar(512) not null constraint tracks_pkey primary key,
|
95
|
+
anonymous_id varchar(512),
|
96
|
+
app_name varchar(512),
|
97
|
+
context_campaign_content varchar(512)
|
98
|
+
context_campaign_medium varchar(512),
|
99
|
+
context_campaign_name varchar(512),
|
100
|
+
context_campaign_source varchar(512),
|
101
|
+
context_ip varchar(512),
|
102
|
+
context_library_name varchar(512),
|
103
|
+
context_library_version varchar(512),
|
104
|
+
context_page_path varchar(512),
|
105
|
+
context_page_referrer varchar(512),
|
106
|
+
context_page_search varchar(512),
|
107
|
+
context_page_title varchar(512),
|
108
|
+
context_page_url varchar(512),
|
109
|
+
context_user_agent varchar(512),
|
110
|
+
event varchar(512),
|
111
|
+
event_text varchar(512),
|
112
|
+
original_timestamp timestamp,
|
113
|
+
received_at timestamp,
|
114
|
+
segment_dedupe_id varchar(512),
|
115
|
+
sent_at timestamp,
|
116
|
+
timestamp timestamp,
|
117
|
+
user_id varchar(512),
|
118
|
+
user_uid varchar(512),
|
119
|
+
uuid bigint,
|
120
|
+
uuid_ts timestamp default ('now'::text)::timestamp without time zone,
|
121
|
+
)
|
122
|
+
|
123
|
+
```
|
124
|
+
|
88
125
|
#### Resources
|
89
126
|
* [logstash-devutils](https://rubygems.org/gems/logstash-devutils) gem with logstash helper methods and tools
|
90
127
|
* [spec_helper.rb source](https://github.com/elastic/logstash-devutils/blob/master/lib/logstash/devutils/rspec/spec_helper.rb)
|
Binary file
|
data/lib/de/flapdoodle/embed/de.flapdoodle.embed.process/2.0.2/de.flapdoodle.embed.process-2.0.2.jar
ADDED
Binary file
|
@@ -70,7 +70,6 @@ module Charrington
|
|
70
70
|
sql = "SELECT * FROM #{schema}#{table_name} LIMIT 1;"
|
71
71
|
stmt, rs = executeQuery(prep_sql(sql))
|
72
72
|
meta_data = rs.getMetaData()
|
73
|
-
stmt.close unless stmt.nil?
|
74
73
|
column_count = meta_data.getColumnCount()
|
75
74
|
(1..column_count).map {|i| meta_data.getColumnName(i) }
|
76
75
|
ensure
|
@@ -81,8 +80,7 @@ module Charrington
|
|
81
80
|
stmt = connection.prepareStatement(prep_sql(sql))
|
82
81
|
stmt.execute()
|
83
82
|
rescue Java::OrgPostgresqlUtil::PSQLException => e
|
84
|
-
|
85
|
-
self.logger.info "PSQLException: #{e.message}"
|
83
|
+
self.logger.error "PSQLException: #{e.message}"
|
86
84
|
ensure
|
87
85
|
stmt.close unless stmt.nil?
|
88
86
|
end
|
@@ -91,13 +91,11 @@ module Charrington
|
|
91
91
|
stmt = connection.prepareStatement(prep_sql(sql))
|
92
92
|
stmt.execute()
|
93
93
|
rescue Java::JavaSql::SQLException => e
|
94
|
-
|
95
|
-
self.logger.info "Alter Redshift SQLException: #{e.message}, with SQL: #{sql}"
|
94
|
+
self.logger.error "Alter Redshift SQLException: #{e.message}, with SQL: #{sql}"
|
96
95
|
rescue => e
|
97
|
-
|
98
|
-
self.logger.info "Alter Redshift Unknown exception: #{e.message}"
|
96
|
+
self.logger.error "Alter Redshift Unknown exception: #{e.message}, with SQL: #{sql}"
|
99
97
|
ensure
|
100
|
-
self.logger.
|
98
|
+
self.logger.error "Within ensure block of execute in alter_redshift_table.rb and value of stmt.nil?: #{stmt.nil?}"
|
101
99
|
stmt.close unless stmt.nil?
|
102
100
|
end
|
103
101
|
|
@@ -85,8 +85,7 @@ module Charrington
|
|
85
85
|
statement = connection.prepareStatement( sql.gsub(/\s+/, " ").strip )
|
86
86
|
statement.execute()
|
87
87
|
rescue Java::OrgPostgresqlUtil::PSQLException => e
|
88
|
-
|
89
|
-
self.logger.info "PSQLException: #{e.message}"
|
88
|
+
self.logger.error "PSQLException: #{e.message} sql=#{sql}"
|
90
89
|
ensure
|
91
90
|
statement.close unless statement.nil?
|
92
91
|
end
|
@@ -84,7 +84,6 @@ module Charrington
|
|
84
84
|
statement = connection.prepareStatement( sql.gsub(/\s+/, " ").strip )
|
85
85
|
statement.execute()
|
86
86
|
rescue Java::JavaSql::SQLException => e
|
87
|
-
puts "Redshift SQLException: #{e.message}"
|
88
87
|
self.logger.info "Redshift SQLException: #{e.message}, with SQL: #{sql}"
|
89
88
|
ensure
|
90
89
|
self.logger.info "Within ensure block of create_redshift_table.rb and value of statement.nil?: #{statement.nil?}"
|
@@ -3,6 +3,7 @@ require File.join(File.dirname(__FILE__), "create_redshift_table")
|
|
3
3
|
require File.join(File.dirname(__FILE__), "alter_postgres_table")
|
4
4
|
require File.join(File.dirname(__FILE__), "alter_redshift_table")
|
5
5
|
require File.join(File.dirname(__FILE__), "service")
|
6
|
+
require 'time'
|
6
7
|
|
7
8
|
module Charrington
|
8
9
|
class Insert
|
@@ -12,8 +13,11 @@ module Charrington
|
|
12
13
|
include Service
|
13
14
|
include LogStash::Util::Loggable
|
14
15
|
attr_accessor :event, :should_retry
|
15
|
-
attr_reader :connection, :schema, :table_name, :columns, :driver, :opts
|
16
|
+
attr_reader :connection, :schema, :table_name, :columns, :driver, :opts, :transformer, :tracks
|
16
17
|
attr_reader :event_as_json_keyword, :enable_event_as_json_keyword
|
18
|
+
# TODO create a current_table_columns (alter_postgres_table.rb) query on the tracks table to get the current columns
|
19
|
+
@@redshift_tracks_columns = %w(id app_name received_at uuid uuid_ts anonymous_id context_ip context_library_name context_library_version context_page_path context_page_referrer context_page_title context_page_url context_user_agent event event_text original_timestamp sent_at timestamp user_id user_uid context_campaign_medium context_campaign_name context_page_search context_campaign_source segment_dedupe_id context_campaign_content)
|
20
|
+
@@postgres_tracks_columns = %w(anonymous_user app_name event published_at session_ip session_library_name session_library_version session_page_path session_page_referrer session_page_search session_page_title session_page_url session_user_agent user_id user_uid)
|
17
21
|
|
18
22
|
Error = Class.new(StandardError)
|
19
23
|
EventNil = Class.new(Error)
|
@@ -22,7 +26,9 @@ module Charrington
|
|
22
26
|
|
23
27
|
def initialize(connection, event, opts = {})
|
24
28
|
raise EventNil, "Table name is nil" if event.nil?
|
29
|
+
@transformer = opts[:transformer]
|
25
30
|
@event = event.to_hash
|
31
|
+
@tracks = create_tracks(@event)
|
26
32
|
event_name = event["event"].to_s.strip
|
27
33
|
raise TableNameNil, "Table name is nil" if event_name.empty?
|
28
34
|
|
@@ -41,13 +47,16 @@ module Charrington
|
|
41
47
|
|
42
48
|
def call
|
43
49
|
self.logger.info "Attempting insert into table name: #{table_name}"
|
44
|
-
|
50
|
+
insert_sql = insert_event_statement
|
51
|
+
insert_stmt = connection.prepareStatement(insert_sql)
|
45
52
|
self.logger.info "Insert statement passed into prepareStatement is: #{insert_stmt}"
|
53
|
+
insert_stmt = add_statement_event_params(insert_stmt, event)
|
54
|
+
self.logger.info "Insert statement to be run is: #{insert_stmt.toString}"
|
55
|
+
insert_stmt.execute
|
56
|
+
|
57
|
+
self.logger.info "Attempting insert into tracks table"
|
58
|
+
do_tracks_insert
|
46
59
|
|
47
|
-
stmt = connection.prepareStatement(insert_stmt)
|
48
|
-
stmt = add_statement_event_params(stmt)
|
49
|
-
self.logger.info "Insert statement to be run is: #{insert_stmt}"
|
50
|
-
stmt.execute
|
51
60
|
should_retry
|
52
61
|
rescue Java::JavaSql::SQLException => e
|
53
62
|
case e.getSQLState()
|
@@ -58,65 +67,112 @@ module Charrington
|
|
58
67
|
self.logger.info "Received Java::JavaSql::SQLException with error sql state of 42703, moving to alter table"
|
59
68
|
should_retry = alter_table
|
60
69
|
else
|
61
|
-
raise InsertFailed, "Charrington:
|
70
|
+
raise InsertFailed, "SQLException (Charrington:Insert) #{e.message} #{insert_sql}"
|
62
71
|
end
|
63
72
|
should_retry
|
64
73
|
|
65
74
|
rescue => e
|
66
|
-
raise InsertFailed, "Charrington:
|
75
|
+
raise InsertFailed, "SQLException (Charrington:Insert) #{e.message} #{insert_sql}"
|
67
76
|
ensure
|
68
|
-
|
77
|
+
insert_stmt.close unless insert_stmt.nil?
|
69
78
|
cleanup
|
70
79
|
end
|
71
80
|
|
81
|
+
def do_tracks_insert
|
82
|
+
tracks_sql = insert_tracks_statement
|
83
|
+
tracks_stmt = connection.prepareStatement(tracks_sql)
|
84
|
+
tracks_stmt = add_statement_event_params(tracks_stmt, tracks)
|
85
|
+
self.logger.info "Insert tracks statment to be run: #{tracks_stmt.toString}"
|
86
|
+
tracks_stmt.execute
|
87
|
+
rescue Java::JavaSql::SQLException => e
|
88
|
+
self.logger.error("SQLException (Charrington:Insert) Insert tracks entry failed. #{e.message} #{tracks_sql}")
|
89
|
+
ensure
|
90
|
+
tracks_stmt.close unless tracks_stmt.nil?
|
91
|
+
end
|
92
|
+
|
72
93
|
private
|
73
94
|
|
95
|
+
def tracks_columns
|
96
|
+
is_redshift_transform? ? @@redshift_tracks_columns : @@postgres_tracks_columns
|
97
|
+
end
|
98
|
+
|
99
|
+
def is_redshift?
|
100
|
+
driver == 'redshift'
|
101
|
+
end
|
102
|
+
|
103
|
+
def is_postgres?
|
104
|
+
driver == 'postgres'
|
105
|
+
end
|
106
|
+
|
107
|
+
def is_postgres_transform?
|
108
|
+
transformer == 'postgres'
|
109
|
+
end
|
110
|
+
|
111
|
+
def is_redshift_transform?
|
112
|
+
transformer == 'redshift'
|
113
|
+
end
|
114
|
+
|
74
115
|
def create_table
|
75
|
-
if
|
116
|
+
if is_postgres?
|
76
117
|
Charrington::CreatePostgresTable.call(connection, event, schema, table_name, columns, opts)
|
77
|
-
elsif
|
118
|
+
elsif is_redshift?
|
78
119
|
Charrington::CreateRedshiftTable.call(connection, event, schema, table_name, columns, opts)
|
79
120
|
end
|
80
121
|
end
|
81
122
|
|
82
123
|
def alter_table
|
83
|
-
if
|
124
|
+
if is_postgres?
|
84
125
|
Charrington::AlterPostgresTable.call(connection, event, schema, table_name, columns)
|
85
|
-
elsif
|
126
|
+
elsif is_redshift?
|
86
127
|
Charrington::AlterRedshiftTable.call(connection, event, schema, table_name, columns)
|
87
128
|
end
|
88
129
|
end
|
89
130
|
|
90
131
|
def cleanup
|
91
132
|
@columns.clear if clearable(@columns)
|
133
|
+
@tracks.clear if clearable(@tracks)
|
92
134
|
end
|
93
135
|
|
94
136
|
### Set Variables
|
137
|
+
def create_tracks(event)
|
138
|
+
tracks = event.clone
|
139
|
+
tracks.keys.each do |key|
|
140
|
+
tracks.delete(key) unless tracks_columns.include?(key)
|
141
|
+
end
|
142
|
+
tracks
|
143
|
+
end
|
95
144
|
|
96
145
|
def columns_text
|
97
146
|
@columns_text ||= arr_to_csv(columns)
|
98
147
|
end
|
99
148
|
|
100
|
-
def
|
149
|
+
def tracks_columns_text
|
150
|
+
@tracks_columns_text ||= arr_to_csv(tracks.keys)
|
151
|
+
end
|
152
|
+
|
153
|
+
|
154
|
+
def value_placeholders(columns)
|
101
155
|
('?' * columns.length).split('')
|
102
156
|
end
|
103
157
|
|
104
|
-
def
|
105
|
-
arr_to_csv(value_placeholders)
|
158
|
+
def placeholder_text(columns)
|
159
|
+
arr_to_csv(value_placeholders(columns))
|
106
160
|
end
|
107
161
|
|
108
|
-
def
|
109
|
-
"INSERT INTO #{schema}#{table_name} #{columns_text} VALUES #{
|
162
|
+
def insert_event_statement
|
163
|
+
"INSERT INTO #{schema}#{table_name} #{columns_text} VALUES #{placeholder_text(columns)}"
|
110
164
|
end
|
111
165
|
|
112
|
-
def
|
113
|
-
|
166
|
+
def insert_tracks_statement
|
167
|
+
"INSERT INTO #{schema}tracks #{tracks_columns_text} VALUES #{placeholder_text(tracks.keys)}"
|
114
168
|
end
|
115
169
|
|
116
|
-
def add_statement_event_params(stmt)
|
117
|
-
|
170
|
+
def add_statement_event_params(stmt, map)
|
171
|
+
values = []
|
172
|
+
map.keys.each_with_index do |key, idx|
|
118
173
|
pos = idx + 1
|
119
|
-
value =
|
174
|
+
value = map[key]
|
175
|
+
values << value
|
120
176
|
|
121
177
|
if key == 'published_at' || key == 'sent_at'
|
122
178
|
format = java.text.SimpleDateFormat.new("yyyy-MM-dd'T'HH:mm:ss'Z'")
|
@@ -181,11 +237,9 @@ module Charrington
|
|
181
237
|
statement = connection.prepareStatement( sql.gsub(/\s+/, " ").strip )
|
182
238
|
statement.execute()
|
183
239
|
rescue Java::OrgPostgresqlUtil::PSQLException => e
|
184
|
-
|
185
|
-
self.logger.info "PSQLException: #{e.message}, with SQL: #{sql}"
|
240
|
+
self.logger.error "PSQLException: #{e.message}, with SQL: #{sql}"
|
186
241
|
rescue Java::JavaSql::SQLException => e
|
187
|
-
|
188
|
-
self.logger.info "Redshift SQLException: #{e.message}, with SQL: #{sql}"
|
242
|
+
self.logger.error "Redshift SQLException: #{e.message}, with SQL: #{sql}"
|
189
243
|
ensure
|
190
244
|
statement.close unless statement.nil?
|
191
245
|
end
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
@@ -80,6 +80,42 @@ RSpec.shared_context 'postgres' do
|
|
80
80
|
conn.close if !conn.nil? and !conn.isClosed
|
81
81
|
end
|
82
82
|
|
83
|
+
def create_tracks_table(schema = '')
|
84
|
+
sql = <<-SQL
|
85
|
+
CREATE TABLE IF NOT EXISTS #{ schema.empty? ? schema : schema + '.'}tracks
|
86
|
+
(
|
87
|
+
id VARCHAR(512) NOT NULL CONSTRAINT tracks_pkey PRIMARY KEY,
|
88
|
+
anonymous_id VARCHAR(512),
|
89
|
+
app_name VARCHAR(512),
|
90
|
+
context_campaign_content VARCHAR(512),
|
91
|
+
context_campaign_medium VARCHAR(512),
|
92
|
+
context_campaign_name VARCHAR(512),
|
93
|
+
context_campaign_source VARCHAR(512),
|
94
|
+
context_ip VARCHAR(512),
|
95
|
+
context_library_name VARCHAR(512),
|
96
|
+
context_library_version VARCHAR(512),
|
97
|
+
context_page_path VARCHAR(512),
|
98
|
+
context_page_referrer VARCHAR(512),
|
99
|
+
context_page_search VARCHAR(512),
|
100
|
+
context_page_title VARCHAR(512),
|
101
|
+
context_page_url VARCHAR(512),
|
102
|
+
context_user_agent VARCHAR(512),
|
103
|
+
event VARCHAR(512),
|
104
|
+
event_text VARCHAR(512),
|
105
|
+
original_timestamp TIMESTAMP,
|
106
|
+
received_at TIMESTAMP,
|
107
|
+
segment_dedupe_id VARCHAR(512),
|
108
|
+
sent_at TIMESTAMP,
|
109
|
+
timestamp TIMESTAMP,
|
110
|
+
user_id VARCHAR(512),
|
111
|
+
user_uid VARCHAR(512),
|
112
|
+
uuid bigint,
|
113
|
+
uuid_ts TIMESTAMP DEFAULT ('now'::text)::TIMESTAMP without TIME ZONE
|
114
|
+
)
|
115
|
+
SQL
|
116
|
+
create(sql)
|
117
|
+
end
|
118
|
+
|
83
119
|
def drop_table(table)
|
84
120
|
execute_update("DROP TABLE IF EXISTS #{table}")
|
85
121
|
end
|
@@ -27,6 +27,8 @@ describe LogStash::Outputs::Charrington do
|
|
27
27
|
end
|
28
28
|
|
29
29
|
it 'creates a table and inserts a record' do
|
30
|
+
drop_table('tracks')
|
31
|
+
create_tracks_table
|
30
32
|
drop_table('schemaless')
|
31
33
|
run_pipeline
|
32
34
|
expect(query('SELECT * FROM schemaless')).to match_array([{id: "1", app_name: "Web App", event: "schemaless", inserted_at: a_kind_of(String), :meta_type => "XML", :published_at => "2019-07-29 20:09:18"}])
|
@@ -90,6 +92,8 @@ describe LogStash::Outputs::Charrington do
|
|
90
92
|
end
|
91
93
|
|
92
94
|
it 'creates a table and inserts the first record and alters the table for the second record' do
|
95
|
+
drop_table('tracks')
|
96
|
+
create_tracks_table
|
93
97
|
drop_table('metadata')
|
94
98
|
run_pipeline
|
95
99
|
expect(query('SELECT * FROM metadata')).to match_array([
|
@@ -123,6 +127,8 @@ describe LogStash::Outputs::Charrington do
|
|
123
127
|
end
|
124
128
|
|
125
129
|
it 'can insert into an existing table' do
|
130
|
+
drop_table('tracks')
|
131
|
+
create_tracks_table
|
126
132
|
# setup pre-existing data
|
127
133
|
drop_table('existing')
|
128
134
|
create_table('CREATE TABLE existing (id SERIAL PRIMARY KEY, inserted_at TIMESTAMP DEFAULT NOW(), app_name VARCHAR(255))')
|
@@ -164,6 +170,8 @@ describe LogStash::Outputs::Charrington do
|
|
164
170
|
end
|
165
171
|
|
166
172
|
it 'creates a table and inserts the first record and alters the table for the second record' do
|
173
|
+
drop_table('tracks')
|
174
|
+
create_tracks_table('dea_test')
|
167
175
|
create("CREATE SCHEMA IF NOT EXISTS #{schema}")
|
168
176
|
drop_table("#{schema}.from_agent")
|
169
177
|
run_pipeline
|
@@ -210,7 +218,7 @@ describe LogStash::Outputs::Charrington do
|
|
210
218
|
<<-CONFIG
|
211
219
|
input {
|
212
220
|
generator {
|
213
|
-
message => '{"app_name": "Web App", "event": "schemaless", "meta": { "type": "XML" }, "published_at": "2019-07-29T20:09:18Z" }'
|
221
|
+
message => '{"app_name": "Web App", "event": "schemaless", "meta": { "type": "XML" }, "published_at": "2019-07-29T20:09:18Z", "user_id": "123", "user_uid": "456" }'
|
214
222
|
codec => 'json'
|
215
223
|
count => 1
|
216
224
|
}
|
@@ -228,12 +236,17 @@ describe LogStash::Outputs::Charrington do
|
|
228
236
|
end
|
229
237
|
|
230
238
|
it 'creates a table and inserts a record with data the shape of redshift' do
|
239
|
+
drop_table('tracks')
|
240
|
+
create_tracks_table
|
231
241
|
drop_table('schemaless')
|
232
242
|
run_pipeline
|
233
243
|
expect(query('SELECT * FROM schemaless')).to match_array([
|
234
|
-
{:anonymous_id=>"", :app_name=>"Web App", :event=>"schemaless", :event_text=>"schemaless", :id=>a_kind_of(String), :original_timestamp=>a_kind_of(String), :received_at=>a_kind_of(String), :sent_at=>"2019-07-29 20:09:18", :timestamp=>a_kind_of(String), :type=>"XML", :uuid_ts=>a_kind_of(String)}
|
244
|
+
{:anonymous_id=>"", :app_name=>"Web App", :event=>"schemaless", :event_text=>"schemaless", :id=>a_kind_of(String), :original_timestamp=>a_kind_of(String), :received_at=>a_kind_of(String), :sent_at=>"2019-07-29 20:09:18", :timestamp=>a_kind_of(String), :type=>"XML", :user_id=>"123", :user_uid=>"456", :uuid_ts=>a_kind_of(String)}
|
235
245
|
])
|
236
246
|
expect(query('SELECT COUNT(1) FROM schemaless').first[:count]).to eq("1")
|
247
|
+
expect(query("SELECT * FROM tracks")).to match_array([
|
248
|
+
{:id=>a_kind_of(String), :anonymous_id=>"", :app_name=>"Web App", :context_campaign_content=>nil, :context_campaign_medium=>nil, :context_campaign_name=>nil, :context_campaign_source=>nil, :context_ip=>nil, :context_library_name=>nil, :context_library_version=>nil, :context_page_path=>nil, :context_page_referrer=>nil, :context_page_search=>nil, :context_page_title=>nil, :context_page_url=>nil, :context_user_agent=>nil, :event=>"schemaless", :event_text=>"schemaless", :original_timestamp=>nil, :received_at=>nil, :segment_dedupe_id=>nil, :sent_at=>a_kind_of(String), :timestamp=>nil, :user_id=>"123", :user_uid=>"456", :uuid=>nil, :uuid_ts=>a_kind_of(String)}
|
249
|
+
])
|
237
250
|
end
|
238
251
|
end
|
239
252
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-charrington
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.14
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- dconger
|
@@ -182,6 +182,8 @@ files:
|
|
182
182
|
- README.md
|
183
183
|
- THANKS.md
|
184
184
|
- lib/com/zaxxer/HikariCP/2.7.2/HikariCP-2.7.2.jar
|
185
|
+
- lib/commons-io/commons-io/2.4/commons-io-2.4.jar
|
186
|
+
- lib/de/flapdoodle/embed/de.flapdoodle.embed.process/2.0.2/de.flapdoodle.embed.process-2.0.2.jar
|
185
187
|
- lib/logstash-output-charrington_jars.rb
|
186
188
|
- lib/logstash/outputs/charrington.rb
|
187
189
|
- lib/logstash/outputs/charrington/alter_postgres_table.rb
|
@@ -193,9 +195,16 @@ files:
|
|
193
195
|
- lib/logstash/outputs/charrington/service.rb
|
194
196
|
- lib/logstash/outputs/charrington/transform_postgres.rb
|
195
197
|
- lib/logstash/outputs/charrington/transform_redshift.rb
|
198
|
+
- lib/net/java/dev/jna/jna-platform/4.0.0/jna-platform-4.0.0.jar
|
199
|
+
- lib/net/java/dev/jna/jna/4.0.0/jna-4.0.0.jar
|
200
|
+
- lib/org/apache/commons/commons-compress/1.10/commons-compress-1.10.jar
|
201
|
+
- lib/org/apache/commons/commons-lang3/3.1/commons-lang3-3.1.jar
|
196
202
|
- lib/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar
|
197
203
|
- lib/org/apache/logging/log4j/log4j-slf4j-impl/2.6.2/log4j-slf4j-impl-2.6.2.jar
|
204
|
+
- lib/org/postgresql/postgresql/42.2.5/postgresql-42.2.5.jar
|
198
205
|
- lib/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar
|
206
|
+
- lib/ru/yandex/qatools/embed/postgresql-embedded/2.10/postgresql-embedded-2.10.jar
|
207
|
+
- lib/ru/yandex/qatools/embed/postgresql-embedded/2.8/postgresql-embedded-2.8.jar
|
199
208
|
- logstash-output-charrington.gemspec
|
200
209
|
- spec/charrington_spec_helper.rb
|
201
210
|
- spec/logstash-output-charrington_test_jars.rb
|