logstash-output-charrington 0.3.15 → 0.3.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5bc7440fc12c2dee8a2c47f7c30c9f95cd11c7ecf6905ccefc5cbecf21e6588b
-  data.tar.gz: 409036adf2e56121fac87141d06686beda6d0d878bc7f1de1de14b58e58efd7b
+  metadata.gz: 94729829f8d0a679e21b264079ba18ded80a374afc2eedff99a063b9c40381bf
+  data.tar.gz: 382f8290e530f6ee78a93f53ab81ce7f5c6ab27385795aeb313ca8309f5ed1bf
 SHA512:
-  metadata.gz: d793f1abbf99da18b9076bb1c36d101bc52657694cdef5ef645c80dc39d16092eedbd28b423fc45285934514c2210ccfb1f1ca2288fe947d501c3a08d7b92a1e
-  data.tar.gz: 0500b6e35d903f450f29c3e521073486b5f523ffec9f3c89111610c65adf75e3b778050ca2340724dcbf03611dd07a57cac6f84bd796a50427bb1e8ee276919e
+  metadata.gz: 5b9e425e3eaf61785ce59e63fea1f39db54611f1cc1176bc5fc82c5be49a730b94627a394164ee76014a2a80d4940fc0fbde0b0582cf84e3e45f673195ba041f
+  data.tar.gz: 66f92a5bfb76cb400add4feff87409600baff3f70ede790ee98904df6bd9786831acc4a567167b64768b90f6048640d97c73cb8994c9a0e94c0b06ed4302c3ef

data/lib/logstash/outputs/charrington/create_postgres_table.rb

@@ -5,6 +5,8 @@ module Charrington
     # This service will create a table dynamically based on the JSON structure.
     # This is potentially called from Insert when an insert fails.
 
+    @@timestamp_columns = %w(published_at sent_at original_timestamp received_at timestamp)
+
     include Service
     include LogStash::Util::Loggable
     attr_reader :connection, :event, :table_name, :columns, :schema, :opts, :transformer
@@ -37,7 +39,7 @@ module Charrington
 
     def set_column_types
       columns.each do |column|
-        if column == "published_at" || column == "sent_at"
+        if @@timestamp_columns.include?(column)
           column_types << "#{column} TIMESTAMP"
           next
         end
@@ -56,7 +58,7 @@ module Charrington
         when true, false
           column_types << "#{column} BOOLEAN"
         else
-          column_types << "#{column} VARCHAR"
+          column_types << "#{column} VARCHAR(512)"
         end
       end
     end
@@ -69,9 +71,6 @@ module Charrington
         ]
       else
         [
-          "original_timestamp TIMESTAMP DEFAULT NOW()",
-          "received_at TIMESTAMP DEFAULT NOW()",
-          "timestamp TIMESTAMP DEFAULT NOW()",
           "uuid_ts TIMESTAMP DEFAULT NOW()"
         ]
       end

data/lib/logstash/outputs/charrington/insert.rb

@@ -18,6 +18,7 @@ module Charrington
     # TODO create a current_table_columns (alter_postgres_table.rb) query on the tracks table to get the current columns
     @@redshift_tracks_columns = %w(id app_name received_at uuid uuid_ts anonymous_id context_ip context_library_name context_library_version context_page_path context_page_referrer context_page_title context_page_url context_user_agent event event_text original_timestamp sent_at timestamp user_id user_uid context_campaign_medium context_campaign_name context_page_search context_campaign_source segment_dedupe_id context_campaign_content)
     @@postgres_tracks_columns = %w(anonymous_user app_name event published_at session_ip session_library_name session_library_version session_page_path session_page_referrer session_page_search session_page_title session_page_url session_user_agent user_id user_uid)
+    @@timestamp_columns = %w(published_at sent_at original_timestamp received_at timestamp)
 
     Error = Class.new(StandardError)
     EventNil = Class.new(Error)
@@ -174,7 +175,7 @@ module Charrington
         value = map[key]
         values << value
 
-        if key == 'published_at' || key == 'sent_at'
+        if @@timestamp_columns.include?(key)
           format = java.text.SimpleDateFormat.new("yyyy-MM-dd'T'HH:mm:ss'Z'")
           parsed = format.parse(value)
           time = java.sql.Timestamp.new(parsed.getTime)

data/lib/logstash/outputs/charrington/transform_redshift.rb

@@ -29,6 +29,9 @@ module Charrington
 
       handle_key_transform(event, "anonymous_id", "anonymous_user")
       handle_key_transform(event, "sent_at", "published_at")
+      handle_key_transform(event, "original_timestamp", "sent_at")
+      handle_key_transform(event, "received_at", "sent_at")
+      handle_key_transform(event, "timestamp", "sent_at")
 
       handle_meta_section(event)
 

data/logstash-output-charrington.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-charrington'
-  s.version = '0.3.15'
+  s.version = '0.3.16'
 
   s.licenses = ['Apache-2.0']
   s.homepage = 'https://gitlab.podium.com/engineering/analytics/logstash-output-charrington'

data/spec/outputs/charrington_spec.rb

@@ -253,4 +253,37 @@ describe LogStash::Outputs::Charrington do
     end
   end
 
+  describe 'a new payload has timestamp-related keys set to sent_at' do
+    let(:config) do
+      <<-CONFIG
+      input {
+        generator {
+          message => '{"app_name": "Web App", "event": "schemaless", "meta": { "type": "XML" }, "published_at": "2019-07-29T20:09:18Z", "user_id": "123", "user_uid": "456" }'
+          codec => 'json'
+          count => 1
+        }
+      }
+
+      output {
+        charrington {
+          connection_string => '#{@url}'
+          driver_jar_path => '#{driver_path}'
+          schema => ''
+          transformer => 'redshift'
+        }
+      }
+      CONFIG
+    end
+
+    it 'creates a table with timestamp-related keys and inserts a record with timestamp-related keys same as sent_at' do
+      drop_table('tracks')
+      create_tracks_table
+      drop_table('schemaless')
+      run_pipeline
+      expect(query('SELECT * FROM schemaless')).to match_array([
+        {:anonymous_id=>"", :app_name=>"Web App", :event=>"schemaless", :event_text=>"schemaless", :id=>a_kind_of(String), :original_timestamp=>"2019-07-29 20:09:18", :received_at=>"2019-07-29 20:09:18", :sent_at=>"2019-07-29 20:09:18", :timestamp=>"2019-07-29 20:09:18", :type=>"XML", :user_id=>"123", :user_uid=>"456", :uuid_ts=>a_kind_of(String)}
+      ])
+      expect(query('SELECT COUNT(1) FROM schemaless').first[:count]).to eq("1")
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-charrington
 version: !ruby/object:Gem::Version
-  version: 0.3.15
+  version: 0.3.16
 platform: ruby
 authors:
 - dconger
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-07-31 00:00:00.000000000 Z
+date: 2019-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -182,8 +182,6 @@ files:
 - README.md
 - THANKS.md
 - lib/com/zaxxer/HikariCP/2.7.2/HikariCP-2.7.2.jar
-- lib/commons-io/commons-io/2.4/commons-io-2.4.jar
-- lib/de/flapdoodle/embed/de.flapdoodle.embed.process/2.0.2/de.flapdoodle.embed.process-2.0.2.jar
 - lib/logstash-output-charrington_jars.rb
 - lib/logstash/outputs/charrington.rb
 - lib/logstash/outputs/charrington/alter_postgres_table.rb
@@ -195,16 +193,9 @@ files:
 - lib/logstash/outputs/charrington/service.rb
 - lib/logstash/outputs/charrington/transform_postgres.rb
 - lib/logstash/outputs/charrington/transform_redshift.rb
-- lib/net/java/dev/jna/jna-platform/4.0.0/jna-platform-4.0.0.jar
-- lib/net/java/dev/jna/jna/4.0.0/jna-4.0.0.jar
-- lib/org/apache/commons/commons-compress/1.10/commons-compress-1.10.jar
-- lib/org/apache/commons/commons-lang3/3.1/commons-lang3-3.1.jar
 - lib/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar
 - lib/org/apache/logging/log4j/log4j-slf4j-impl/2.6.2/log4j-slf4j-impl-2.6.2.jar
-- lib/org/postgresql/postgresql/42.2.5/postgresql-42.2.5.jar
 - lib/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar
-- lib/ru/yandex/qatools/embed/postgresql-embedded/2.10/postgresql-embedded-2.10.jar
-- lib/ru/yandex/qatools/embed/postgresql-embedded/2.8/postgresql-embedded-2.8.jar
 - logstash-output-charrington.gemspec
 - spec/charrington_spec_helper.rb
 - spec/logstash-output-charrington_test_jars.rb