logstash-output-charrington 0.3.9 → 0.3.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e031c490f1a81f62f0a781e259ce3872712255e014e11a41fc7420e120368fd8
-  data.tar.gz: f54fb87f22fed7eda64e068897a2942323f08fb54dd9cac4972ce58a3ae0217e
+  metadata.gz: 3e19aa1ec683b1b07dd8bd6cb7b715e382d9bcf97f839339be15c47e49749db3
+  data.tar.gz: 845b806354d17539f84b7b3b31c6925a1c5220d85364a89d61086e3734fa7084
 SHA512:
-  metadata.gz: eeef47c0d66f3b1533adb9eb539e4db501b190b1ac47c89b1ad71c1e12f6748b7a957fbb4f3e08c58e75d45ae50186469e871ceb7e487a7595e81901a04f1ef7
-  data.tar.gz: 8f8da734bb7054bf04da564032a2d17402c905e2160ed03a78dc05f35b05faf6d67312566cc278c88ca521c2f8f86ef03d9ac9d3cd42e02d096fde38f4022eb0
+  metadata.gz: f3f61f69b23d8a28db703c1ca76952056953cd223a715322db5b0f422a1ad18d561e0d28dacb2518eaaa8799c3a9b08b718818129a41a5cd54e430dbd33c11fc
+  data.tar.gz: 6dff6e868f6264a77c70af63f018f8ab150f8097e03acf4521b2e068e61dce49baa3dc520aabc71be60bcf276bf33f90a8d5271d2f5e7b360c7a265b8b1e5f84

data/lib/logstash/outputs/charrington/create_postgres_table.rb

@@ -36,6 +36,11 @@ module Charrington
 
     def set_column_types
       columns.each do |column|
+        if column == "published_at" || column == "sent_at"
+          column_types << "#{column} TIMESTAMP"
+          next
+        end
+
         case event[column]
         when Time, LogStash::Timestamp
           column_types << "#{column} TIMESTAMP"

data/lib/logstash/outputs/charrington/create_redshift_table.rb

@@ -41,6 +41,9 @@ module Charrington
         if column == "id"
           column_types << "#{column} VARCHAR(512) NOT NULL distkey CONSTRAINT #{table_name}_pkey primary key"
           next
+        elsif column == "sent_at"
+          column_types << "#{column} TIMESTAMP"
+          next
         end
         case event[column]
         when Time, LogStash::Timestamp

data/lib/logstash/outputs/charrington/insert.rb

@@ -23,16 +23,13 @@ module Charrington
     def initialize(connection, event, opts = {})
       raise EventNil, "Table name is nil" if event.nil?
       @event = event.to_hash
-      event_name = event["event"].to_s.downcase.strip
+      event_name = event["event"].to_s.strip
       raise TableNameNil, "Table name is nil" if event_name.empty?
 
       @connection = connection
       @schema = opts[:schema].empty? ? '' : "#{opts[:schema]}."
 
-      @table_name = "#{event_name
-                         .gsub(/[^a-z0-9]+/, "_")
-                         .gsub(/\A_+/, "")
-                         .gsub(/_+\z/, "")}"
+      @table_name = underscore(event_name)
 
       @columns = event.keys
       @should_retry = false
@@ -120,6 +117,14 @@ module Charrington
         pos = idx + 1
         value = event[key]
 
+        if key == 'published_at' || key == 'sent_at'
+          format = java.text.SimpleDateFormat.new("yyyy-MM-dd'T'HH:mm:ss'Z'")
+          parsed = format.parse(value)
+          time = java.sql.Timestamp.new(parsed.getTime)
+          stmt.setTimestamp(pos, time)
+          next
+        end
+
         case value
         when Time
           stmt.setString(pos, value.strftime(STRFTIME_FMT))
@@ -158,6 +163,17 @@ module Charrington
       obj.is_a? Hash or obj.is_a? Array
     end
 
+    def underscore(str)
+      str.
+      gsub(/::/, '/').
+      gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+      gsub(/([a-z\d])([A-Z])/,'\1_\2').
+      downcase.
+      gsub(/[^a-z0-9]+/, "_").
+      gsub(/\A_+/, "").
+      gsub(/_+\z/, "")
+    end
+
     ### SQL
 
     def execute(connection, sql)

data/lib/logstash/outputs/charrington/transform_redshift.rb

@@ -62,7 +62,14 @@ module Charrington
     end
 
     def underscore_event_name(event_name)
-      event_name.to_s.downcase.strip.gsub(/[^a-z0-9]+/, "_")
+      event_name.
+      gsub(/::/, '/').
+      gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+      gsub(/([a-z\d])([A-Z])/,'\1_\2').
+      downcase.
+      gsub(/[^a-z0-9]+/, "_").
+      gsub(/\A_+/, "").
+      gsub(/_+\z/, "")
     end
 
     def transform_session_stuff(hash)

data/logstash-output-charrington.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-charrington'
-  s.version = '0.3.9'
+  s.version = '0.3.12'
 
   s.licenses = ['Apache-2.0']
   s.homepage = 'https://gitlab.podium.com/engineering/analytics/logstash-output-charrington'

data/spec/outputs/charrington_spec.rb

@@ -9,7 +9,7 @@ describe LogStash::Outputs::Charrington do
       <<-CONFIG
       input {
         generator {
-          message => '{"app_name": "Web App", "event": "schemaless", "meta": { "type": "XML" } }'
+          message => '{"app_name": "Web App", "event": "schemaless", "meta": { "type": "XML" }, "published_at": "2019-07-29T20:09:18Z" }'
           codec => 'json'
           count => 1
         }
@@ -19,6 +19,7 @@ describe LogStash::Outputs::Charrington do
         charrington {
           connection_string => '#{@url}'
           driver_jar_path => '#{driver_path}'
+          schema => ''
         }
       }
       CONFIG
@@ -27,7 +28,7 @@ describe LogStash::Outputs::Charrington do
     it 'creates a table and inserts a record' do
       drop_table('schemaless')
       run_pipeline
-      expect(query('SELECT * FROM schemaless')).to match_array([{id: "1", app_name: "Web App", event: "schemaless", inserted_at: a_kind_of(String), :meta_type => "XML"}])
+      expect(query('SELECT * FROM schemaless')).to match_array([{id: "1", app_name: "Web App", event: "schemaless", inserted_at: a_kind_of(String), :meta_type => "XML", :published_at => "2019-07-29 20:09:18"}])
       expect(query('SELECT COUNT(1) FROM schemaless').first[:count]).to eq("1")
     end
   end
@@ -47,6 +48,7 @@ describe LogStash::Outputs::Charrington do
         charrington {
           connection_string => '#{@url}'
           driver_jar_path => '#{driver_path}'
+          schema => ''
         }
       }
       CONFIG
@@ -78,6 +80,7 @@ describe LogStash::Outputs::Charrington do
         charrington {
           connection_string => '#{@url}'
           driver_jar_path => '#{driver_path}'
+          schema => ''
         }
       }
       CONFIG
@@ -109,6 +112,7 @@ describe LogStash::Outputs::Charrington do
         charrington {
           connection_string => '#{@url}'
           driver_jar_path => '#{driver_path}'
+          schema => ''
         }
       }
       CONFIG
@@ -166,6 +170,33 @@ describe LogStash::Outputs::Charrington do
     end
   end
 
-end
+  describe 'a new payload with one event camelCase' do
+    let(:config) do
+      <<-CONFIG
+      input {
+        generator {
+          message => '{"app_name": "Web App", "event": "camelCase this", "meta": { "type": "XML" } }'
+          codec => 'json'
+          count => 1
+        }
+      }
 
+      output {
+        charrington {
+          connection_string => '#{@url}'
+          driver_jar_path => '#{driver_path}'
+          schema => ''
+        }
+      }
+      CONFIG
+    end
 
+    it 'creates a table and inserts a record' do
+      drop_table('camel_case_this')
+      run_pipeline
+      expect(query('SELECT * FROM camel_case_this')).to match_array([{id: "1", app_name: "Web App", event: "camelCase this", inserted_at: a_kind_of(String), :meta_type => "XML"}])
+      expect(query('SELECT COUNT(1) FROM camel_case_this').first[:count]).to eq("1")
+    end
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-charrington
 version: !ruby/object:Gem::Version
-  version: 0.3.9
+  version: 0.3.12
 platform: ruby
 authors:
 - dconger
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-07-29 00:00:00.000000000 Z
+date: 2019-07-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement