logstash-output-charrington 0.3.12 → 0.3.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3e19aa1ec683b1b07dd8bd6cb7b715e382d9bcf97f839339be15c47e49749db3
-  data.tar.gz: 845b806354d17539f84b7b3b31c6925a1c5220d85364a89d61086e3734fa7084
+  metadata.gz: 9bb02ce087011aef5d96e4a5ea26e7151ff87d185e24b2812a24c95319c322b0
+  data.tar.gz: 6a5ea166ed113d948d766339f35f049e68bd2309e23899e25c7ae5e9345dce33
 SHA512:
-  metadata.gz: f3f61f69b23d8a28db703c1ca76952056953cd223a715322db5b0f422a1ad18d561e0d28dacb2518eaaa8799c3a9b08b718818129a41a5cd54e430dbd33c11fc
-  data.tar.gz: 6dff6e868f6264a77c70af63f018f8ab150f8097e03acf4521b2e068e61dce49baa3dc520aabc71be60bcf276bf33f90a8d5271d2f5e7b360c7a265b8b1e5f84
+  metadata.gz: 1a963e2a2c2a6e182fcb63c202b857e55814e7dc4cf7983bc7e34e3f3af7141ecb9c7bd656c52d15559905f978993cc9442e33ce29f927e0140e0fc2553fadf1
+  data.tar.gz: e257a03e1a42b74c14b01b0a4cd5e67ad285e8a8de9e3a3b7f5bd934c8c97c228149808e1d429bf17d3f1ee3658ba9b84bc326a16b163f0212a1a9f9abe14c28

data/lib/logstash/outputs/charrington.rb

@@ -85,6 +85,8 @@ class LogStash::Outputs::Charrington < LogStash::Outputs::Base
   # The database schema
   config :schema, validate: :string, required: false
 
+  config :transformer, validate: :string, default: 'redshift'
+
   def register
     @logger.info('JDBC - Starting up')
 
@@ -105,7 +107,8 @@ class LogStash::Outputs::Charrington < LogStash::Outputs::Base
                schema: schema,
                max_retries: @max_flush_exceptions,
                retry_initial_interval: @retry_initial_interval,
-               driver: driver }
+               driver: driver,
+               transformer: @transformer }
       Charrington::Process.call(connection, event, opts)
     rescue => e
       @logger.error("Unable to process event. Event dropped. #{e.message}")

data/lib/logstash/outputs/charrington/create_postgres_table.rb

@@ -7,18 +7,19 @@ module Charrington
 
     include Service
     include LogStash::Util::Loggable
-    attr_reader :connection, :event, :table_name, :columns, :schema
+    attr_reader :connection, :event, :table_name, :columns, :schema, :opts, :transformer
     attr_accessor :column_types
 
     Error = Class.new(StandardError)
     CreateFailed = Class.new(Error)
 
-    def initialize(connection, event, schema, table_name, columns)
+    def initialize(connection, event, schema, table_name, columns, opts = {})
       @connection = connection
       @event = event.to_hash
       @table_name = table_name
       @schema = schema
       @columns = columns
+      @transformer = opts[:transformer]
       @column_types = initial_columns
     end
 
@@ -61,8 +62,19 @@ module Charrington
     end
 
     def initial_columns
-      # Postgres
-      ["id SERIAL PRIMARY KEY", "inserted_at TIMESTAMP DEFAULT NOW()"]
+      if transformer == "postgres"
+        [
+          "id SERIAL PRIMARY KEY",
+          "inserted_at TIMESTAMP DEFAULT NOW()"
+        ]
+      else
+        [
+          "original_timestamp TIMESTAMP DEFAULT NOW()",
+          "received_at TIMESTAMP DEFAULT NOW()",
+          "timestamp TIMESTAMP DEFAULT NOW()",
+          "uuid_ts TIMESTAMP DEFAULT NOW()"
+        ]
+      end
     end
 
     def create_table

data/lib/logstash/outputs/charrington/create_redshift_table.rb

@@ -7,13 +7,13 @@ module Charrington
 
     include Service
     include LogStash::Util::Loggable
-    attr_reader :connection, :event, :table_name, :columns, :schema
+    attr_reader :connection, :event, :table_name, :columns, :schema, :opts
     attr_accessor :column_types
 
     Error = Class.new(StandardError)
     CreateFailed = Class.new(Error)
 
-    def initialize(connection, event, schema, table_name, columns)
+    def initialize(connection, event, schema, table_name, columns, opts = {})
       @connection = connection
       @event = event.to_hash
       @schema = schema

data/lib/logstash/outputs/charrington/insert.rb

@@ -12,7 +12,7 @@ module Charrington
     include Service
     include LogStash::Util::Loggable
     attr_accessor :event, :should_retry
-    attr_reader :connection, :schema, :table_name, :columns, :driver
+    attr_reader :connection, :schema, :table_name, :columns, :driver, :opts
     attr_reader :event_as_json_keyword, :enable_event_as_json_keyword
 
     Error = Class.new(StandardError)
@@ -36,6 +36,7 @@ module Charrington
       @enable_event_as_json_keyword = opts[:enable_event_as_json_keyword]
       @event_as_json_keyword = opts[:event_as_json_keyword]
       @driver = opts[:driver]
+      @opts = opts
     end
 
     def call
@@ -72,9 +73,9 @@ module Charrington
 
     def create_table
       if driver == "postgres"
-        Charrington::CreatePostgresTable.call(connection, event, schema, table_name, columns)
+        Charrington::CreatePostgresTable.call(connection, event, schema, table_name, columns, opts)
       elsif driver == "redshift"
-        Charrington::CreateRedshiftTable.call(connection, event, schema, table_name, columns)
+        Charrington::CreateRedshiftTable.call(connection, event, schema, table_name, columns, opts)
       end
     end
 

data/lib/logstash/outputs/charrington/process.rb

@@ -7,7 +7,7 @@ module Charrington
 
     include Service
     include LogStash::Util::Loggable
-    attr_reader :event, :connection, :opts, :max_retries, :schema, :retry_max_interval, :driver
+    attr_reader :event, :connection, :opts, :max_retries, :schema, :retry_max_interval, :driver, :transformer
     attr_accessor :retry_interval, :should_retry
 
     Error = Class.new(StandardError)
@@ -24,6 +24,7 @@ module Charrington
       @retry_max_interval = opts[:retry_max_interval] || 2
       @retry_interval = opts[:retry_initial_interval] || 2
       @driver = opts[:driver]
+      @transformer = opts[:transformer]
 
       @attempts = 1
       @should_retry = true
@@ -31,12 +32,12 @@ module Charrington
 
     def call
       while should_retry do
-        transformed = case driver
+        transformed = case transformer
                         when "redshift"
-                          self.logger.info "Found driver for redshift with event of: #{event}"
+                          self.logger.info "Found transformer of #{transformer} for driver of #{driver} with event of: #{event}"
                           Charrington::TransformRedshift.call(event)
                         else
-                          self.logger.info "Found driver for postgres with event of: #{event}"
+                          self.logger.info "Found transformer of #{transformer} for driver of #{driver} with event of: #{event}"
                           Charrington::TransformPostgres.call(event)
                         end
         self.logger.info "Transformed event into: #{transformed}"

data/logstash-output-charrington.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-charrington'
-  s.version = '0.3.12'
+  s.version = '0.3.13'
 
   s.licenses = ['Apache-2.0']
   s.homepage = 'https://gitlab.podium.com/engineering/analytics/logstash-output-charrington'

data/spec/outputs/charrington_spec.rb

@@ -20,6 +20,7 @@ describe LogStash::Outputs::Charrington do
           connection_string => '#{@url}'
           driver_jar_path => '#{driver_path}'
           schema => ''
+          transformer => 'postgres'
         }
       }
       CONFIG
@@ -49,6 +50,7 @@ describe LogStash::Outputs::Charrington do
           connection_string => '#{@url}'
           driver_jar_path => '#{driver_path}'
           schema => ''
+          transformer => 'postgres'
         }
       }
       CONFIG
@@ -81,6 +83,7 @@ describe LogStash::Outputs::Charrington do
           connection_string => '#{@url}'
           driver_jar_path => '#{driver_path}'
           schema => ''
+          transformer => 'postgres'
         }
       }
       CONFIG
@@ -113,6 +116,7 @@ describe LogStash::Outputs::Charrington do
           connection_string => '#{@url}'
           driver_jar_path => '#{driver_path}'
           schema => ''
+          transformer => 'postgres'
         }
       }
       CONFIG
@@ -153,6 +157,7 @@ describe LogStash::Outputs::Charrington do
           connection_string => '#{@url}'
           driver_jar_path => '#{driver_path}'
           schema => '#{schema}'
+          transformer => 'postgres'
         }
       }
       CONFIG
@@ -186,6 +191,7 @@ describe LogStash::Outputs::Charrington do
           connection_string => '#{@url}'
           driver_jar_path => '#{driver_path}'
           schema => ''
+          transformer => 'postgres'
         }
       }
       CONFIG
@@ -199,4 +205,36 @@ describe LogStash::Outputs::Charrington do
     end
   end
 
+  describe 'a new payload with one event transformed to be redshift shape' do
+    let(:config) do
+      <<-CONFIG
+      input {
+        generator {
+          message => '{"app_name": "Web App", "event": "schemaless", "meta": { "type": "XML" }, "published_at": "2019-07-29T20:09:18Z" }'
+          codec => 'json'
+          count => 1
+        }
+      }
+
+      output {
+        charrington {
+          connection_string => '#{@url}'
+          driver_jar_path => '#{driver_path}'
+          schema => ''
+          transformer => 'redshift'
+        }
+      }
+      CONFIG
+    end
+
+    it 'creates a table and inserts a record with data the shape of redshift' do
+      drop_table('schemaless')
+      run_pipeline
+      expect(query('SELECT * FROM schemaless')).to match_array([
+        {:anonymous_id=>"", :app_name=>"Web App", :event=>"schemaless", :event_text=>"schemaless", :id=>a_kind_of(String), :original_timestamp=>a_kind_of(String), :received_at=>a_kind_of(String), :sent_at=>"2019-07-29 20:09:18", :timestamp=>a_kind_of(String), :type=>"XML", :uuid_ts=>a_kind_of(String)}
+      ])
+      expect(query('SELECT COUNT(1) FROM schemaless').first[:count]).to eq("1")
+    end
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-charrington
 version: !ruby/object:Gem::Version
-  version: 0.3.12
+  version: 0.3.13
 platform: ruby
 authors:
 - dconger