logstash-output-charrington 0.3.22 → 0.3.26

data/lib/logstash/outputs/charrington/insert.rb

@@ -1,24 +1,25 @@
-require File.join(File.dirname(__FILE__), "create_postgres_table")
-require File.join(File.dirname(__FILE__), "create_redshift_table")
-require File.join(File.dirname(__FILE__), "alter_postgres_table")
-require File.join(File.dirname(__FILE__), "alter_redshift_table")
-require File.join(File.dirname(__FILE__), "service")
+# frozen_string_literal: true
+
+require File.join(File.dirname(__FILE__), 'create_postgres_table')
+require File.join(File.dirname(__FILE__), 'create_redshift_table')
+require File.join(File.dirname(__FILE__), 'alter_postgres_table')
+require File.join(File.dirname(__FILE__), 'alter_redshift_table')
+require File.join(File.dirname(__FILE__), 'service')
 require 'time'
 
 module Charrington
-  class Insert
-    # This service assumes that the data is already clean and in a flattened hash format.
-    # The Transform service should be called before calling this.
-
+  # This service assumes that the data is already clean and in a flattened hash format.
+  # The Transform service should be called before calling this.
+  class Insert # rubocop:disable Metrics/ClassLength
     include Service
     include LogStash::Util::Loggable
     attr_accessor :event, :should_retry
-    attr_reader :connection, :schema, :table_name, :columns, :driver, :opts, :transformer, :tracks
-    attr_reader :event_as_json_keyword, :enable_event_as_json_keyword
-    # TODO create a current_table_columns (alter_postgres_table.rb) query on the tracks table to get the current columns
-    @@redshift_tracks_columns = %w(id action app_name received_at uuid uuid_ts anonymous_id context_ip context_library_name context_library_version context_page_path context_page_referrer context_page_title context_page_url context_user_agent event event_text original_timestamp sent_at timestamp user_id user_uid context_campaign_medium context_campaign_name context_page_search context_campaign_source segment_dedupe_id context_campaign_content)
-    @@postgres_tracks_columns = %w(anonymous_user app_name event published_at session_ip session_library_name session_library_version session_page_path session_page_referrer session_page_search session_page_title session_page_url session_user_agent user_id user_uid)
-    @@timestamp_columns = %w(published_at sent_at original_timestamp received_at timestamp)
+    attr_reader :connection, :schema, :table_name, :columns, :driver, :opts, :transformer, :tracks, :event_as_json_keyword, :enable_event_as_json_keyword
+
+    # TODO: create a current_table_columns (alter_postgres_table.rb) query on the tracks table to get the current columns
+    REDSHIFT_TRACKS_COLUMNS = %w[id action app_name received_at uuid uuid_ts anonymous_id context_ip context_library_name context_library_version context_page_path context_page_referrer context_page_title context_page_url context_user_agent event event_text original_timestamp sent_at timestamp user_id user_uid context_campaign_medium context_campaign_name context_page_search context_campaign_source segment_dedupe_id context_campaign_content].freeze
+    POSTGRES_TRACKS_COLUMNS = %w[anonymous_user app_name event published_at session_ip session_library_name session_library_version session_page_path session_page_referrer session_page_search session_page_title session_page_url session_user_agent user_id user_uid].freeze
+    TIMESTAMP_COLUMNS = %w[published_at sent_at original_timestamp received_at timestamp].freeze
 
     Error = Class.new(StandardError)
     EventNil = Class.new(Error)
@@ -26,19 +27,20 @@ module Charrington
     InsertFailed = Class.new(Error)
 
     def initialize(connection, event, opts = {})
-      raise EventNil, "Table name is nil" if event.nil?
+      raise EventNil, 'Table name is nil' if event.nil?
+
       @transformer = opts[:transformer]
       @event = event.to_hash
       @tracks = create_tracks(@event)
-      event_name = event["event"].to_s.strip
-      raise TableNameNil, "Table name is nil" if event_name.empty?
+      event_name = event['event'].to_s.strip
+      raise TableNameNil, 'Table name is nil' if event_name.empty?
 
       @connection = connection
       @schema = opts[:schema].empty? ? '' : "#{opts[:schema]}."
 
       @table_name = underscore(event_name)
 
-      @columns = event.keys.map{|x| underscore(x)}
+      @columns = event.keys.map { |x| underscore(x) }
       @should_retry = false
       @enable_event_as_json_keyword = opts[:enable_event_as_json_keyword]
       @event_as_json_keyword = opts[:event_as_json_keyword]
@@ -47,35 +49,34 @@ module Charrington
     end
 
     def call
-      self.logger.info "Attempting insert into table name: #{table_name}"
+      logger.info "Attempting insert into table name: #{table_name}"
       insert_sql = insert_event_statement
       insert_stmt = connection.prepareStatement(insert_sql)
-      self.logger.info "Insert statement passed into prepareStatement is: #{insert_stmt}"
+      logger.info "Insert statement passed into prepareStatement is: #{insert_stmt}"
       insert_stmt = add_statement_event_params(insert_stmt, event)
-      self.logger.info "Insert statement to be run is: #{insert_stmt.toString}"
+      logger.info "Insert statement to be run is: #{insert_stmt.toString}"
       insert_stmt.execute
 
-      self.logger.info "Attempting insert into tracks table"
+      logger.info 'Attempting insert into tracks table'
       do_tracks_insert
 
       should_retry
     rescue Java::JavaSql::SQLException => e
-      case e.getSQLState()
-      when "42P01"
-        self.logger.info "Received Java::JavaSql::SQLException with error sql state of 42P01, moving to create table"
+      case e.getSQLState
+      when '42P01'
+        logger.info 'Received Java::JavaSql::SQLException with error sql state of 42P01, moving to create table'
         should_retry = create_table
-      when "42703"
-        self.logger.info "Received Java::JavaSql::SQLException with error sql state of 42703, moving to alter table"
+      when '42703'
+        logger.info 'Received Java::JavaSql::SQLException with error sql state of 42703, moving to alter table'
         should_retry = alter_table
       else
         raise InsertFailed, "SQLException (Charrington:Insert) #{e.message} #{insert_sql}"
       end
       should_retry
-
-    rescue => e
+    rescue StandardError => e
       raise InsertFailed, "SQLException (Charrington:Insert) #{e.message} #{insert_sql}"
     ensure
-      insert_stmt.close unless insert_stmt.nil?
+      insert_stmt&.close
       cleanup
     end
 
@@ -83,48 +84,48 @@ module Charrington
       tracks_sql = insert_tracks_statement
       tracks_stmt = connection.prepareStatement(tracks_sql)
       tracks_stmt = add_statement_event_params(tracks_stmt, tracks)
-      self.logger.info "Insert tracks statment to be run: #{tracks_stmt.toString}"
+      logger.info "Insert tracks statment to be run: #{tracks_stmt.toString}"
       tracks_stmt.execute
     rescue Java::JavaSql::SQLException => e
-      self.logger.error("SQLException (Charrington:Insert) Insert tracks entry failed. #{e.message} #{tracks_sql}")
+      logger.error("SQLException (Charrington:Insert) Insert tracks entry failed. #{e.message} #{tracks_sql}")
     ensure
-      tracks_stmt.close unless tracks_stmt.nil?
+      tracks_stmt&.close
     end
 
     private
 
     def tracks_columns
-      is_redshift_transform? ? @@redshift_tracks_columns : @@postgres_tracks_columns
+      redshift_transform? ? REDSHIFT_TRACKS_COLUMNS : POSTGRES_TRACKS_COLUMNS
     end
 
-    def is_redshift?
+    def redshift?
       driver == 'redshift'
     end
 
-    def is_postgres?
+    def postgres?
       driver == 'postgres'
     end
 
-    def is_postgres_transform?
+    def postgres_transform?
       transformer == 'postgres'
     end
 
-    def is_redshift_transform?
+    def redshift_transform?
       transformer == 'redshift'
     end
 
     def create_table
-      if is_postgres?
+      if postgres?
         Charrington::CreatePostgresTable.call(connection, event, schema, table_name, columns, opts)
-      elsif is_redshift?
+      elsif redshift?
         Charrington::CreateRedshiftTable.call(connection, event, schema, table_name, columns, opts)
       end
     end
 
     def alter_table
-      if is_postgres?
+      if postgres?
         Charrington::AlterPostgresTable.call(connection, event, schema, table_name, columns)
-      elsif is_redshift?
+      elsif redshift?
         Charrington::AlterRedshiftTable.call(connection, event, schema, table_name, columns)
       end
     end
@@ -137,7 +138,7 @@ module Charrington
     ### Set Variables
     def create_tracks(event)
       tracks = event.clone
-      tracks.keys.each do |key|
+      tracks.each_key do |key|
         tracks.delete(key) unless tracks_columns.include?(key)
       end
       tracks
@@ -151,7 +152,6 @@ module Charrington
       @tracks_columns_text ||= arr_to_csv(tracks.keys)
     end
 
-
     def value_placeholders(columns)
       ('?' * columns.length).split('')
     end
@@ -168,19 +168,19 @@ module Charrington
       "INSERT INTO #{schema}tracks #{tracks_columns_text} VALUES #{placeholder_text(tracks.keys)}"
     end
 
-    def add_statement_event_params(stmt, map)
+    def add_statement_event_params(stmt, map) # rubocop:disable Metrics/CyclomaticComplexity
       values = []
-      map.keys.each_with_index do |key, idx|
+      map.keys.each_with_index do |key, idx| # rubocop:disable Metrics/BlockLength
         pos = idx + 1
         value = map[key]
         values << value
 
-        if @@timestamp_columns.include?(key)
+        if TIMESTAMP_COLUMNS.include?(key)
           begin
             time = parse_date(value)
             stmt.setTimestamp(pos, time)
             next
-          rescue java.text.ParseException => e
+          rescue java.text.ParseException
             time = parse_date(value, "yyyy-MM-dd'T'HH:mm:ss'Z'")
             stmt.setTimestamp(pos, time)
             next
@@ -203,7 +203,7 @@ module Charrington
         when Float
           stmt.setFloat(pos, value)
         when String
-          stmt.setString(pos, value[0,254]) # truncate at 254 string characters
+          stmt.setString(pos, value[0, 254]) # truncate at 254 string characters
         when Array, Hash
           stmt.setString(pos, value.to_json)
         when true, false
@@ -223,35 +223,35 @@ module Charrington
     end
 
     def arr_to_csv(arr)
-      '(' + arr.join(', ') + ')'
+      "(#{arr.join(', ')})"
     end
 
     def clearable(obj)
-      obj.is_a? Hash or obj.is_a? Array
+      obj.is_a?(Hash) || obj.is_a?(Array)
     end
 
     def underscore(str)
-      str.
-      gsub(/::/, '/').
-      gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
-      gsub(/([a-z\d])([A-Z])/,'\1_\2').
-      downcase.
-      gsub(/[^a-z0-9]+/, "_").
-      gsub(/\A_+/, "").
-      gsub(/_+\z/, "")[0,64]
+      str
+        .gsub(/::/, '/')
+        .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+        .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+        .downcase
+        .gsub(/[^a-z0-9]+/, '_')
+        .gsub(/\A_+/, '')
+        .gsub(/_+\z/, '')[0, 64]
     end
 
     ### SQL
 
     def execute(connection, sql)
-      statement = connection.prepareStatement( sql.gsub(/\s+/, " ").strip )
-      statement.execute()
+      statement = connection.prepareStatement(sql.gsub(/\s+/, ' ').strip)
+      statement.execute
     rescue Java::OrgPostgresqlUtil::PSQLException => e
-      self.logger.error "PSQLException: #{e.message}, with SQL: #{sql}"
+      logger.error "PSQLException: #{e.message}, with SQL: #{sql}"
     rescue Java::JavaSql::SQLException => e
-      self.logger.error "Redshift SQLException: #{e.message}, with SQL: #{sql}"
+      logger.error "Redshift SQLException: #{e.message}, with SQL: #{sql}"
     ensure
-      statement.close unless statement.nil?
+      statement&.close
     end
   end
 end

data/lib/logstash/outputs/charrington/process.rb

@@ -1,10 +1,11 @@
-require File.join(File.dirname(__FILE__), "service")
+# frozen_string_literal: true
+
+require File.join(File.dirname(__FILE__), 'service')
 
 module Charrington
+  # This service starts the process of attempting to insert a row.
+  # It handles retries where applicable.
   class Process
-    # This service starts the process of attempting to insert a row.
-    # It handles retries where applicable.
-
     include Service
     include LogStash::Util::Loggable
     attr_reader :event, :connection, :opts, :max_retries, :schema, :retry_max_interval, :driver, :transformer
@@ -14,8 +15,9 @@ module Charrington
     ProcessFailed = Class.new(Error)
     EventNil = Class.new(Error)
 
-    def initialize(connection, event, opts={})
-      raise EventNil, "Event is nil" if event.nil?
+    def initialize(connection, event, opts = {})
+      raise EventNil, 'Event is nil' if event.nil?
+
       @connection = connection
       @event = event.to_hash
       @opts = opts
@@ -31,18 +33,18 @@ module Charrington
     end
 
     def call
-      while should_retry do
-        transformed = case transformer
-                        when "redshift"
-                          self.logger.info "Found transformer of #{transformer} for driver of #{driver} with event of: #{event}"
-                          Charrington::TransformRedshift.call(event)
-                        else
-                          self.logger.info "Found transformer of #{transformer} for driver of #{driver} with event of: #{event}"
-                          Charrington::TransformPostgres.call(event)
-                        end
-        self.logger.info "Transformed event into: #{transformed}"
+      while should_retry
+        logger.info "Found transformer of #{transformer} for driver of #{driver} with event of: #{event}"
+        transformed =
+          case transformer
+          when 'redshift'
+            Charrington::TransformRedshift.call(event)
+          else
+            Charrington::TransformPostgres.call(event)
+          end
+        logger.info "Transformed event into: #{transformed}"
         should_retry = Charrington::Insert.call(connection, transformed, opts)
-        break if !should_retry
+        break unless should_retry
 
         @attempts += 1
         break if @attempts > max_retries
@@ -51,10 +53,10 @@ module Charrington
         # Double the interval for the next time through to achieve exponential backoff
         sleep_interval
       end
-    rescue => e
+    rescue StandardError => e
       raise ProcessFailed, e.message
     ensure
-      connection.close unless connection.nil?
+      connection&.close
       @event.clear if clearable(@event)
     end
 
@@ -63,11 +65,13 @@ module Charrington
     def sleep_interval
       sleep(retry_interval)
       doubled = retry_interval * 2
+      # rubocop:disable Lint/UselessAssignment
       retry_interval = doubled > retry_max_interval ? retry_max_interval : doubled
+      # rubocop:enable Lint/UselessAssignment
     end
 
     def clearable(obj)
-      obj.is_a? Hash or obj.is_a? Array
+      obj.is_a?(Hash) || obj.is_a?(Array)
     end
   end
 end

data/lib/logstash/outputs/charrington/service.rb

@@ -1,9 +1,13 @@
 # frozen_string_literal: true
+
+# Service mix in that provides a `call` helper method
 module Service
   def self.included(base)
     base.extend ClassMethods
   end
 
+  # Adds a `call` method that creates a new instance
+  # and delegates to the child's call method
   module ClassMethods
     def call(*args)
       new(*args).call

data/lib/logstash/outputs/charrington/transform_postgres.rb

@@ -1,6 +1,9 @@
-require File.join(File.dirname(__FILE__), "service")
+# frozen_string_literal: true
+
+require File.join(File.dirname(__FILE__), 'service')
 
 module Charrington
+  # Inserts and modifies a Postgres database
   class TransformPostgres
     include Service
     attr_accessor :event
@@ -11,11 +14,12 @@ module Charrington
     TableNameNil = Class.new(Error)
     ColumnBlacklist = Class.new(Error)
 
-    KEY_FILTER_BLACKLIST = ['host','path','jwt','sequence']
-    KEY_RAISE_BLACKLIST = ['id', 'inserted_at']
+    KEY_FILTER_BLACKLIST = %w[host path jwt sequence].freeze
+    KEY_RAISE_BLACKLIST = %w[id inserted_at].freeze
 
     def initialize(event)
-      raise EventNil, "Event is nil" if event.nil?
+      raise EventNil, 'Event is nil' if event.nil?
+
       event = event.to_hash
       @event = drop_keys(event)
       @top_level_keys = @event.keys
@@ -34,11 +38,11 @@ module Charrington
     def check_blacklist
       arr = []
       KEY_RAISE_BLACKLIST.each { |k| arr << k if event.keys.include?(k) }
-      raise ColumnBlacklist, "Event contains these blacklisted keys: #{arr.join(",")}" unless arr.empty?
+      raise ColumnBlacklist, "Event contains these blacklisted keys: #{arr.join(',')}" unless arr.empty?
     end
 
     def drop_keys(event)
-      event.delete_if {|k, _v| k.start_with?("@") || KEY_FILTER_BLACKLIST.include?(k) }
+      event.delete_if { |k, _v| k.start_with?('@') || KEY_FILTER_BLACKLIST.include?(k) }
     end
 
     def flatten_hash(hash)

data/lib/logstash/outputs/charrington/transform_redshift.rb

@@ -1,7 +1,10 @@
-require File.join(File.dirname(__FILE__), "service")
+# frozen_string_literal: true
+
+require File.join(File.dirname(__FILE__), 'service')
 require 'securerandom'
 
 module Charrington
+  # Inserts and modifies a Postgres/Redshift database
   class TransformRedshift
     include Service
     attr_accessor :event
@@ -12,11 +15,12 @@ module Charrington
     TableNameNil = Class.new(Error)
     ColumnBlacklist = Class.new(Error)
 
-    KEY_FILTER_BLACKLIST = ['host','path','jwt','sequence']
-    KEY_RAISE_BLACKLIST = ['inserted_at']
+    KEY_FILTER_BLACKLIST = %w[host path jwt sequence].freeze
+    KEY_RAISE_BLACKLIST = ['inserted_at'].freeze
 
     def initialize(event)
-      raise EventNil, "Event is nil" if event.nil?
+      raise EventNil, 'Event is nil' if event.nil?
+
       event = event.to_hash
       @event = drop_keys(event)
       @top_level_keys = @event.keys
@@ -27,17 +31,17 @@ module Charrington
       handle_event_key(event)
       add_id_to_event(event)
 
-      handle_key_transform(event, "anonymous_id", "anonymous_user")
-      handle_key_transform(event, "sent_at", "published_at")
-      handle_key_transform(event, "original_timestamp", "sent_at")
-      handle_key_transform(event, "received_at", "sent_at")
-      handle_key_transform(event, "timestamp", "sent_at")
+      handle_key_transform(event, 'anonymous_id', 'anonymous_user')
+      handle_key_transform(event, 'sent_at', 'published_at')
+      handle_key_transform(event, 'original_timestamp', 'sent_at')
+      handle_key_transform(event, 'received_at', 'sent_at')
+      handle_key_transform(event, 'timestamp', 'sent_at')
 
       handle_meta_section(event)
 
       transform_session_stuff(event)
 
-      event.delete_if {|k, _v| ['session', 'meta', 'published_at', 'anonymous_user'].include?(k) }
+      event.delete_if { |k, _v| %w[session meta published_at anonymous_user].include?(k) }
 
       event
     end
@@ -45,65 +49,65 @@ module Charrington
     private
 
     def handle_key_transform(hash, key_that_should_be_there, key_to_take_value_from)
-      unless hash.has_key?(key_that_should_be_there)
-        hash[key_that_should_be_there] = hash[key_to_take_value_from] || ""
-      else
+      if hash.key?(key_that_should_be_there)
         hash
+      else
+        hash[key_that_should_be_there] = hash[key_to_take_value_from] || ''
       end
     end
 
     def add_id_to_event(hash)
-      hash["id"] = SecureRandom.hex(10)
+      hash['id'] = SecureRandom.hex(10)
     end
 
     def handle_event_key(hash)
-      event_name = hash["event"] || ""
+      event_name = hash['event'] || ''
 
-      hash["event_text"] = event_name
+      hash['event_text'] = event_name
 
-      hash["event"] = underscore_event_name(event_name)
+      hash['event'] = underscore_event_name(event_name)
     end
 
     def underscore_event_name(event_name)
-      event_name.
-      gsub(/::/, '/').
-      gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
-      gsub(/([a-z\d])([A-Z])/,'\1_\2').
-      downcase.
-      gsub(/[^a-z0-9]+/, "_").
-      gsub(/\A_+/, "").
-      gsub(/_+\z/, "")
+      event_name
+        .gsub(/::/, '/')
+        .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+        .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+        .downcase
+        .gsub(/[^a-z0-9]+/, '_')
+        .gsub(/\A_+/, '')
+        .gsub(/_+\z/, '')
     end
 
     def transform_session_stuff(hash)
-      session_stuff = hash["session"] || {}
+      session_stuff = hash['session'] || {}
 
-      session_stuff.each {|k, v|
-        next unless k.kind_of?(String)
+      session_stuff.each do |k, v|
+        next unless k.is_a?(String)
 
         hash["context_#{k}"] = v
-      }
+      end
     end
 
     def handle_meta_section(hash)
-      meta_section = hash["meta"] || {}
+      meta_section = hash['meta'] || {}
 
-      meta_section.each {|k, v|
-        next unless k.kind_of?(String)
-        next if [Array, Hash].map { |type| v.kind_of?(type) }.any?
+      meta_section.each do |k, v|
+        next unless k.is_a?(String)
+        next if [Array, Hash].map { |type| v.is_a?(type) }.any?
 
         hash[k] = v
-      }
+      end
     end
 
     def check_blacklist
       arr = []
       KEY_RAISE_BLACKLIST.each { |k| arr << k if event.keys.include?(k) }
-      raise ColumnBlacklist, "Event contains these blacklisted keys: #{arr.join(",")}" unless arr.empty?
+      raise ColumnBlacklist, "Event contains these blacklisted keys: #{arr.join(',')}" unless arr.empty?
     end
 
     def drop_keys(event)
-      event.delete_if {|k, _v| k.start_with?("@") || KEY_FILTER_BLACKLIST.include?(k) }
+      event.delete_if { |k, _v| k.start_with?('@') || KEY_FILTER_BLACKLIST.include?(k) }
     end
 
     def flatten_hash(hash)