logstash-output-charrington 0.3.21 → 0.3.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b495c8c2b48eb4a9f0a61ebd67cb215003b4971d58d63241f4ca4678da1ccb26
-  data.tar.gz: 424fd9a3e3eac5e0c7f893cd3613e9f38a38d15ef63c19a843605c8651aef06a
+  metadata.gz: bbc89a196e03294552a4b25fef760461874c3eb8fad194845e127cc517d1a53f
+  data.tar.gz: a330a0c2c09a9d4c8746244e3320e76ce121bfee85e97304ddebf87deccf9b31
 SHA512:
-  metadata.gz: 5eb47165c378727eee8e7371defca71ffbb840a2e555b91a0e802bb75722888ec770938497a2b956ed8a7ecd874d0bd632ad8e76673afb93fc6a85fa1d784370
-  data.tar.gz: ed5876a86067c11dcbc1776ab8b11388783c58d758a815ba61bcaa991de0c606aff1c9d47c7fd4fc164fbb50f63878211385431121ce8b34e1c56cc4f56542b3
+  metadata.gz: e206bed7a803e0f999931fdddc0f95769cab0ed47ac9813ae27934decf695de7f1a5d38f95f01168c3d8ae6eede046c5b247a31a1efb8a6919ec34cc32da6905
+  data.tar.gz: d94bcd0d15c6fe32bcaa937f3a78231f592e2dffd90217e8a8635faaef3747c121989e8552ae66d5a019be9cf3e5ba066c375922608f25a332dcaf596e5efa20

data/Gemfile

@@ -1,11 +1,13 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 gemspec
 
-logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
-use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+logstash_path = ENV['LOGSTASH_PATH'] || '../../logstash'
+use_logstash_source = ENV['LOGSTASH_SOURCE'] && ENV['LOGSTASH_SOURCE'].to_s == '1'
 
 if Dir.exist?(logstash_path) && use_logstash_source
-  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
-  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+  gem 'logstash-core', path: "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', path: "#{logstash_path}/logstash-core-plugin-api"
 end

data/lib/logstash/outputs/charrington/alter_postgres_table.rb

@@ -1,10 +1,11 @@
-require File.join(File.dirname(__FILE__), "service")
+# frozen_string_literal: true
+
+require File.join(File.dirname(__FILE__), 'service')
 
 module Charrington
+  # This service will add columns to an existing table dynamically based on finding new keys in the JSON structure.
+  # This is potentially called from Insert when an insert fails.
   class AlterPostgresTable
-    # This service will add columns to an existing table dynamically based on finding new keys in the JSON structure.
-    # This is potentially called from Insert when an insert fails.
-
     include Service
     include LogStash::Util::Loggable
     attr_reader :connection, :event, :table_name, :columns, :schema
@@ -26,7 +27,7 @@ module Charrington
       set_column_types
       alter_table
       true
-    rescue => e
+    rescue StandardError => e
       raise AlterFailed, e.message
     ensure
       @column_types.clear if @column_types.is_a? Array
@@ -41,69 +42,68 @@ module Charrington
     def columns_fragment
       column_types.map do |column|
         "ADD COLUMN IF NOT EXISTS #{column}"
-      end.join(",")
+      end.join(',')
     end
 
     def set_column_types
-      (columns - current_table_columns).each_with_index do |key, idx|
-
-        case event[key]
-        when Time, LogStash::Timestamp
-          column_types << "#{key} TIMESTAMP"
-        when Date
-          column_types << "#{key} DATE"
-        when Integer
-          column_types << "#{key} BIGINT"
-        when BigDecimal
-          column_types << "#{key} DECIMAL"
-        when Float
-          column_types << "#{key} DOUBLE PRECISION"
-        when true, false
-          column_types << "#{key} BOOLEAN"
-        else
-          column_types << "#{key} VARCHAR"
-        end
+      (columns - current_table_columns).each_with_index do |key, _idx|
+        column_types << case event[key]
+                        when Time, LogStash::Timestamp
+                          "#{key} TIMESTAMP"
+                        when Date
+                          "#{key} DATE"
+                        when Integer
+                          "#{key} BIGINT"
+                        when BigDecimal
+                          "#{key} DECIMAL"
+                        when Float
+                          "#{key} DOUBLE PRECISION"
+                        when true, false
+                          "#{key} BOOLEAN"
+                        else
+                          "#{key} VARCHAR"
+                        end
       end
     end
 
     def current_table_columns
       sql = "SELECT * FROM #{schema}#{table_name} LIMIT 1;"
-      stmt, rs = executeQuery(prep_sql(sql))
-      meta_data = rs.getMetaData()
-      column_count = meta_data.getColumnCount()
-      (1..column_count).map {|i| meta_data.getColumnName(i) }
+      stmt, rs = execute_query(prep_sql(sql))
+      meta_data = rs.getMetaData
+      column_count = meta_data.getColumnCount
+      (1..column_count).map { |i| meta_data.getColumnName(i) }
     ensure
-      stmt.close unless stmt.nil?
+      stmt&.close
     end
 
     def execute(sql)
       stmt = connection.prepareStatement(prep_sql(sql))
-      stmt.execute()
+      stmt.execute
     rescue Java::OrgPostgresqlUtil::PSQLException => e
-      self.logger.error "PSQLException: #{e.message}"
+      logger.error "PSQLException: #{e.message}"
     ensure
-      stmt.close unless stmt.nil?
+      stmt&.close
     end
 
-    def executeQuery(sql)
-      stmt = connection.createStatement()
+    def execute_query(sql)
+      stmt = connection.createStatement
       # only close the statement if something goes wrong
       # otherwise, the caller is responsible for closing the
       # statement when they are doen with the result set
-      return stmt, stmt.executeQuery(prep_sql(sql))
+      [stmt, stmt.execute_query(prep_sql(sql))]
     rescue Java::OrgPostgresqlUtil::PSQLException => e
       puts "PSQLException: #{e.message}"
-      self.logger.info "PSQLException: #{e.message}"
-      stmt.close unless stmt.nil?
+      logger.info "PSQLException: #{e.message}"
+      stmt&.close
       # @logger.error("#{e.message}")
-    rescue => e
+    rescue StandardError => e
       puts "Unknown exception: #{e.message}"
-      self.logger.info "Unknown exception: #{e.message}"
-      stmt.close unless stmt.nil?
+      logger.info "Unknown exception: #{e.message}"
+      stmt&.close
     end
 
     def prep_sql(sql)
-      sql.gsub(/\s+/, " ").strip
+      sql.gsub(/\s+/, ' ').strip
     end
   end
 end

data/lib/logstash/outputs/charrington/alter_redshift_table.rb

@@ -1,10 +1,11 @@
-require File.join(File.dirname(__FILE__), "service")
+# frozen_string_literal: true
+
+require File.join(File.dirname(__FILE__), 'service')
 
 module Charrington
+  # This service will add columns to an existing table dynamically based on finding new keys in the JSON structure.
+  # This is potentially called from Insert when an insert fails.
   class AlterRedshiftTable
-    # This service will add columns to an existing table dynamically based on finding new keys in the JSON structure.
-    # This is potentially called from Insert when an insert fails.
-
     include Service
     include LogStash::Util::Loggable
     attr_reader :connection, :event, :table_name, :columns, :schema
@@ -26,7 +27,7 @@ module Charrington
       set_column_types
       alter_table
       true
-    rescue => e
+    rescue StandardError => e
       raise AlterFailed, e.message
     ensure
       @column_types.clear if @column_types.is_a? Array
@@ -35,89 +36,89 @@ module Charrington
     private
 
     def alter_table
-      execute_list(get_list_of_alter_table_stmts)
+      execute_list(list_of_alter_table_stmts)
     end
 
-    def get_list_of_alter_table_stmts
+    def list_of_alter_table_stmts
       column_types.map do |column|
         "ALTER TABLE #{schema}#{table_name} ADD COLUMN #{column}"
       end
     end
 
     def set_column_types
-      (columns - current_table_columns).each_with_index do |key, idx|
-        self.logger.info "New column: #{key}, because of event: #{event}"
-
-        case event[key]
-        when Time, LogStash::Timestamp
-          column_types << "#{key} TIMESTAMP"
-        when Date
-          column_types << "#{key} DATE"
-        when Integer
-          column_types << "#{key} BIGINT"
-        when BigDecimal
-          column_types << "#{key} DECIMAL"
-        when Float
-          column_types << "#{key} DOUBLE PRECISION"
-        when true, false
-          column_types << "#{key} BOOLEAN"
-        else
-          column_types << "#{key} VARCHAR(512)"
-        end
+      (columns - current_table_columns).each_with_index do |key, _idx|
+        logger.info "New column: #{key}, because of event: #{event}"
+
+        column_types << case event[key]
+                        when Time, LogStash::Timestamp
+                          "#{key} TIMESTAMP"
+                        when Date
+                          "#{key} DATE"
+                        when Integer
+                          "#{key} BIGINT"
+                        when BigDecimal
+                          "#{key} DECIMAL"
+                        when Float
+                          "#{key} DOUBLE PRECISION"
+                        when true, false
+                          "#{key} BOOLEAN"
+                        else
+                          "#{key} VARCHAR(512)"
+                        end
       end
     end
 
     def current_table_columns
       sql = "SELECT * FROM #{schema}#{table_name} LIMIT 1;"
-      stmt, rs = executeQuery(prep_sql(sql))
-      meta_data = rs.getMetaData()
-      column_count = meta_data.getColumnCount()
-      (1..column_count).map {|i| meta_data.getColumnName(i) }
+      stmt, rs = execute_query(prep_sql(sql))
+      meta_data = rs.getMetaData
+      column_count = meta_data.getColumnCount
+      (1..column_count).map { |i| meta_data.getColumnName(i) }
     ensure
-      self.logger.info "Within ensure block of current_table_columns in alter_redshift_table.rb and value of stmt.nil?: #{stmt.nil?}"
-      stmt.close unless stmt.nil?
+      logger.info "Within ensure block of current_table_columns in alter_redshift_table.rb and value of stmt.nil?: #{stmt.nil?}"
+      stmt&.close
     end
 
     def execute_list(list_of_sql_stmts)
-      self.logger.info "Received list of sql statments to execute: #{list_of_sql_stmts}"
+      logger.info "Received list of sql statments to execute: #{list_of_sql_stmts}"
 
       list_of_sql_stmts.each_with_index do |sql, idx|
-        self.logger.info "Executing ALTER TABLE statement with index #{idx} and sql of: #{sql}"
+        logger.info "Executing ALTER TABLE statement with index #{idx} and sql of: #{sql}"
         execute(sql)
       end
     end
 
     def execute(sql)
       stmt = connection.prepareStatement(prep_sql(sql))
-      stmt.execute()
+      stmt.execute
     rescue Java::JavaSql::SQLException => e
-      self.logger.error "Alter Redshift SQLException: #{e.message}, with SQL: #{sql}"
-    rescue => e
-      self.logger.error "Alter Redshift Unknown exception: #{e.message}, with SQL: #{sql}"
+      logger.error "Alter Redshift SQLException: #{e.message}, with SQL: #{sql}"
+    rescue StandardError => e
+      logger.error "Alter Redshift Unknown exception: #{e.message}, with SQL: #{sql}"
     ensure
-      self.logger.error "Within ensure block of execute in alter_redshift_table.rb and value of stmt.nil?: #{stmt.nil?}"
-      stmt.close unless stmt.nil?
+      logger.error "Within ensure block of execute in alter_redshift_table.rb and value of stmt.nil?: #{stmt.nil?}"
+      stmt&.close
     end
 
-    def executeQuery(sql)
-      stmt = connection.createStatement()
+    def execute_query(sql)
+      stmt = connection.createStatement
       # only close the statement if something goes wrong
       # otherwise, the caller is responsible for closing the
       # statement when they are doen with the result set
-      return stmt, stmt.executeQuery(prep_sql(sql))
+      [stmt, stmt.execute_query(prep_sql(sql))]
     rescue Java::JavaSql::SQLException => e
       puts "execute query SQLException: #{e.message}"
-      self.logger.info "execute query SQLException: #{e.message}, with SQL: #{sql}"
-      stmt.close unless stmt.nil?
+      logger.info "execute query SQLException: #{e.message}, with SQL: #{sql}"
+      stmt&.close
       # @logger.error("#{e.message}")
-    rescue => e
+    rescue StandardError => e
       puts "execute query Unknown exception: #{e.message}"
-      self.logger.info "execute query Unknown exception: #{e.message}, with SQL: #{sql}"
-      stmt.close unless stmt.nil?
+      logger.info "execute query Unknown exception: #{e.message}, with SQL: #{sql}"
+      stmt&.close
     end
 
     def prep_sql(sql)
-      sql.gsub(/\s+/, " ").strip
+      sql.gsub(/\s+/, ' ').strip
     end
   end
 end

data/lib/logstash/outputs/charrington/create_postgres_table.rb

@@ -1,11 +1,12 @@
-require File.join(File.dirname(__FILE__), "service")
+# frozen_string_literal: true
+
+require File.join(File.dirname(__FILE__), 'service')
 
 module Charrington
+  # This service will create a table dynamically based on the JSON structure.
+  # This is potentially called from Insert when an insert fails.
   class CreatePostgresTable
-    # This service will create a table dynamically based on the JSON structure.
-    # This is potentially called from Insert when an insert fails.
-
-    @@timestamp_columns = %w(published_at sent_at original_timestamp received_at timestamp)
+    TIMESTAMP_COLUMNS = %w[published_at sent_at original_timestamp received_at timestamp].freeze
 
     include Service
     include LogStash::Util::Loggable
@@ -15,7 +16,7 @@ module Charrington
     Error = Class.new(StandardError)
     CreateFailed = Class.new(Error)
 
-    def initialize(connection, event, schema, table_name, columns, opts = {})
+    def initialize(connection, event, schema, table_name, columns, opts = {}) # rubocop:disable Metrics/ParameterLists
       @connection = connection
       @event = event.to_hash
       @table_name = table_name
@@ -29,7 +30,7 @@ module Charrington
       set_column_types
       create_table
       true
-    rescue => e
+    rescue StandardError => e
       raise CreateFailed, e.message
     ensure
       @column_types.clear if @column_types.is_a? Array
@@ -39,39 +40,39 @@ module Charrington
 
     def set_column_types
       columns.each do |column|
-        if @@timestamp_columns.include?(column)
+        if TIMESTAMP_COLUMNS.include?(column)
           column_types << "#{column} TIMESTAMP"
           next
         end
 
-        case event[column]
-        when Time, LogStash::Timestamp
-          column_types << "#{column} TIMESTAMP"
-        when Date
-          column_types << "#{column} DATE"
-        when Integer
-          column_types << "#{column} BIGINT"
-        when BigDecimal
-          column_types << "#{column} DECIMAL"
-        when Float
-          column_types << "#{column} DOUBLE PRECISION"
-        when true, false
-          column_types << "#{column} BOOLEAN"
-        else
-          column_types << "#{column} VARCHAR(512)"
-        end
+        column_types << case event[column]
+                        when Time, LogStash::Timestamp
+                          "#{column} TIMESTAMP"
+                        when Date
+                          "#{column} DATE"
+                        when Integer
+                          "#{column} BIGINT"
+                        when BigDecimal
+                          "#{column} DECIMAL"
+                        when Float
+                          "#{column} DOUBLE PRECISION"
+                        when true, false
+                          "#{column} BOOLEAN"
+                        else
+                          "#{column} VARCHAR(512)"
+                        end
       end
     end
 
     def initial_columns
-      if transformer == "postgres"
+      if transformer == 'postgres'
         [
-          "id SERIAL PRIMARY KEY",
-          "inserted_at TIMESTAMP DEFAULT NOW()"
+          'id SERIAL PRIMARY KEY',
+          'inserted_at TIMESTAMP DEFAULT NOW()'
         ]
       else
         [
-          "uuid_ts TIMESTAMP DEFAULT NOW()"
+          'uuid_ts TIMESTAMP DEFAULT NOW()'
         ]
       end
     end
@@ -81,12 +82,12 @@ module Charrington
     end
 
     def execute(sql)
-      statement = connection.prepareStatement( sql.gsub(/\s+/, " ").strip )
-      statement.execute()
+      statement = connection.prepareStatement(sql.gsub(/\s+/, ' ').strip)
+      statement.execute
     rescue Java::OrgPostgresqlUtil::PSQLException => e
-      self.logger.error "PSQLException: #{e.message} sql=#{sql}"
+      logger.error "PSQLException: #{e.message} sql=#{sql}"
     ensure
-      statement.close unless statement.nil?
+      statement&.close
     end
   end
 end

data/lib/logstash/outputs/charrington/create_redshift_table.rb

@@ -1,10 +1,11 @@
-require File.join(File.dirname(__FILE__), "service")
+# frozen_string_literal: true
+
+require File.join(File.dirname(__FILE__), 'service')
 
 module Charrington
+  # This service will create a table dynamically based on the JSON structure.
+  # This is potentially called from Insert when an insert fails.
   class CreateRedshiftTable
-    # This service will create a table dynamically based on the JSON structure.
-    # This is potentially called from Insert when an insert fails.
-
     include Service
     include LogStash::Util::Loggable
     attr_reader :connection, :event, :table_name, :columns, :schema, :opts
@@ -13,7 +14,7 @@ module Charrington
     Error = Class.new(StandardError)
     CreateFailed = Class.new(Error)
 
-    def initialize(connection, event, schema, table_name, columns, opts = {})
+    def initialize(connection, event, schema, table_name, columns, _opts = {}) # rubocop:disable Metrics/ParameterLists
       @connection = connection
       @event = event.to_hash
       @schema = schema
@@ -24,10 +25,10 @@ module Charrington
 
     def call
       set_column_types
-      self.logger.info "Finished running set_column_types and now have column_types for create table of: #{column_types}"
+      logger.info "Finished running set_column_types and now have column_types for create table of: #{column_types}"
       create_table
       true
-    rescue => e
+    rescue StandardError => e
       raise CreateFailed, e.message
     ensure
       @column_types.clear if @column_types.is_a? Array
@@ -36,31 +37,32 @@ module Charrington
     private
 
     # https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_TABLE_NEW.html
-    def set_column_types
+    def set_column_types # rubocop:disable Metrics/CyclomaticComplexity
       columns.each do |column|
-        if column == "id"
+        case column
+        when 'id'
           column_types << "#{column} VARCHAR(512) NOT NULL distkey CONSTRAINT #{table_name}_pkey primary key"
           next
-        elsif column == "sent_at"
+        when 'sent_at'
           column_types << "#{column} TIMESTAMP"
           next
         end
-        case event[column]
-        when Time, LogStash::Timestamp
-          column_types << "#{column} TIMESTAMP"
-        when Date
-          column_types << "#{column} DATE"
-        when Integer
-          column_types << "#{column} BIGINT"
-        when BigDecimal
-          column_types << "#{column} DECIMAL"
-        when Float
-          column_types << "#{column} DOUBLE PRECISION"
-        when true, false
-          column_types << "#{column} BOOLEAN"
-        else
-          column_types << "#{column} VARCHAR(512)"
-        end
+        column_types << case event[column]
+                        when Time, LogStash::Timestamp
+                          "#{column} TIMESTAMP"
+                        when Date
+                          "#{column} DATE"
+                        when Integer
+                          "#{column} BIGINT"
+                        when BigDecimal
+                          "#{column} DECIMAL"
+                        when Float
+                          "#{column} DOUBLE PRECISION"
+                        when true, false
+                          "#{column} BOOLEAN"
+                        else
+                          "#{column} VARCHAR(512)"
+                        end
       end
     end
 
@@ -79,15 +81,15 @@ module Charrington
     end
 
     def execute(sql)
-      self.logger.info "Running sql of: #{sql}"
+      logger.info "Running sql of: #{sql}"
 
-      statement = connection.prepareStatement( sql.gsub(/\s+/, " ").strip )
-      statement.execute()
+      statement = connection.prepareStatement(sql.gsub(/\s+/, ' ').strip)
+      statement.execute
     rescue Java::JavaSql::SQLException => e
-      self.logger.info "Redshift SQLException: #{e.message}, with SQL: #{sql}"
+      logger.info "Redshift SQLException: #{e.message}, with SQL: #{sql}"
     ensure
-      self.logger.info "Within ensure block of create_redshift_table.rb and value of statement.nil?: #{statement.nil?}"
-      statement.close unless statement.nil?
+      logger.info "Within ensure block of create_redshift_table.rb and value of statement.nil?: #{statement.nil?}"
+      statement&.close
     end
   end
 end