logstash-output-charrington 0.3.22 → 0.3.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 11c3f9faf285c7c896341946f3fe6c11a7bab32aa4b9cd78c65d901db92c0402
-  data.tar.gz: 6d339e684929d5ff60d47719076cbad0e6035f1f7a6d721dfa0bd9577c973973
+  metadata.gz: 9e1dc7c6c92672c4d42224e9a9e617e3a2cd3599e2ec65d03822b1b095c23714
+  data.tar.gz: '06886089b25d6ff468a6741f82f70be806f5d11a56d63f0709c1796b32ddbee4'
 SHA512:
-  metadata.gz: d094aa65b3e2432b7c427305e641969c1b6003898f1192a42362c30fc4585f5e7c43a5eebd2a95645748384d195032ce027c66381df42f800c14efabd20065fc
-  data.tar.gz: 31f14f4b121a06a4e04689c430f633d5cda296d15f99fb07c5190d61646c4d41c3b2105af17bd9e575676c83cb264208e68180506a7e1e179f8ab436c779e2f3
+  metadata.gz: 0c28fb16d1494ac30c28d05102367475b094bee1160b5d42688da24ad9e1a0511a0533c338b661cdaa6e89b1c1bdc7a4d7cb4a049544b8d8d37524c6bf308ebc
+  data.tar.gz: f2fb7a4ea94a0e401d324452826c936dddf0b5624cedd4ec30f81ef85bb0ffc178547307ef4888fa822a3378b8c9e371f00a75c3c61b81e968295022408d1030

data/Gemfile

@@ -1,11 +1,13 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 gemspec
 
-logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
-use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+logstash_path = ENV['LOGSTASH_PATH'] || '../../logstash'
+use_logstash_source = ENV['LOGSTASH_SOURCE'] && ENV['LOGSTASH_SOURCE'].to_s == '1'
 
 if Dir.exist?(logstash_path) && use_logstash_source
-  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
-  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+  gem 'logstash-core', path: "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', path: "#{logstash_path}/logstash-core-plugin-api"
 end

data/lib/logstash/outputs/charrington/alter_postgres_table.rb

@@ -1,10 +1,11 @@
-require File.join(File.dirname(__FILE__), "service")
+# frozen_string_literal: true
+
+require File.join(File.dirname(__FILE__), 'service')
 
 module Charrington
+  # This service will add columns to an existing table dynamically based on finding new keys in the JSON structure.
+  # This is potentially called from Insert when an insert fails.
   class AlterPostgresTable
-    # This service will add columns to an existing table dynamically based on finding new keys in the JSON structure.
-    # This is potentially called from Insert when an insert fails.
-
     include Service
     include LogStash::Util::Loggable
     attr_reader :connection, :event, :table_name, :columns, :schema
@@ -26,7 +27,7 @@ module Charrington
       set_column_types
       alter_table
       true
-    rescue => e
+    rescue StandardError => e
       raise AlterFailed, e.message
     ensure
       @column_types.clear if @column_types.is_a? Array
@@ -41,69 +42,68 @@ module Charrington
     def columns_fragment
       column_types.map do |column|
         "ADD COLUMN IF NOT EXISTS #{column}"
-      end.join(",")
+      end.join(',')
     end
 
     def set_column_types
-      (columns - current_table_columns).each_with_index do |key, idx|
-
-        case event[key]
-        when Time, LogStash::Timestamp
-          column_types << "#{key} TIMESTAMP"
-        when Date
-          column_types << "#{key} DATE"
-        when Integer
-          column_types << "#{key} BIGINT"
-        when BigDecimal
-          column_types << "#{key} DECIMAL"
-        when Float
-          column_types << "#{key} DOUBLE PRECISION"
-        when true, false
-          column_types << "#{key} BOOLEAN"
-        else
-          column_types << "#{key} VARCHAR"
-        end
+      (columns - current_table_columns).each_with_index do |key, _idx|
+        column_types << case event[key]
+                        when Time, LogStash::Timestamp
+                          "#{key} TIMESTAMP"
+                        when Date
+                          "#{key} DATE"
+                        when Integer
+                          "#{key} BIGINT"
+                        when BigDecimal
+                          "#{key} DECIMAL"
+                        when Float
+                          "#{key} DOUBLE PRECISION"
+                        when true, false
+                          "#{key} BOOLEAN"
+                        else
+                          "#{key} VARCHAR"
+                        end
       end
     end
 
     def current_table_columns
       sql = "SELECT * FROM #{schema}#{table_name} LIMIT 1;"
-      stmt, rs = executeQuery(prep_sql(sql))
-      meta_data = rs.getMetaData()
-      column_count = meta_data.getColumnCount()
-      (1..column_count).map {|i| meta_data.getColumnName(i) }
+      stmt, rs = execute_query(prep_sql(sql))
+      meta_data = rs.getMetaData
+      column_count = meta_data.getColumnCount
+      (1..column_count).map { |i| meta_data.getColumnName(i) }
     ensure
-      stmt.close unless stmt.nil?
+      stmt&.close
     end
 
     def execute(sql)
       stmt = connection.prepareStatement(prep_sql(sql))
-      stmt.execute()
+      stmt.execute
     rescue Java::OrgPostgresqlUtil::PSQLException => e
-      self.logger.error "PSQLException: #{e.message}"
+      logger.error "PSQLException: #{e.message}"
     ensure
-      stmt.close unless stmt.nil?
+      stmt&.close
     end
 
-    def executeQuery(sql)
-      stmt = connection.createStatement()
+    def execute_query(sql)
+      stmt = connection.createStatement
       # only close the statement if something goes wrong
       # otherwise, the caller is responsible for closing the
       # statement when they are doen with the result set
-      return stmt, stmt.executeQuery(prep_sql(sql))
+      [stmt, stmt.execute_query(prep_sql(sql))]
     rescue Java::OrgPostgresqlUtil::PSQLException => e
       puts "PSQLException: #{e.message}"
-      self.logger.info "PSQLException: #{e.message}"
-      stmt.close unless stmt.nil?
+      logger.info "PSQLException: #{e.message}"
+      stmt&.close
       # @logger.error("#{e.message}")
-    rescue => e
+    rescue StandardError => e
       puts "Unknown exception: #{e.message}"
-      self.logger.info "Unknown exception: #{e.message}"
-      stmt.close unless stmt.nil?
+      logger.info "Unknown exception: #{e.message}"
+      stmt&.close
     end
 
     def prep_sql(sql)
-      sql.gsub(/\s+/, " ").strip
+      sql.gsub(/\s+/, ' ').strip
     end
   end
 end

data/lib/logstash/outputs/charrington/alter_redshift_table.rb

@@ -1,10 +1,11 @@
-require File.join(File.dirname(__FILE__), "service")
+# frozen_string_literal: true
+
+require File.join(File.dirname(__FILE__), 'service')
 
 module Charrington
+  # This service will add columns to an existing table dynamically based on finding new keys in the JSON structure.
+  # This is potentially called from Insert when an insert fails.
   class AlterRedshiftTable
-    # This service will add columns to an existing table dynamically based on finding new keys in the JSON structure.
-    # This is potentially called from Insert when an insert fails.
-
     include Service
     include LogStash::Util::Loggable
     attr_reader :connection, :event, :table_name, :columns, :schema
@@ -26,7 +27,7 @@ module Charrington
       set_column_types
       alter_table
       true
-    rescue => e
+    rescue StandardError => e
       raise AlterFailed, e.message
     ensure
       @column_types.clear if @column_types.is_a? Array
@@ -35,89 +36,89 @@ module Charrington
     private
 
     def alter_table
-      execute_list(get_list_of_alter_table_stmts)
+      execute_list(list_of_alter_table_stmts)
     end
 
-    def get_list_of_alter_table_stmts
+    def list_of_alter_table_stmts
       column_types.map do |column|
         "ALTER TABLE #{schema}#{table_name} ADD COLUMN #{column}"
       end
     end
 
     def set_column_types
-      (columns - current_table_columns).each_with_index do |key, idx|
-        self.logger.info "New column: #{key}, because of event: #{event}"
-
-        case event[key]
-        when Time, LogStash::Timestamp
-          column_types << "#{key} TIMESTAMP"
-        when Date
-          column_types << "#{key} DATE"
-        when Integer
-          column_types << "#{key} BIGINT"
-        when BigDecimal
-          column_types << "#{key} DECIMAL"
-        when Float
-          column_types << "#{key} DOUBLE PRECISION"
-        when true, false
-          column_types << "#{key} BOOLEAN"
-        else
-          column_types << "#{key} VARCHAR(512)"
-        end
+      (columns - current_table_columns).each_with_index do |key, _idx|
+        logger.info "New column: #{key}, because of event: #{event}"
+
+        column_types << case event[key]
+                        when Time, LogStash::Timestamp
+                          "#{key} TIMESTAMP"
+                        when Date
+                          "#{key} DATE"
+                        when Integer
+                          "#{key} BIGINT"
+                        when BigDecimal
+                          "#{key} DECIMAL"
+                        when Float
+                          "#{key} DOUBLE PRECISION"
+                        when true, false
+                          "#{key} BOOLEAN"
+                        else
+                          "#{key} VARCHAR(512)"
+                        end
       end
     end
 
     def current_table_columns
       sql = "SELECT * FROM #{schema}#{table_name} LIMIT 1;"
-      stmt, rs = executeQuery(prep_sql(sql))
-      meta_data = rs.getMetaData()
-      column_count = meta_data.getColumnCount()
-      (1..column_count).map {|i| meta_data.getColumnName(i) }
+      stmt, rs = execute_query(prep_sql(sql))
+      meta_data = rs.getMetaData
+      column_count = meta_data.getColumnCount
+      (1..column_count).map { |i| meta_data.getColumnName(i) }
     ensure
-      self.logger.info "Within ensure block of current_table_columns in alter_redshift_table.rb and value of stmt.nil?: #{stmt.nil?}"
-      stmt.close unless stmt.nil?
+      logger.info "Within ensure block of current_table_columns in alter_redshift_table.rb and value of stmt.nil?: #{stmt.nil?}"
+      stmt&.close
     end
 
     def execute_list(list_of_sql_stmts)
-      self.logger.info "Received list of sql statments to execute: #{list_of_sql_stmts}"
+      logger.info "Received list of sql statments to execute: #{list_of_sql_stmts}"
 
       list_of_sql_stmts.each_with_index do |sql, idx|
-        self.logger.info "Executing ALTER TABLE statement with index #{idx} and sql of: #{sql}"
+        logger.info "Executing ALTER TABLE statement with index #{idx} and sql of: #{sql}"
         execute(sql)
       end
     end
 
     def execute(sql)
       stmt = connection.prepareStatement(prep_sql(sql))
-      stmt.execute()
+      stmt.execute
     rescue Java::JavaSql::SQLException => e
-      self.logger.error "Alter Redshift SQLException: #{e.message}, with SQL: #{sql}"
-    rescue => e
-      self.logger.error "Alter Redshift Unknown exception: #{e.message}, with SQL: #{sql}"
+      logger.error "Alter Redshift SQLException: #{e.message}, with SQL: #{sql}"
+    rescue StandardError => e
+      logger.error "Alter Redshift Unknown exception: #{e.message}, with SQL: #{sql}"
     ensure
-      self.logger.error "Within ensure block of execute in alter_redshift_table.rb and value of stmt.nil?: #{stmt.nil?}"
-      stmt.close unless stmt.nil?
+      logger.error "Within ensure block of execute in alter_redshift_table.rb and value of stmt.nil?: #{stmt.nil?}"
+      stmt&.close
     end
 
-    def executeQuery(sql)
-      stmt = connection.createStatement()
+    def execute_query(sql)
+      stmt = connection.createStatement
       # only close the statement if something goes wrong
       # otherwise, the caller is responsible for closing the
       # statement when they are doen with the result set
-      return stmt, stmt.executeQuery(prep_sql(sql))
+      [stmt, stmt.execute_query(prep_sql(sql))]
     rescue Java::JavaSql::SQLException => e
       puts "execute query SQLException: #{e.message}"
-      self.logger.info "execute query SQLException: #{e.message}, with SQL: #{sql}"
-      stmt.close unless stmt.nil?
+      logger.info "execute query SQLException: #{e.message}, with SQL: #{sql}"
+      stmt&.close
       # @logger.error("#{e.message}")
-    rescue => e
+    rescue StandardError => e
       puts "execute query Unknown exception: #{e.message}"
-      self.logger.info "execute query Unknown exception: #{e.message}, with SQL: #{sql}"
-      stmt.close unless stmt.nil?
+      logger.info "execute query Unknown exception: #{e.message}, with SQL: #{sql}"
+      stmt&.close
     end
 
     def prep_sql(sql)
-      sql.gsub(/\s+/, " ").strip
+      sql.gsub(/\s+/, ' ').strip
     end
   end
 end

data/lib/logstash/outputs/charrington/create_postgres_table.rb

@@ -1,11 +1,12 @@
-require File.join(File.dirname(__FILE__), "service")
+# frozen_string_literal: true
+
+require File.join(File.dirname(__FILE__), 'service')
 
 module Charrington
+  # This service will create a table dynamically based on the JSON structure.
+  # This is potentially called from Insert when an insert fails.
   class CreatePostgresTable
-    # This service will create a table dynamically based on the JSON structure.
-    # This is potentially called from Insert when an insert fails.
-
-    @@timestamp_columns = %w(published_at sent_at original_timestamp received_at timestamp)
+    TIMESTAMP_COLUMNS = %w[published_at sent_at original_timestamp received_at timestamp].freeze
 
     include Service
     include LogStash::Util::Loggable
@@ -15,7 +16,7 @@ module Charrington
     Error = Class.new(StandardError)
     CreateFailed = Class.new(Error)
 
-    def initialize(connection, event, schema, table_name, columns, opts = {})
+    def initialize(connection, event, schema, table_name, columns, opts = {}) # rubocop:disable Metrics/ParameterLists
       @connection = connection
       @event = event.to_hash
       @table_name = table_name
@@ -29,7 +30,7 @@ module Charrington
       set_column_types
       create_table
       true
-    rescue => e
+    rescue StandardError => e
       raise CreateFailed, e.message
     ensure
       @column_types.clear if @column_types.is_a? Array
@@ -39,39 +40,39 @@ module Charrington
 
     def set_column_types
       columns.each do |column|
-        if @@timestamp_columns.include?(column)
+        if TIMESTAMP_COLUMNS.include?(column)
           column_types << "#{column} TIMESTAMP"
           next
         end
 
-        case event[column]
-        when Time, LogStash::Timestamp
-          column_types << "#{column} TIMESTAMP"
-        when Date
-          column_types << "#{column} DATE"
-        when Integer
-          column_types << "#{column} BIGINT"
-        when BigDecimal
-          column_types << "#{column} DECIMAL"
-        when Float
-          column_types << "#{column} DOUBLE PRECISION"
-        when true, false
-          column_types << "#{column} BOOLEAN"
-        else
-          column_types << "#{column} VARCHAR(512)"
-        end
+        column_types << case event[column]
+                        when Time, LogStash::Timestamp
+                          "#{column} TIMESTAMP"
+                        when Date
+                          "#{column} DATE"
+                        when Integer
+                          "#{column} BIGINT"
+                        when BigDecimal
+                          "#{column} DECIMAL"
+                        when Float
+                          "#{column} DOUBLE PRECISION"
+                        when true, false
+                          "#{column} BOOLEAN"
+                        else
+                          "#{column} VARCHAR(512)"
+                        end
       end
     end
 
     def initial_columns
-      if transformer == "postgres"
+      if transformer == 'postgres'
         [
-          "id SERIAL PRIMARY KEY",
-          "inserted_at TIMESTAMP DEFAULT NOW()"
+          'id SERIAL PRIMARY KEY',
+          'inserted_at TIMESTAMP DEFAULT NOW()'
         ]
       else
         [
-          "uuid_ts TIMESTAMP DEFAULT NOW()"
+          'uuid_ts TIMESTAMP DEFAULT NOW()'
         ]
       end
     end
@@ -81,12 +82,12 @@ module Charrington
     end
 
     def execute(sql)
-      statement = connection.prepareStatement( sql.gsub(/\s+/, " ").strip )
-      statement.execute()
+      statement = connection.prepareStatement(sql.gsub(/\s+/, ' ').strip)
+      statement.execute
     rescue Java::OrgPostgresqlUtil::PSQLException => e
-      self.logger.error "PSQLException: #{e.message} sql=#{sql}"
+      logger.error "PSQLException: #{e.message} sql=#{sql}"
     ensure
-      statement.close unless statement.nil?
+      statement&.close
     end
   end
 end

data/lib/logstash/outputs/charrington/create_redshift_table.rb

@@ -1,10 +1,11 @@
-require File.join(File.dirname(__FILE__), "service")
+# frozen_string_literal: true
+
+require File.join(File.dirname(__FILE__), 'service')
 
 module Charrington
+  # This service will create a table dynamically based on the JSON structure.
+  # This is potentially called from Insert when an insert fails.
   class CreateRedshiftTable
-    # This service will create a table dynamically based on the JSON structure.
-    # This is potentially called from Insert when an insert fails.
-
     include Service
     include LogStash::Util::Loggable
     attr_reader :connection, :event, :table_name, :columns, :schema, :opts
@@ -13,7 +14,7 @@ module Charrington
     Error = Class.new(StandardError)
     CreateFailed = Class.new(Error)
 
-    def initialize(connection, event, schema, table_name, columns, opts = {})
+    def initialize(connection, event, schema, table_name, columns, _opts = {}) # rubocop:disable Metrics/ParameterLists
       @connection = connection
       @event = event.to_hash
       @schema = schema
@@ -24,10 +25,10 @@ module Charrington
 
     def call
       set_column_types
-      self.logger.info "Finished running set_column_types and now have column_types for create table of: #{column_types}"
+      logger.info "Finished running set_column_types and now have column_types for create table of: #{column_types}"
       create_table
       true
-    rescue => e
+    rescue StandardError => e
       raise CreateFailed, e.message
     ensure
       @column_types.clear if @column_types.is_a? Array
@@ -36,31 +37,32 @@ module Charrington
     private
 
     # https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_TABLE_NEW.html
-    def set_column_types
+    def set_column_types # rubocop:disable Metrics/CyclomaticComplexity
       columns.each do |column|
-        if column == "id"
+        case column
+        when 'id'
           column_types << "#{column} VARCHAR(512) NOT NULL distkey CONSTRAINT #{table_name}_pkey primary key"
           next
-        elsif column == "sent_at"
+        when 'sent_at'
           column_types << "#{column} TIMESTAMP"
           next
         end
-        case event[column]
-        when Time, LogStash::Timestamp
-          column_types << "#{column} TIMESTAMP"
-        when Date
-          column_types << "#{column} DATE"
-        when Integer
-          column_types << "#{column} BIGINT"
-        when BigDecimal
-          column_types << "#{column} DECIMAL"
-        when Float
-          column_types << "#{column} DOUBLE PRECISION"
-        when true, false
-          column_types << "#{column} BOOLEAN"
-        else
-          column_types << "#{column} VARCHAR(512)"
-        end
+        column_types << case event[column]
+                        when Time, LogStash::Timestamp
+                          "#{column} TIMESTAMP"
+                        when Date
+                          "#{column} DATE"
+                        when Integer
+                          "#{column} BIGINT"
+                        when BigDecimal
+                          "#{column} DECIMAL"
+                        when Float
+                          "#{column} DOUBLE PRECISION"
+                        when true, false
+                          "#{column} BOOLEAN"
+                        else
+                          "#{column} VARCHAR(512)"
+                        end
       end
     end
 
@@ -79,15 +81,15 @@ module Charrington
     end
 
     def execute(sql)
-      self.logger.info "Running sql of: #{sql}"
+      logger.info "Running sql of: #{sql}"
 
-      statement = connection.prepareStatement( sql.gsub(/\s+/, " ").strip )
-      statement.execute()
+      statement = connection.prepareStatement(sql.gsub(/\s+/, ' ').strip)
+      statement.execute
     rescue Java::JavaSql::SQLException => e
-      self.logger.info "Redshift SQLException: #{e.message}, with SQL: #{sql}"
+      logger.info "Redshift SQLException: #{e.message}, with SQL: #{sql}"
     ensure
-      self.logger.info "Within ensure block of create_redshift_table.rb and value of statement.nil?: #{statement.nil?}"
-      statement.close unless statement.nil?
+      logger.info "Within ensure block of create_redshift_table.rb and value of statement.nil?: #{statement.nil?}"
+      statement&.close
     end
   end
 end