logstash-output-charrington 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc7913c9e276dc1f40fa722cb271f21d0e19ab3270b7fbd854cc7057c6a50bc3
-  data.tar.gz: 4be9c45b7e08e2ef3ab84b07109096305cfcc33a84b3cc9625292d8e572bf86b
+  metadata.gz: d8166adfa6e084d6e0fd56921edbc8b71dae22e6e5e282c86f73753692f14dbf
+  data.tar.gz: 5b0286ebc0540a5152a11ffacd441d465c6cc44b7c67e66a4b08f5c825588dca
 SHA512:
-  metadata.gz: 13ed00f23566b13bd75c3935154d0c57336fe0be4e5ea1f80a0a529b068468159863fcd564e4f0ce730645d2a8ffe49d467f86fa6c0addd8b7fa8ac8b525e45a
-  data.tar.gz: 242d0f1f8174e627f45d32924f7864d78969cb92f2f20141b9311a02e5d1cb329c9c4b13b62b79c60bb252df0df3fdb0d4eeed492609ddc0ba2e00d4a1176739
+  metadata.gz: b977663c714b04890525419c98773e80f2e55993ff3cfc0e50ca3dee6d2b06e81a948a39d1e0004ba156ed219482592ddb0f604514310a95fd6e9ac66312db5d
+  data.tar.gz: 32a8eee42bf6a88c3c3a244805546edc2a5cbe67626bd5b12450c2c963b1967f7aea2821cfbd068478a71a2364f4ca24e1fc5759c76f7ebb6bdffb64c5011e43

data/lib/logstash/outputs/charrington.rb

@@ -7,33 +7,40 @@ require 'java'
 require 'logstash-output-charrington_jars'
 require 'json'
 require 'bigdecimal'
+require 'pry'
+require File.join(File.dirname(__FILE__), "charrington/process")
+require File.join(File.dirname(__FILE__), "charrington/transform")
+require File.join(File.dirname(__FILE__), "charrington/insert")
 
 # Write events to a SQL engine, using JDBC.
-#
-# It is upto the user of the plugin to correctly configure the plugin. This
-# includes correctly crafting the SQL statement, and matching the number of
-# parameters correctly.
-#
+# It is upto the user of the plugin to correctly configure the plugin.
+
+# This class is responsible for setting things up, creating the connection,
+# and handling retries. Charrington::Insert is where the insert
+# is attempted. If that fails, it will try to either
+# create a table via Charrington::CreateTable
+# or alter an existing one via Charrington::AlterTable
+
 class LogStash::Outputs::Charrington < LogStash::Outputs::Base
   concurrency :shared
 
   STRFTIME_FMT = '%Y-%m-%d %T.%L'.freeze
 
-  RETRYABLE_SQLSTATE_CLASSES = [
-    # Classes of retryable SQLSTATE codes
-    # Not all in the class will be retryable. However, this is the best that
-    # we've got right now.
-    # If a custom state code is required, set it in retry_sql_states.
-    '08', # Connection Exception
-    '24', # Invalid Cursor State (Maybe retry-able in some circumstances)
-    '25', # Invalid Transaction State
-    '40', # Transaction Rollback
-    '53', # Insufficient Resources
-    '54', # Program Limit Exceeded (MAYBE)
-    '55', # Object Not In Prerequisite State
-    '57', # Operator Intervention
-    '58', # System Error
-  ].freeze
+  # RETRYABLE_SQLSTATE_CLASSES = [
+  #   # Classes of retryable SQLSTATE codes
+  #   # Not all in the class will be retryable. However, this is the best that
+  #   # we've got right now.
+  #   # If a custom state code is required, set it in retry_sql_states.
+  #   '08', # Connection Exception
+  #   '24', # Invalid Cursor State (Maybe retry-able in some circumstances)
+  #   '25', # Invalid Transaction State
+  #   '40', # Transaction Rollback
+  #   '53', # Insufficient Resources
+  #   '54', # Program Limit Exceeded (MAYBE)
+  #   '55', # Object Not In Prerequisite State
+  #   '57', # Operator Intervention
+  #   '58', # System Error
+  # ].freeze
 
   config_name 'charrington'
 
@@ -56,26 +63,12 @@ class LogStash::Outputs::Charrington < LogStash::Outputs::Base
   # jdbc password - optional, maybe in the connection string
   config :password, validate: :string, required: false
 
-  # [ "insert into table (message) values(?)", "%{message}" ]
-  config :statement, validate: :array, required: true
-
-  # If this is an unsafe statement, use event.sprintf
-  # This also has potential performance penalties due to having to create a
-  # new statement for each event, rather than adding to the batch and issuing
-  # multiple inserts in 1 go
-  config :unsafe_statement, validate: :boolean, default: false
-
   # Number of connections in the pool to maintain
   config :max_pool_size, validate: :number, default: 5
 
   # Connection timeout
   config :connection_timeout, validate: :number, default: 10000
 
-  # We buffer a certain number of events before flushing that out to SQL.
-  # This setting controls how many events will be buffered before sending a
-  # batch of events.
-  config :flush_size, validate: :number, default: 1000
-
   # Set initial interval in seconds between retries. Doubled on each retry up to `retry_max_interval`
   config :retry_initial_interval, validate: :number, default: 2
 
@@ -94,7 +87,6 @@ class LogStash::Outputs::Charrington < LogStash::Outputs::Base
   # Maximum number of sequential failed attempts, before we stop retrying.
   # If set to < 1, then it will infinitely retry.
   # At the default values this is a little over 10 minutes
-
   config :max_flush_exceptions, validate: :number, default: 10
 
   config :max_repeat_exceptions, obsolete: 'This has been replaced by max_flush_exceptions - which behaves slightly differently. Please check the documentation.'
@@ -114,25 +106,23 @@ class LogStash::Outputs::Charrington < LogStash::Outputs::Base
     @logger.info('JDBC - Starting up')
 
     load_jar_files!
-
     @stopping = Concurrent::AtomicBoolean.new(false)
 
-    @logger.warn('JDBC - Flush size is set to > 1000') if @flush_size > 1000
-
-    if @statement.empty?
-      @logger.error('JDBC - No statement provided. Configuration error.')
-    end
-
-    if !@unsafe_statement && @statement.length < 2
-      @logger.error("JDBC - Statement has no parameters. No events will be inserted into SQL as you're not passing any event data. Likely configuration error.")
-    end
-
     setup_and_test_pool!
   end
 
   def multi_receive(events)
-    events.each_slice(@flush_size) do |slice|
-      retrying_submit(slice)
+    events.each do |event|
+      connection = get_connection
+      break unless connection
+
+      opts = { connection: connection,
+               schema: @schema,
+               max_retries: @max_flush_exceptions,
+               retry_initial_interval: @retry_initial_interval }
+
+      Charrington::Process.call(connection, event, opts)
+      connection.close unless connection.nil?
     end
   end
 
@@ -145,19 +135,14 @@ class LogStash::Outputs::Charrington < LogStash::Outputs::Base
   private
 
   def setup_and_test_pool!
-    # Setup pool
     @pool = Java::ComZaxxerHikari::HikariDataSource.new
-
-    @pool.setAutoCommit(@driver_auto_commit)
     @pool.setDriverClassName(@driver_class) if @driver_class
-
-    @pool.setJdbcUrl(@connection_string)
-
     @pool.setUsername(@username) if @username
     @pool.setPassword(@password) if @password
-
     @pool.setMaximumPoolSize(@max_pool_size)
     @pool.setConnectionTimeout(@connection_timeout)
+    @pool.setAutoCommit(@driver_auto_commit)
+    @pool.setJdbcUrl(@connection_string)
 
     validate_connection_timeout = (@connection_timeout / 1000) / 2
 
@@ -176,16 +161,15 @@ class LogStash::Outputs::Charrington < LogStash::Outputs::Base
     test_connection.close
   end
 
+  # Load jar from driver path
   def load_jar_files!
-    # Load jar from driver path
     unless @driver_jar_path.nil?
       raise LogStash::ConfigurationError, 'JDBC - Could not find jar file at given path. Check config.' unless File.exist? @driver_jar_path
       require @driver_jar_path
       return
     end
 
-    # Revert original behaviour of loading from vendor directory
-    # if no path given
+    # Revert original behaviour of loading from vendor directory if no path given
     jarpath = if ENV['LOGSTASH_HOME']
                 File.join(ENV['LOGSTASH_HOME'], '/vendor/jar/jdbc/*.jar')
               else
@@ -203,192 +187,11 @@ class LogStash::Outputs::Charrington < LogStash::Outputs::Base
     end
   end
 
-  def create_statement(event)
-    event = event.to_hash
-    hashed = event.delete_if {|k, _v| k.start_with?("@") || k == 'host' || k == 'path' }
-
-    columns = '(' + hashed.keys.join(', ') + ')'
-
-    value_placeholders = ('?' * hashed.length).split('')
-    values = '(' + value_placeholders.join(', ') + ')'
-
-    table_name = create_table_name(event)
-    return "INSERT INTO #{table_name} #{columns} VALUES #{values}", hashed.keys
-  end
-
-  def create_table_name(event)
-    raise TableNameNil.new("Table name is nil", event) if event.nil?
-
-    event = event.to_hash["event"].to_s.strip
-    raise TableNameNil.new("Table name is nil", event) if event.empty?
-
-    schema = @schema.empty? ? '' : "#{@schema}."
-    "#{schema}#{event.gsub(/[ \-_]+/, "_").downcase}"
-  end
-
-  def prepared_statement(keys)
-    keys.map do |key|
-      turn_into_wrapped(key)
-    end
-  end
-
-  def turn_into_wrapped(key)
-    "[#{key}]"
-  end
-
-  def submit(events)
-    connection = nil
-    statement = nil
-    events_to_retry = []
-
-    begin
-      connection = @pool.getConnection
-    rescue => e
-      log_jdbc_exception(e, true, nil)
-      # If a connection is not available, then the server has gone away
-      # We're not counting that towards our retry count.
-      return events, false
-    end
-
-    events.each do |event|
-
-
-      begin
-        ins, columns = create_statement(event)
-        keys_we_care_about = prepared_statement(columns)
-        statement = connection.prepareStatement(
-          ins
-        )
-        statement = add_statement_event_params(statement, event, keys_we_care_about)
-        statement.execute
-      rescue TableNameNil => e
-        @logger.error("#{e.message} event=#{e.event}")
-      rescue => e
-        @logger.error "Rescue from SQLException #{e.message}"
-        create_statement = make_create_statement(event, columns)
-        puts 'create_statement'
-        puts create_statement
-
-        statement = connection.prepareStatement(
-          create_statement
-        )
-        statement.execute
-        @logger.debug('Created new Table.')
-        events_to_retry.push(event)
-      ensure
-        statement.close unless statement.nil?
-      end
-    end
-
-    connection.close unless connection.nil?
-
-    return events_to_retry, true
-  end
-
-  def retrying_submit(actions)
-    # Initially we submit the full list of actions
-    submit_actions = actions
-    count_as_attempt = true
-
-    attempts = 1
-
-    sleep_interval = @retry_initial_interval
-    while @stopping.false? and (submit_actions and !submit_actions.empty?)
-      return if !submit_actions || submit_actions.empty? # If everything's a success we move along
-      # We retry whatever didn't succeed
-      submit_actions, count_as_attempt = submit(submit_actions)
-
-      # Everything was a success!
-      break if !submit_actions || submit_actions.empty?
-
-      if @max_flush_exceptions > 0 and count_as_attempt == true
-        attempts += 1
-
-        if attempts > @max_flush_exceptions
-          @logger.error("JDBC - max_flush_exceptions has been reached. #{submit_actions.length} events have been unable to be sent to SQL and are being dropped. See previously logged exceptions for details.")
-          break
-        end
-      end
-
-      # If we're retrying the action sleep for the recommended interval
-      # Double the interval for the next time through to achieve exponential backoff
-      Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
-      sleep_interval = next_sleep_interval(sleep_interval)
-    end
-  end
-
-  def make_create_statement(event, keys_we_care_about)
-    columns = []
-
-    keys_we_care_about.each_with_index do |key, idx|
-      wrapped = turn_into_wrapped(key)
-
-      case event.get(wrapped)
-      when Time, LogStash::Timestamp
-        columns << "#{key} TIMESTAMP"
-      when Integer
-        columns << "#{key} BIGINT"
-      when BigDecimal
-        columns << "#{key} DECIMAL"
-      when Float
-        columns << "#{key} DOUBLE PRECISION"
-      when String, Array, Hash
-        columns << "#{key} VARCHAR"
-      when true, false
-        columns << "#{key} BOOLEAN"
-      end
-    end
-
-    "CREATE TABLE IF NOT EXISTS #{create_table_name(event)} (#{columns.join(', ')})"
-  end
-
-  def add_statement_event_params(statement, event, keys_we_care_about)
-    keys_we_care_about.each_with_index do |key, idx|
-      if @enable_event_as_json_keyword == true and key.is_a? String and key == @event_as_json_keyword
-        value = event.to_json
-      elsif key.is_a? String
-        value = event.get(key)
-        if value.nil? and key =~ /%\{/
-          value = event.sprintf(key)
-        end
-      else
-        value = key
-      end
-
-      case value
-      when Time
-        statement.setString(idx + 1, value.strftime(STRFTIME_FMT))
-      when LogStash::Timestamp
-        statement.setString(idx + 1, value.time.strftime(STRFTIME_FMT))
-      when Integer
-        if value > 2147483647 or value < -2147483648
-          statement.setLong(idx + 1, value)
-        else
-          statement.setInt(idx + 1, value)
-        end
-      when BigDecimal
-        statement.setBigDecimal(idx + 1, value.to_java)
-      when Float
-        statement.setFloat(idx + 1, value)
-      when String
-        statement.setString(idx + 1, value)
-      when Array, Hash
-        statement.setString(idx + 1, value.to_json)
-      when true, false
-        statement.setBoolean(idx + 1, value)
-      else
-        statement.setString(idx + 1, nil)
-      end
-    end
-
-    statement
-  end
-
-  def retry_exception?(exception, event)
-    retrying = (exception.respond_to? 'getSQLState' and (RETRYABLE_SQLSTATE_CLASSES.include?(exception.getSQLState.to_s[0,2]) or @retry_sql_states.include?(exception.getSQLState)))
-    log_jdbc_exception(exception, retrying, event)
-
-    retrying
+  def get_connection
+    connection = @pool.getConnection
+  rescue => e
+    log_jdbc_exception(e, true, nil)
+    false
   end
 
   def log_jdbc_exception(exception, retrying, event)
@@ -400,7 +203,7 @@ class LogStash::Outputs::Charrington < LogStash::Outputs::Base
     loop do
       # TODO reformat event output so that it only shows the fields necessary.
 
-      @logger.send(log_method, log_text, :exception => current_exception, :statement => @statement[0], :event => event)
+      @logger.send(log_method, log_text, :exception => current_exception, :event => event)
 
       if current_exception.respond_to? 'getNextException'
         current_exception = current_exception.getNextException()
@@ -411,18 +214,4 @@ class LogStash::Outputs::Charrington < LogStash::Outputs::Base
       break if current_exception == nil
     end
   end
-
-  def next_sleep_interval(current_interval)
-    doubled = current_interval * 2
-    doubled > @retry_max_interval ? @retry_max_interval : doubled
-  end
-end # class LogStash::Outputs::Charrington
-
-class TableNameNil < StandardError
-  attr_reader :event
-
-  def initialize(msg='Table name is nil', event={})
-    @event = event
-    super(msg)
-  end
 end

data/lib/logstash/outputs/charrington/alter_table.rb

@@ -0,0 +1,98 @@
+require File.join(File.dirname(__FILE__), "service")
+
+module Charrington
+  class AlterTable
+    # This service will add columns to an existing table dynamically based on finding new keys in the JSON structure.
+    # This is potentially called from Insert when an insert fails.
+
+    include Service
+    attr_reader :connection, :event, :table_name, :columns
+    attr_accessor :column_types
+
+    Error = Class.new(StandardError)
+    AlterFailed = Class.new(Error)
+
+    def initialize(connection, event, table_name, columns)
+      @connection = connection
+      @event = event
+      @table_name = table_name
+      @columns = columns
+      @column_types = []
+    end
+
+    def call
+      set_column_types
+      alter_table
+      true
+    rescue => e
+      raise AlterFailed, e.message
+    ensure
+      @column_types.clear if @column_types.is_a? Array
+    end
+
+    private
+
+    def alter_table
+      execute("ALTER TABLE IF EXISTS #{table_name} #{columns_fragment}")
+    end
+
+    def columns_fragment
+      column_types.map do |column|
+        "ADD COLUMN IF NOT EXISTS #{column}"
+      end.join(",")
+    end
+
+    def set_column_types
+      (columns - current_table_columns).each_with_index do |key, idx|
+
+        case event[key]
+        when Time, LogStash::Timestamp
+          column_types << "#{key} TIMESTAMP"
+        when Date
+          column_types << "#{key} DATE"
+        when Integer
+          column_types << "#{key} BIGINT"
+        when BigDecimal
+          column_types << "#{key} DECIMAL"
+        when Float
+          column_types << "#{key} DOUBLE PRECISION"
+        when true, false
+          column_types << "#{key} BOOLEAN"
+        else
+          column_types << "#{key} VARCHAR"
+        end
+      end
+    end
+
+    def current_table_columns
+      sql = "SELECT * FROM #{table_name} LIMIT 1;"
+      rs = executeQuery(sql)
+      meta_data = rs.getMetaData()
+      column_count = meta_data.getColumnCount()
+
+      (1..column_count).map {|i| meta_data.getColumnName(i) }
+    end
+
+    def execute(sql)
+      stmt = connection.prepareStatement(prep_sql(sql))
+      stmt.execute()
+    rescue Java::OrgPostgresqlUtil::PSQLException => e
+      # @logger.error("#{e.message}")
+    ensure
+      stmt.close unless stmt.nil?
+    end
+
+    def executeQuery(sql)
+      stmt = connection.createStatement()
+      stmt.executeQuery(prep_sql(sql))
+    rescue Java::OrgPostgresqlUtil::PSQLException => e
+      # @logger.error("#{e.message}")
+    ensure
+      stmt.close unless stmt.nil?
+    end
+
+    def prep_sql(sql)
+      sql.gsub(/\s+/, " ").strip
+    end
+  end
+end

data/lib/logstash/outputs/charrington/create_table.rb

@@ -0,0 +1,69 @@
+require File.join(File.dirname(__FILE__), "service")
+
+module Charrington
+  class CreateTable
+    # This service will create a table dynamically based on the JSON structure.
+    # This is potentially called from Insert when an insert fails.
+
+    include Service
+    attr_reader :connection, :event, :table_name, :columns
+    attr_accessor :column_types
+
+    Error = Class.new(StandardError)
+    CreateFailed = Class.new(Error)
+
+    def initialize(connection, event, table_name, columns)
+      @connection = connection
+      @event = event.to_hash
+      @table_name = table_name
+      @columns = columns
+      @column_types = []
+    end
+
+    def call
+      set_column_types
+      create_table
+      true
+    rescue => e
+      raise CreateFailed, e.message
+    ensure
+      @column_types.clear if @column_types.is_a? Array
+    end
+
+    private
+
+    def set_column_types
+      columns.each do |column|
+        case event[column]
+        when Time, LogStash::Timestamp
+          column_types << "#{column} TIMESTAMP"
+        when Date
+          column_types << "#{column} DATE"
+        when Integer
+          column_types << "#{column} BIGINT"
+        when BigDecimal
+          column_types << "#{column} DECIMAL"
+        when Float
+          column_types << "#{column} DOUBLE PRECISION"
+        when true, false
+          column_types << "#{column} BOOLEAN"
+        else
+          column_types << "#{column} VARCHAR"
+        end
+      end
+    end
+
+    def create_table
+      execute("CREATE TABLE IF NOT EXISTS #{table_name} (#{column_types.join(', ')})")
+    end
+
+    def execute(sql)
+      statement = connection.prepareStatement( sql.gsub(/\s+/, " ").strip )
+      statement.execute()
+    rescue Java::OrgPostgresqlUtil::PSQLException => e
+      # @logger.error("#{e.message}")
+    ensure
+      statement.close unless statement.nil?
+    end
+  end
+end

data/lib/logstash/outputs/charrington/insert.rb

@@ -0,0 +1,141 @@
+require File.join(File.dirname(__FILE__), "create_table")
+require File.join(File.dirname(__FILE__), "alter_table")
+require File.join(File.dirname(__FILE__), "service")
+
+module Charrington
+  class Insert
+    # This service assumes that the data is already clean and in a flattened hash format.
+    # The Transform service should be called before calling this.
+
+    include Service
+    attr_accessor :event, :should_retry
+    attr_reader :connection, :schema, :table_name, :columns
+    attr_reader :event_as_json_keyword, :enable_event_as_json_keyword
+
+    Error = Class.new(StandardError)
+    EventNil = Class.new(Error)
+    TableNameNil = Class.new(Error)
+    InsertFailed = Class.new(Error)
+
+    def initialize(connection, event, opts = {})
+      raise EventNil, "Table name is nil" if event.nil?
+      @event = event.to_hash
+
+      event_name = event["event"].to_s.downcase.strip
+      raise TableNameNil, "Table name is nil" if event_name.empty?
+
+      @connection = connection
+      @schema = opts[:schema].empty? ? '' : "#{opts[:schema]}."
+      @table_name = "#{@schema}#{event_name.gsub(/[^a-z0-9]+/, "_")}"
+
+      @columns = event.keys
+      @should_retry = false
+      @enable_event_as_json_keyword = opts[:enable_event_as_json_keyword]
+      @event_as_json_keyword = opts[:event_as_json_keyword]
+    end
+
+    def call
+      stmt = connection.prepareStatement(insert_statement)
+      stmt = add_statement_event_params(stmt)
+      stmt.execute
+      should_retry
+    rescue Java::OrgPostgresqlUtil::PSQLException => e
+      case e.getSQLState()
+      when "42P01"
+        should_retry = Charrington::CreateTable.call(connection, event, table_name, columns)
+      when "42703"
+        should_retry = Charrington::AlterTable.call(connection, event, table_name, columns)
+      else
+        raise InsertFailed, "Charrington: Rescue from SQLException #{e.message}"
+      end
+      should_retry
+    rescue => e
+      raise InsertFailed, "Charrington: Rescue from SQLException #{e.message}"
+    ensure
+      stmt.close unless stmt.nil?
+      cleanup
+    end
+
+    private
+
+    def cleanup
+      @columns.clear if clearable(@columns)
+    end
+
+    ### Set Variables
+
+    def columns_text
+      @columns_text ||= arr_to_csv(columns)
+    end
+
+    def value_placeholders
+      ('?' * columns.length).split('')
+    end
+
+    def insert_values
+      arr_to_csv(value_placeholders)
+    end
+
+    def insert_statement
+      "INSERT INTO #{table_name} #{columns_text} VALUES #{insert_values}"
+    end
+
+    def prepared_statement
+      columns.map { |column| "[#{column}]" }
+    end
+
+    def add_statement_event_params(stmt)
+      columns.each_with_index do |key, idx|
+        pos = idx + 1
+        value = event[key]
+
+        case value
+        when Time
+          stmt.setString(pos, value.strftime(STRFTIME_FMT))
+        when LogStash::Timestamp
+          stmt.setString(pos, value.time.strftime(STRFTIME_FMT))
+        when Integer
+          if value > 2147483647 || value < -2147483648
+            stmt.setLong(pos, value)
+          else
+            stmt.setInt(pos, value)
+          end
+        when BigDecimal
+          stmt.setBigDecimal(pos, value.to_java)
+        when Float
+          stmt.setFloat(pos, value)
+        when String
+          stmt.setString(pos, value)
+        when Array, Hash
+          stmt.setString(pos, value.to_json)
+        when true, false
+          stmt.setBoolean(pos, value)
+        else
+          stmt.setString(pos, nil)
+        end
+      end
+      stmt
+    end
+
+    ### Helpers
+
+    def arr_to_csv(arr)
+      '(' + arr.join(', ') + ')'
+    end
+
+    def clearable(obj)
+      obj.is_a? Hash or obj.is_a? Array
+    end
+
+    ### SQL
+
+    def execute(connection, sql)
+      statement = connection.prepareStatement( sql.gsub(/\s+/, " ").strip )
+      statement.execute()
+    rescue Java::OrgPostgresqlUtil::PSQLException => e
+      @logger.error("#{e.message}")
+    ensure
+      statement.close unless statement.nil?
+    end
+  end
+end

data/lib/logstash/outputs/charrington/process.rb

@@ -0,0 +1,62 @@
+require File.join(File.dirname(__FILE__), "service")
+
+module Charrington
+  class Process
+    # This service starts the process of attempting to insert a row.
+    # It handles retries where applicable.
+
+    include Service
+    attr_reader :event, :connection, :opts, :max_retries, :schema, :retry_max_interval
+    attr_accessor :retry_interval, :should_retry
+
+    Error = Class.new(StandardError)
+    ProcessFailed = Class.new(Error)
+    EventNil = Class.new(Error)
+
+    def initialize(connection, event, opts={})
+      raise EventNil, "Event is nil" if event.nil?
+      @connection = connection
+      @event = event.to_hash
+      @opts = opts
+
+      @max_retries = opts[:max_retries] || 10
+      @retry_max_interval = opts[:retry_max_interval] || 2
+      @retry_interval = opts[:retry_initial_interval] || 2
+
+      @attempts = 1
+      @should_retry = true
+    end
+
+    def call
+      while should_retry do
+        transformed = Charrington::Transform.call(event)
+        should_retry = Charrington::Insert.call(connection, transformed, opts)
+        break if !should_retry
+
+        @attempts += 1
+        break if @attempts > max_retries
+
+        # If we're retrying the action, sleep for the recommended interval
+        # Double the interval for the next time through to achieve exponential backoff
+        sleep_interval
+      end
+    rescue => e
+      raise ProcessFailed, e.message
+    ensure
+      connection.close unless connection.nil?
+      @event.clear if clearable(@event)
+    end
+
+    private
+
+    def sleep_interval
+      sleep(retry_interval)
+      doubled = retry_interval * 2
+      retry_interval = doubled > retry_max_interval ? retry_max_interval : doubled
+    end
+
+    def clearable(obj)
+      obj.is_a? Hash or obj.is_a? Array
+    end
+  end
+end

data/lib/logstash/outputs/charrington/service.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+module Service
+  def self.included(base)
+    base.extend ClassMethods
+  end
+
+  module ClassMethods
+    def call(*args)
+      new(*args).call
+    end
+  end
+end

data/lib/logstash/outputs/charrington/transform.rb

@@ -0,0 +1,47 @@
+require File.join(File.dirname(__FILE__), "service")
+
+module Charrington
+  class Transform
+    include Service
+    attr_accessor :event
+    attr_reader :top_level_keys
+
+    Error = Class.new(StandardError)
+    EventNil = Class.new(Error)
+    TableNameNil = Class.new(Error)
+
+    KEY_BLACKLIST = ['host','path','jwt']
+
+    def initialize(event)
+      raise EventNil, "Event is nil" if event.nil?
+      event = event.is_a?(Hash) ? event : event.to_hash
+      @event = drop_keys(event)
+      @top_level_keys = @event.keys
+    end
+
+    def call
+      flattened = flatten_hash(event)
+      top_level_keys.each { |k| event.delete(k) }
+      flattened.each_pair { |key, val| event[key] = val }
+      event
+    end
+
+    private
+
+    def drop_keys(event)
+      event.delete_if {|k, _v| k.start_with?("@") || KEY_BLACKLIST.include?(k) }
+    end
+
+    def flatten_hash(hash)
+      hash.each_with_object({}) do |(k, v), acc|
+        if v.is_a? Hash
+          flatten_hash(v).map do |h_k, h_v|
+            acc["#{k}_#{h_k}"] = h_v
+          end
+        else
+          acc[k] = v
+        end
+      end
+    end
+  end
+end

data/logstash-output-charrington.gemspec

@@ -1,6 +1,7 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-charrington'
-  s.version = '0.1.1'
+  s.version = '0.2.0'
+
   s.licenses = ['Apache License (2.0)']
   s.summary = 'This plugin allows you to output to SQL, via JDBC'
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install 'logstash-output-charrington'. This gem is not a stand-alone program"
@@ -27,4 +28,6 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'jar-dependencies'
   s.add_development_dependency 'ruby-maven', '~> 3.3'
   s.add_development_dependency 'rubocop', '0.41.2'
+  s.add_development_dependency 'logstash-input-generator'
+  s.add_development_dependency 'logstash-codec-json'
 end

data/spec/outputs/charrington_spec.rb

@@ -1,11 +1,44 @@
-# require_relative '../charrington_spec_helper'
-#
-# describe LogStash::Outputs::Jdbc do
-#   context 'when initializing' do
-#     it 'shouldn\'t register without a config' do
-#       expect do
-#         LogStash::Plugin.lookup('output', 'charrington').new
-#       end.to raise_error(LogStash::ConfigurationError)
-#     end
-#   end
-# end
+require_relative '../charrington_spec_helper'
+
+describe LogStash::Outputs::Charrington do
+  describe 'when initializing' do
+    it 'shouldn\'t register without a config' do
+      expect do
+        LogStash::Plugin.lookup('output', 'charrington').new
+      end.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+
+  describe 'integration tests' do
+    config <<-CONFIG
+    input {
+      generator {
+        message => '{"id": "abc", "app_name": "Web App", "event": "Hi - Dan"}'
+        codec => 'json'
+        count => 1
+      }
+    }
+
+    output {
+      charrington {
+        connection_string => 'jdbc:postgresql://localhost:5432/winston?user=postgres&password=postgres'
+        driver_jar_path => '/projects/logstash-output-charrington/vendor/postgresql-42.2.5.jar'
+        schema => 'dea'
+      }
+    }
+    CONFIG
+
+    agent do
+      puts "IT'S WORKING!!!!!"
+    end
+  end
+
+  # context 'running' do
+  #   it 'should transform some JSON' do
+  #     transformed = Charrington::Transform.call({"a" => 1})
+  #     puts transformed
+  #   end
+  # end
+end
+
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-charrington
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - dconger
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-05-30 00:00:00.000000000 Z
+date: 2019-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -102,6 +102,34 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.41.2
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-input-generator
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-json
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install 'logstash-output-charrington'.
   This gem is not a stand-alone program
@@ -118,6 +146,12 @@ files:
 - lib/com/zaxxer/HikariCP/2.7.2/HikariCP-2.7.2.jar
 - lib/logstash-output-charrington_jars.rb
 - lib/logstash/outputs/charrington.rb
+- lib/logstash/outputs/charrington/alter_table.rb
+- lib/logstash/outputs/charrington/create_table.rb
+- lib/logstash/outputs/charrington/insert.rb
+- lib/logstash/outputs/charrington/process.rb
+- lib/logstash/outputs/charrington/service.rb
+- lib/logstash/outputs/charrington/transform.rb
 - lib/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar
 - lib/org/apache/logging/log4j/log4j-slf4j-impl/2.6.2/log4j-slf4j-impl-2.6.2.jar
 - lib/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar