logstash-output-charrington 0.2.2 → 0.3.0

data/lib/logstash/outputs/charrington/process.rb

@@ -6,7 +6,7 @@ module Charrington
     # It handles retries where applicable.
 
     include Service
-    attr_reader :event, :connection, :opts, :max_retries, :schema, :retry_max_interval
+    attr_reader :event, :connection, :opts, :max_retries, :schema, :retry_max_interval, :driver
     attr_accessor :retry_interval, :should_retry
 
     Error = Class.new(StandardError)
@@ -22,6 +22,7 @@ module Charrington
       @max_retries = opts[:max_retries] || 10
       @retry_max_interval = opts[:retry_max_interval] || 2
       @retry_interval = opts[:retry_initial_interval] || 2
+      @driver = opts[:driver]
 
       @attempts = 1
       @should_retry = true
@@ -29,7 +30,12 @@ module Charrington
 
     def call
       while should_retry do
-        transformed = Charrington::Transform.call(event)
+        transformed = case driver
+                        when "redshift"
+                          Charrington::TransformRedshift.call(event)
+                        else
+                          Charrington::TransformPostgres.call(event)
+                        end
         should_retry = Charrington::Insert.call(connection, transformed, opts)
         break if !should_retry
 

data/lib/logstash/outputs/charrington/{transform.rb → transform_postgres.rb}

@@ -1,7 +1,7 @@
 require File.join(File.dirname(__FILE__), "service")
 
 module Charrington
-  class Transform
+  class TransformPostgres
     include Service
     attr_accessor :event
     attr_reader :top_level_keys

data/lib/logstash/outputs/charrington/transform_redshift.rb

@@ -0,0 +1,102 @@
+require File.join(File.dirname(__FILE__), "service")
+require 'securerandom'
+
+module Charrington
+  class TransformRedshift
+    include Service
+    attr_accessor :event
+    attr_reader :top_level_keys
+
+    Error = Class.new(StandardError)
+    EventNil = Class.new(Error)
+    TableNameNil = Class.new(Error)
+    ColumnBlacklist = Class.new(Error)
+
+    KEY_FILTER_BLACKLIST = ['host','path','jwt','sequence']
+    KEY_RAISE_BLACKLIST = ['inserted_at']
+
+    def initialize(event)
+      raise EventNil, "Event is nil" if event.nil?
+      event = event.to_hash
+      @event = drop_keys(event)
+      @top_level_keys = @event.keys
+      check_blacklist
+    end
+
+    def call
+      handle_event_key(event)
+      add_id_to_event(event)
+
+      handle_key_transform(event, "anonymous_id", "anonymous_user")
+      handle_key_transform(event, "sent_at", "published_at")
+
+      handle_meta_section(event)
+
+      transform_session_stuff(event)
+
+      event.delete_if {|k, _v| ['session', 'meta', 'published_at', 'anonymous_user'].include?(k) }
+
+      event
+    end
+
+    private
+
+    def handle_key_transform(hash, key_that_should_be_there, key_to_take_value_from)
+      unless hash.has_key?(key_that_should_be_there)
+        hash[key_that_should_be_there] = hash[key_to_take_value_from] || ""
+      else
+        hash
+      end
+    end
+
+    def add_id_to_event(hash)
+      hash["id"] = SecureRandom.hex(10)
+    end
+
+    def handle_event_key(hash)
+      event_name = hash["event"] || ""
+
+      hash["event_text"] = event_name
+
+      hash["event"] = underscore_event_name(event_name)
+    end
+
+    def underscore_event_name(event_name)
+      event_name.to_s.downcase.strip.gsub(/[^a-z0-9]+/, "_")
+    end
+
+    def transform_session_stuff(hash)
+      session_stuff = hash["session"] || {}
+
+      session_stuff.each {|k, v| hash["context_#{k}"] = v }
+    end
+
+    def handle_meta_section(hash)
+      meta_section = hash["meta"] || {}
+
+      meta_section.each {|k, v| hash[k] = v }
+    end
+
+    def check_blacklist
+      arr = []
+      KEY_RAISE_BLACKLIST.each { |k| arr << k if event.keys.include?(k) }
+      raise ColumnBlacklist, "Event contains these blacklisted keys: #{arr.join(",")}" unless arr.empty?
+    end
+
+    def drop_keys(event)
+      event.delete_if {|k, _v| k.start_with?("@") || KEY_FILTER_BLACKLIST.include?(k) }
+    end
+
+    def flatten_hash(hash)
+      hash.each_with_object({}) do |(k, v), acc|
+        if v.is_a? Hash
+          flatten_hash(v).map do |h_k, h_v|
+            acc["#{k}_#{h_k}"] = h_v
+          end
+        else
+          acc[k] = v
+        end
+      end
+    end
+  end
+end

data/logstash-output-charrington.gemspec

@@ -1,8 +1,9 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-charrington'
-  s.version = '0.2.2'
+  s.version = '0.3.0'
 
-  s.licenses = ['Apache License (2.0)']
+  s.licenses = ['Apache-2.0']
+  s.homepage = 'https://gitlab.podium.com/engineering/analytics/logstash-output-charrington'
   s.summary = 'This plugin allows you to output to SQL, via JDBC'
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install 'logstash-output-charrington'. This gem is not a stand-alone program"
   s.authors = ['dconger', 'brianbroderick', 'spencerdcarlson']
@@ -17,18 +18,19 @@ Gem::Specification.new do |s|
   s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'output' }
 
   # Gem dependencies
-  #
   s.add_runtime_dependency 'logstash-core-plugin-api', ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency 'logstash-codec-plain'
-  s.add_development_dependency 'logstash-devutils'
+  s.add_runtime_dependency 'logstash-codec-plain', '~> 3.0', '>= 3.0.6'
 
+  # The 'install_jars' rake task will download these jars from Maven and put them into the vendor directory
+  # See jar-dependencies gem's wiki - https://github.com/mkristian/jar-dependencies/wiki/declare-jars-inside-gemspec
   s.requirements << "jar 'com.zaxxer:HikariCP', '2.7.2'"
   s.requirements << "jar 'org.apache.logging.log4j:log4j-slf4j-impl', '2.6.2'"
 
-  s.add_development_dependency 'jar-dependencies'
+  s.add_development_dependency 'logstash-devutils', '~> 1.3', '>= 1.3.1'
+  s.add_development_dependency 'jar-dependencies', '~> 0.4.0'
   s.add_development_dependency 'ruby-maven', '~> 3.3'
   s.add_development_dependency 'rubocop', '0.41.2'
-  s.add_development_dependency 'logstash-input-generator'
-  s.add_development_dependency 'logstash-codec-json'
-  s.add_development_dependency 'insist'
+  s.add_development_dependency 'logstash-input-generator', '~> 3.0', '>= 3.0.6'
+  s.add_development_dependency 'logstash-codec-json', '~> 3.0', '>= 3.0.5'
+  s.add_development_dependency 'insist','~> 1.0'
 end

data/spec/charrington_spec_helper.rb

@@ -2,215 +2,102 @@ require 'logstash/devutils/rspec/spec_helper'
 require 'logstash/outputs/charrington'
 require 'stud/temporary'
 require 'java'
+require 'logstash-output-charrington_test_jars'
 require 'securerandom'
+java_import java.util.ArrayList
+java_import java.nio.file.Paths
 
 RSpec::Support::ObjectFormatter.default_instance.max_formatted_output_length = 80000
-
 RSpec.configure do |c|
-
-  def start_service(name)
-    cmd = "sudo /etc/init.d/#{name}* start"
-
-    `which systemctl`
-    if $?.success?
-      cmd = "sudo systemctl start #{name}"
-    end
-
-    `#{cmd}`
-  end
-
-  def stop_service(name)
-    cmd = "sudo /etc/init.d/#{name}* stop"
-
-    `which systemctl`
-    if $?.success?
-      cmd = "sudo systemctl stop #{name}"
-    end
-
-    `#{cmd}`
-  end
-
-end
-
-RSpec.shared_context 'rspec setup' do
-  it 'ensure jar is available' do
-    expect(ENV[jdbc_jar_env]).not_to be_nil, "#{jdbc_jar_env} not defined, required to run tests"
-    expect(File.exist?(ENV[jdbc_jar_env])).to eq(true), "#{jdbc_jar_env} defined, but not valid"
-  end
 end
 
-RSpec.shared_context 'when initializing' do
-  it 'shouldn\'t register with a missing jar file' do
-    jdbc_settings['driver_jar_path'] = nil
-    plugin = LogStash::Plugin.lookup('output', 'jdbc').new(jdbc_settings)
-    expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
+RSpec.shared_context 'pipeline' do
+  let(:run_pipeline) do
+    pipeline = new_pipeline_from_string(config)
+    pipeline.run
   end
 end
 
-RSpec.shared_context 'when outputting messages' do
-  let(:logger) {
-    double("logger")
-  }
-
-  let(:jdbc_test_table) do
-    'logstash_output_jdbc_test'
-  end
-
-  let(:jdbc_drop_table) do
-    "DROP TABLE #{jdbc_test_table}"
-  end
-
-  let(:jdbc_statement_fields) do
-    [
-      {db_field: "created_at",       db_type: "datetime",      db_value: '?',  event_field: '@timestamp'},
-      {db_field: "message",          db_type: "varchar(512)",  db_value: '?',  event_field: 'message'},
-      {db_field: "message_sprintf",  db_type: "varchar(512)",  db_value: '?',  event_field: 'sprintf-%{message}'},
-      {db_field: "static_int",       db_type: "int",           db_value: '?',  event_field: 'int'},
-      {db_field: "static_bigint",    db_type: "bigint",        db_value: '?',  event_field: 'bigint'},
-      {db_field: "static_float",     db_type: "float",         db_value: '?',  event_field: 'float'},
-      {db_field: "static_bool",      db_type: "boolean",       db_value: '?',  event_field: 'bool'},
-      {db_field: "static_bigdec",    db_type: "decimal",       db_value: '?',  event_field: 'bigdec'}
-    ]
-  end
-
-  let(:jdbc_create_table) do
-    fields = jdbc_statement_fields.collect { |entry| "#{entry[:db_field]} #{entry[:db_type]} not null" }.join(", ")
-
-    "CREATE table #{jdbc_test_table} (#{fields})"
-  end
-
-  let(:jdbc_drop_table) do
-    "DROP table #{jdbc_test_table}"
-  end
-
-  let(:jdbc_statement) do
-    fields = jdbc_statement_fields.collect { |entry| "#{entry[:db_field]}" }.join(", ")
-    values = jdbc_statement_fields.collect { |entry| "#{entry[:db_value]}" }.join(", ")
-    statement = jdbc_statement_fields.collect { |entry| entry[:event_field] }
+RSpec.shared_context 'postgres' do
+  def start_database(host='localhost', port=57354, database='winston', user='testuser', password='password')
+    config = Java::RuYandexQatoolsEmbedPostgresql::EmbeddedPostgres::cachedRuntimeConfig(Paths::get('/tmp/charrington-test-db-cache')) # avoid archive extraction every time
+    db = Java::RuYandexQatoolsEmbedPostgresql::EmbeddedPostgres.new
+    @url = db.start(config, host, port, database, user, password, ArrayList.new(["-E", "SQL_ASCII", "--locale=C", "--lc-collate=C", "--lc-ctype=C"]))
 
-    statement.insert(0, "insert into #{jdbc_test_table} (#{fields}) values(#{values})")
+    # setup connection manager
+    @connection_manager = Java::ComZaxxerHikari::HikariDataSource.new
+    @connection_manager.setDriverClassName('org.postgresql.Driver')
+    @connection_manager.setUsername(user)
+    @connection_manager.setPassword(password)
+    @connection_manager.setJdbcUrl(@url)
   end
 
-  let(:systemd_database_service) do
-    nil
+  before(:all) do
+    start_database
   end
 
-  let(:event) do
-    # TODO: Auto generate fields from jdbc_statement_fields
-    LogStash::Event.new({
-      message: "test-message #{SecureRandom.uuid}",
-      float: 12.1,
-      bigint: 4000881632477184,
-      bool: true,
-      int: 1,
-      bigdec: BigDecimal.new("123.123")
-    })
-  end
-
-  let(:plugin) do
-    # Setup logger
-    allow(LogStash::Outputs::Jdbc).to receive(:logger).and_return(logger)
-
-    # XXX: Suppress reflection logging. There has to be a better way around this.
-    allow(logger).to receive(:debug).with(/config LogStash::/)
-
-    # Suppress beta warnings.
-    allow(logger).to receive(:info).with(/Please let us know if you find bugs or have suggestions on how to improve this plugin./)
-
-    # Suppress start up messages.
-    expect(logger).to receive(:info).once.with(/JDBC - Starting up/)
-
-    # Setup plugin
-    output = LogStash::Plugin.lookup('output', 'jdbc').new(jdbc_settings)
-    output.register
-
-    output
-  end
-
-  before :each do
-    # Setup table
-    c = plugin.instance_variable_get(:@pool).getConnection
-
-    # Derby doesn't support IF EXISTS.
-    # Seems like the quickest solution. Bleurgh.
-    begin
-      stmt = c.createStatement
-      stmt.executeUpdate(jdbc_drop_table)
-    rescue
-      # noop
-    ensure
-      stmt.close
+  let(:driver_path) {
+    ENV.fetch('DRIVER_JAR_PATH', Pathname.new("#{Dir.pwd}/vendor/jar-dependencies/test-jars/postgresql-42.2.5.jar").to_s)
+  }
 
-      stmt = c.createStatement
-      stmt.executeUpdate(jdbc_create_table)
-      stmt.close
-      c.close
+  def query(sql)
+    execute(sql, true)
+  end
+
+  def create(sql)
+    execute(sql, false)
+  end
+
+  def insert(sql)
+    execute(sql)
+  end
+
+  def execute(sql, results=false)
+    conn = @connection_manager.getConnection
+    stmt = conn.prepareStatement(sql);
+    if !results
+      stmt.execute
+      return
+    else
+      rs = stmt.executeQuery()
+      meta = rs.getMetaData()
+      n = meta.getColumnCount()
+      results = []
+      while rs.next() do
+        row = {}
+        (1..n).each do |i|
+          row[meta.getColumnName(i).to_sym] = rs.getString(i)
+        end
+        results << row
+      end
+      results
     end
+  rescue => e
+    puts "Error executing query. sql=#{sql} #{e.message}"
+    false
+  ensure
+    stmt.close if !stmt.nil? and !stmt.isClosed
+    conn.close if !conn.nil? and !conn.isClosed
   end
 
-  # Delete table after each
-  after :each do
-    c = plugin.instance_variable_get(:@pool).getConnection
-
-    stmt = c.createStatement
-    stmt.executeUpdate(jdbc_drop_table)
-    stmt.close
-    c.close
-  end
-
-  it 'should save a event' do
-    expect { plugin.multi_receive([event]) }.to_not raise_error
-
-    # Verify the number of items in the output table
-    c = plugin.instance_variable_get(:@pool).getConnection
-
-    # TODO replace this simple count with a check of the actual contents
-
-    stmt = c.prepareStatement("select count(*) as total from #{jdbc_test_table} where message = ?")
-    stmt.setString(1, event.get('message'))
-    rs = stmt.executeQuery
-    count = 0
-    count = rs.getInt('total') while rs.next
-    stmt.close
-    c.close
-
-    expect(count).to eq(1)
+  def drop_table(table)
+    execute_update("DROP TABLE IF EXISTS #{table}")
   end
 
-  it 'should not save event, and log an unretryable exception' do
-    e = event
-    original_event = e.get('message')
-    e.set('message', nil)
-
-    expect(logger).to receive(:error).once.with(/JDBC - Exception. Not retrying/, Hash)
-    expect { plugin.multi_receive([event]) }.to_not raise_error
-
-    e.set('message', original_event)
+  def create_table(sql)
+    execute_update(sql)
   end
 
-  it 'it should retry after a connection loss, and log a warning' do
-    skip "does not run as a service, or known issue with test" if systemd_database_service.nil?
-
-    p = plugin
-
-    # Check that everything is fine right now
-    expect { p.multi_receive([event]) }.not_to raise_error
-
-    stop_service(systemd_database_service)
-
-    # Start a thread to restart the service after the fact.
-    t = Thread.new(systemd_database_service) { |systemd_database_service|
-      sleep 20
-
-      start_service(systemd_database_service)
-    }
-
-    t.run
-
-    expect(logger).to receive(:warn).at_least(:once).with(/JDBC - Exception. Retrying/, Hash)
-    expect { p.multi_receive([event]) }.to_not raise_error
-
-    # Wait for the thread to finish
-    t.join
+  def execute_update(sql)
+    conn = @connection_manager.getConnection
+    stmt = conn.createStatement
+    stmt.executeUpdate(sql)
+    true
+  rescue => e
+    puts "Error executing update. sql=#{sql} #{e.message}"
+    false
+  ensure
+    stmt.close if !stmt.nil? and !stmt.isClosed
+    conn.close if !conn.nil? and !conn.isClosed
   end
 end