logstash-output-charrington 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 95cf47c25df7b4666cfc94d7ca2c4b964164a0d76b57ce62993f8cd5c4d53bcd
-  data.tar.gz: 5a5eb5c0323f16d0b9143d6953dcf24fdf9f430e56a751412142f2f7c01dea93
+  metadata.gz: a50e0dc6a545b094b6632682214046bedf1fe56842b17331c44b890157719ad6
+  data.tar.gz: 1e64be923fef5c32b8a678052d76bf774dadf0214519f5a9aee87052638a90fd
 SHA512:
-  metadata.gz: f88562ff8b73f92e22750359178f301c5fef58bd1e2361c9d1d88e8b2c08bba7b69d001be8c5a87fbd2b741ff6bff70138d954a7f094c7794d2155c9103bea23
-  data.tar.gz: dcdd5585b34e0335693a42a04f4b0897632c7f441ad4cac419975fcca863ea6c3b8eb58154ea8f111557d2d2743c4b4b1d8659c6a5129408134961fed8bf4699
+  metadata.gz: 93f149aec77737579df06f0e82a49524bc49264d09d4938d67fed25a6b6606e495171f70863ca3b73fe7ea092097a64f43028411a2fb85ad58f5e9adc24c1f2b
+  data.tar.gz: b277fd159871eae94471b3cc2329e5327a1904ccb5e1fffbc2c4f53d40f8b867ebe6346f38af374e3cd428307602a3712e05da5deab5a3eef451ed9733c3d3bf

data/README.md

@@ -49,8 +49,45 @@ output {
 ```
 
 ## Build & Publish
+
+**TLDR**
+```bash
+bundle exec rake build                          # build gem
+gem push logstash-output-charrington-x.x.x.gem  # publish
+```
+
+Everything in the [vendor](./vendor) directory (including `test-jars` if it exists) will be packaged with the generated gem.
+Avoid including unnecessary dependencies, by removing the vendor directory and regenerating it with the rake task. 
+See the first command below: 
+ 
 ```bash
+rm -r vendor && bundle exec rake install_jars       # clean vendor directory and re-install runtime jar dependencies
 gem build logstash-output-charrington.gemspec       # build
 gem push logstash-output-charrington-x.x.x.gem      # publish
 gem owner --add <email> logstash-output-charrington # add another authorized publisher
-```
+```
+
+## Testing
+**NOTE:** Downloading test_jars requires maven and Java. You can set them up with asdf: [asdf-java](https://github.com/skotchpine/asdf-java), [asdf-maven](https://github.com/skotchpine/asdf-maven)
+* There is currently an [open bug](https://github.com/skotchpine/asdf-maven/pull/17) with how `asdf-maven` sets `$JAVA_HOME` on a mac. Edit your `~/.asdf/plugins/maven/bin/exec-env` file to fix the bug 
+
+```bash
+bundle exec rake install_test_jars      # install test runtime jar dependencies
+bundle exec rspec
+```
+
+#### Dangling Embedded Postgres
+**TLDR**
+```bash
+lsof -PiTCP -sTCP:LISTEN | grep postgres | awk '{print $2}' | xargs kill
+```
+Sometimes the embedded postgres JAR can leave dangling open processes. This is obvious when you have `export TEST_DEBUG=true`
+and the test output contains a stacktrace that starts with `[2019-06-21T15:18:06,734][ERROR][ru.yandex.qatools.embed.postgresql.PostgresProcess] Failed to read PID file (File '/var/folders/...`.
+To resolve this issue, check for dangling processes using `lsof -PiTCP -sTCP:LISTEN | grep postgres`, then you can kill the process ids
+
+#### Resources
+* [logstash-devutils](https://rubygems.org/gems/logstash-devutils) gem with logstash helper methods and tools
+    * [spec_helper.rb source](https://github.com/elastic/logstash-devutils/blob/master/lib/logstash/devutils/rspec/spec_helper.rb)  
+    * [logstash_helpers.rb source](https://github.com/elastic/logstash-devutils/blob/master/lib/logstash/devutils/rspec/logstash_helpers.rb)
+* [logstash-output-file spec test](https://github.com/logstash-plugins/logstash-output-file/blob/master/spec/outputs/file_spec.rb)
+

data/lib/logstash/outputs/charrington.rb

@@ -7,9 +7,9 @@ require 'java'
 require 'logstash-output-charrington_jars'
 require 'json'
 require 'bigdecimal'
-require 'pry'
 require File.join(File.dirname(__FILE__), "charrington/process")
-require File.join(File.dirname(__FILE__), "charrington/transform")
+require File.join(File.dirname(__FILE__), "charrington/transform_postgres")
+require File.join(File.dirname(__FILE__), "charrington/transform_redshift")
 require File.join(File.dirname(__FILE__), "charrington/insert")
 
 # Write events to a SQL engine, using JDBC.
@@ -83,7 +83,7 @@ class LogStash::Outputs::Charrington < LogStash::Outputs::Base
   config :event_as_json_keyword, validate: :string, default: '@event'
 
   # The database schema
-  config :schema, validate: :string, default: ''
+  config :schema, validate: :string, required: false
 
   def register
     @logger.info('JDBC - Starting up')
@@ -99,10 +99,13 @@ class LogStash::Outputs::Charrington < LogStash::Outputs::Base
       connection = get_connection
       break unless connection
 
+      schema = get_schema(event)
+
       opts = { connection: connection,
-               schema: @schema,
+               schema: schema,
                max_retries: @max_flush_exceptions,
-               retry_initial_interval: @retry_initial_interval }
+               retry_initial_interval: @retry_initial_interval,
+               driver: driver }
       Charrington::Process.call(connection, event, opts)
     rescue => e
       @logger.error("Unable to process event. Event dropped. #{e.message}")
@@ -120,6 +123,15 @@ class LogStash::Outputs::Charrington < LogStash::Outputs::Base
 
   private
 
+  def driver
+    case @driver_class
+    when /redshift/
+      "redshift"
+    else
+      "postgresql"
+    end
+  end
+
   def setup_and_test_pool!
     @pool = Java::ComZaxxerHikari::HikariDataSource.new
     @pool.setDriverClassName(@driver_class) if @driver_class
@@ -173,6 +185,21 @@ class LogStash::Outputs::Charrington < LogStash::Outputs::Base
     end
   end
 
+  def get_schema(event)
+    if !@schema.nil?
+      @schema
+    elsif driver == "redshift"
+      case event.to_hash["app_name"]
+      when "Web App"
+        "dea_webapp"
+      else
+        "dea_mobileapp"
+      end
+    else
+      ""
+    end
+  end
+
   def get_connection
     connection = @pool.getConnection
   rescue => e

data/lib/logstash/outputs/charrington/{alter_table.rb → alter_postgres_table.rb}

@@ -1,21 +1,22 @@
 require File.join(File.dirname(__FILE__), "service")
 
 module Charrington
-  class AlterTable
+  class AlterPostgresTable
     # This service will add columns to an existing table dynamically based on finding new keys in the JSON structure.
     # This is potentially called from Insert when an insert fails.
 
     include Service
-    attr_reader :connection, :event, :table_name, :columns
+    attr_reader :connection, :event, :table_name, :columns, :schema
     attr_accessor :column_types
 
     Error = Class.new(StandardError)
     AlterFailed = Class.new(Error)
 
-    def initialize(connection, event, table_name, columns)
+    def initialize(connection, event, schema, table_name, columns)
       @connection = connection
       @event = event.to_hash
       @table_name = table_name
+      @schema = schema
       @columns = columns
       @column_types = []
     end
@@ -33,7 +34,7 @@ module Charrington
     private
 
     def alter_table
-      execute("ALTER TABLE IF EXISTS #{table_name} #{columns_fragment}")
+      execute("ALTER TABLE IF EXISTS #{schema}#{table_name} #{columns_fragment}")
     end
 
     def columns_fragment
@@ -65,7 +66,7 @@ module Charrington
     end
 
     def current_table_columns
-      sql = "SELECT * FROM #{table_name} LIMIT 1;"
+      sql = "SELECT * FROM #{schema}#{table_name} LIMIT 1;"
       stmt, rs = executeQuery(prep_sql(sql))
       meta_data = rs.getMetaData()
       stmt.close unless stmt.nil?

data/lib/logstash/outputs/charrington/alter_redshift_table.rb

@@ -0,0 +1,109 @@
+require File.join(File.dirname(__FILE__), "service")
+
+module Charrington
+  class AlterRedshiftTable
+    # This service will add columns to an existing table dynamically based on finding new keys in the JSON structure.
+    # This is potentially called from Insert when an insert fails.
+
+    include Service
+    attr_reader :connection, :event, :table_name, :columns, :schema
+    attr_accessor :column_types
+
+    Error = Class.new(StandardError)
+    AlterFailed = Class.new(Error)
+
+    def initialize(connection, event, schema, table_name, columns)
+      @connection = connection
+      @event = event.to_hash
+      @schema = schema
+      @table_name = table_name
+      @columns = columns
+      @column_types = []
+    end
+
+    def call
+      set_column_types
+      alter_table
+      true
+    rescue => e
+      raise AlterFailed, e.message
+    ensure
+      @column_types.clear if @column_types.is_a? Array
+    end
+
+    private
+
+    def alter_table
+      execute("ALTER TABLE #{schema}#{table_name} #{columns_fragment}")
+    end
+
+    def columns_fragment
+      column_types.map do |column|
+        "ADD COLUMN #{column}"
+      end.join(",")
+    end
+
+    def set_column_types
+      (columns - current_table_columns).each_with_index do |key, idx|
+
+        case event[key]
+        when Time, LogStash::Timestamp
+          column_types << "#{key} TIMESTAMP"
+        when Date
+          column_types << "#{key} DATE"
+        when Integer
+          column_types << "#{key} BIGINT"
+        when BigDecimal
+          column_types << "#{key} DECIMAL"
+        when Float
+          column_types << "#{key} DOUBLE PRECISION"
+        when true, false
+          column_types << "#{key} BOOLEAN"
+        else
+          column_types << "#{key} VARCHAR(512)"
+        end
+      end
+    end
+
+    def current_table_columns
+      sql = "SELECT * FROM #{schema}#{table_name} LIMIT 1;"
+      stmt, rs = executeQuery(prep_sql(sql))
+      meta_data = rs.getMetaData()
+      stmt.close unless stmt.nil?
+      column_count = meta_data.getColumnCount()
+      (1..column_count).map {|i| meta_data.getColumnName(i) }
+    ensure
+      stmt.close unless stmt.nil?
+    end
+
+    def execute(sql)
+      stmt = connection.prepareStatement(prep_sql(sql))
+      stmt.execute()
+    rescue Java::JavaSql::SQLException => e
+      puts "Alter Redshift SQLException: #{e.message}"
+    rescue => e
+      puts "Alter Redshift Unknown exception: #{e.message}"
+    ensure
+      stmt.close unless stmt.nil?
+    end
+
+    def executeQuery(sql)
+      stmt = connection.createStatement()
+      # only close the statement if something goes wrong
+      # otherwise, the caller is responsible for closing the
+      # statement when they are doen with the result set
+      return stmt, stmt.executeQuery(prep_sql(sql))
+    rescue Java::JavaSql::SQLException => e
+      puts "execute query SQLException: #{e.message}"
+      stmt.close unless stmt.nil?
+      # @logger.error("#{e.message}")
+    rescue => e
+      puts "execute query Unknown exception: #{e.message}"
+      stmt.close unless stmt.nil?
+    end
+
+    def prep_sql(sql)
+      sql.gsub(/\s+/, " ").strip
+    end
+  end
+end

data/lib/logstash/outputs/charrington/{create_table.rb → create_postgres_table.rb}

@@ -1,21 +1,22 @@
 require File.join(File.dirname(__FILE__), "service")
 
 module Charrington
-  class CreateTable
+  class CreatePostgresTable
     # This service will create a table dynamically based on the JSON structure.
     # This is potentially called from Insert when an insert fails.
 
     include Service
-    attr_reader :connection, :event, :table_name, :columns
+    attr_reader :connection, :event, :table_name, :columns, :schema
     attr_accessor :column_types
 
     Error = Class.new(StandardError)
     CreateFailed = Class.new(Error)
 
-    def initialize(connection, event, table_name, columns)
+    def initialize(connection, event, schema, table_name, columns)
       @connection = connection
       @event = event.to_hash
       @table_name = table_name
+      @schema = schema
       @columns = columns
       @column_types = initial_columns
     end
@@ -59,7 +60,7 @@ module Charrington
     end
 
     def create_table
-      execute("CREATE TABLE IF NOT EXISTS #{table_name} (#{column_types.join(', ')})")
+      execute("CREATE TABLE IF NOT EXISTS #{schema}#{table_name} (#{column_types.join(', ')})")
     end
 
     def execute(sql)

data/lib/logstash/outputs/charrington/create_redshift_table.rb

@@ -0,0 +1,88 @@
+require File.join(File.dirname(__FILE__), "service")
+
+module Charrington
+  class CreateRedshiftTable
+    # This service will create a table dynamically based on the JSON structure.
+    # This is potentially called from Insert when an insert fails.
+
+    include Service
+    attr_reader :connection, :event, :table_name, :columns, :schema
+    attr_accessor :column_types
+
+    Error = Class.new(StandardError)
+    CreateFailed = Class.new(Error)
+
+    def initialize(connection, event, schema, table_name, columns)
+      @connection = connection
+      @event = event.to_hash
+      @schema = schema
+      @table_name = table_name
+      @columns = columns
+      @column_types = initial_columns
+    end
+
+    def call
+      set_column_types
+      create_table
+      true
+    rescue => e
+      raise CreateFailed, e.message
+    ensure
+      @column_types.clear if @column_types.is_a? Array
+    end
+
+    private
+
+    # https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_TABLE_NEW.html
+    def set_column_types
+      columns.each do |column|
+        if column == "id"
+          column_types << "#{column} VARCHAR(512) NOT NULL distkey CONSTRAINT #{table_name}_pkey primary key"
+          puts "ADD ID column #{column_types}"
+          next
+        end
+        case event[column]
+        when Time, LogStash::Timestamp
+          column_types << "#{column} TIMESTAMP"
+        when Date
+          column_types << "#{column} DATE"
+        when Integer
+          column_types << "#{column} BIGINT"
+        when BigDecimal
+          column_types << "#{column} DECIMAL"
+        when Float
+          column_types << "#{column} DOUBLE PRECISION"
+        when true, false
+          column_types << "#{column} BOOLEAN"
+        else
+          column_types << "#{column} VARCHAR(512)"
+        end
+      end
+    end
+
+    def initial_columns
+      [
+        'original_timestamp TIMESTAMP DEFAULT GETDATE()',
+        'received_at TIMESTAMP DEFAULT GETDATE()',
+        'timestamp TIMESTAMP DEFAULT GETDATE()',
+        'uuid_ts TIMESTAMP DEFAULT GETDATE()',
+        "uuid bigint default \"identity\"(22828367, 2, '1,1'::text)"
+      ]
+    end
+
+    def create_table
+      execute("CREATE TABLE IF NOT EXISTS #{schema}#{table_name} (#{column_types.join(', ')}) diststyle key sortkey(received_at)")
+    end
+
+    def execute(sql)
+      puts "CREATE SQL #{sql}"
+      statement = connection.prepareStatement( sql.gsub(/\s+/, " ").strip )
+      puts "CREATE SQL #{sql}"
+      statement.execute()
+    rescue Java::JavaSql::SQLException => e
+      puts "Redshift SQLException: #{e.message}"
+    ensure
+      statement.close unless statement.nil?
+    end
+  end
+end

data/lib/logstash/outputs/charrington/insert.rb

@@ -1,5 +1,7 @@
-require File.join(File.dirname(__FILE__), "create_table")
-require File.join(File.dirname(__FILE__), "alter_table")
+require File.join(File.dirname(__FILE__), "create_postgres_table")
+require File.join(File.dirname(__FILE__), "create_redshift_table")
+require File.join(File.dirname(__FILE__), "alter_postgres_table")
+require File.join(File.dirname(__FILE__), "alter_redshift_table")
 require File.join(File.dirname(__FILE__), "service")
 
 module Charrington
@@ -9,7 +11,7 @@ module Charrington
 
     include Service
     attr_accessor :event, :should_retry
-    attr_reader :connection, :schema, :table_name, :columns
+    attr_reader :connection, :schema, :table_name, :columns, :driver
     attr_reader :event_as_json_keyword, :enable_event_as_json_keyword
 
     Error = Class.new(StandardError)
@@ -20,35 +22,49 @@ module Charrington
     def initialize(connection, event, opts = {})
       raise EventNil, "Table name is nil" if event.nil?
       @event = event.to_hash
-
       event_name = event["event"].to_s.downcase.strip
       raise TableNameNil, "Table name is nil" if event_name.empty?
 
       @connection = connection
       @schema = opts[:schema].empty? ? '' : "#{opts[:schema]}."
-      @table_name = "#{@schema}#{event_name.gsub(/[^a-z0-9]+/, "_")}"
+
+      @table_name = "#{event_name.gsub(/[^a-z0-9]+/, "_")}"
 
       @columns = event.keys
       @should_retry = false
       @enable_event_as_json_keyword = opts[:enable_event_as_json_keyword]
       @event_as_json_keyword = opts[:event_as_json_keyword]
+      @driver = opts[:driver]
     end
 
     def call
-      stmt = connection.prepareStatement(insert_statement)
+      insert_stmt = insert_statement
+      stmt = connection.prepareStatement(insert_stmt)
       stmt = add_statement_event_params(stmt)
       stmt.execute
       should_retry
     rescue Java::OrgPostgresqlUtil::PSQLException => e
       case e.getSQLState()
       when "42P01"
-        should_retry = Charrington::CreateTable.call(connection, event, table_name, columns)
+        should_retry = Charrington::CreatePostgresTable.call(connection, event, schema, table_name, columns)
+      when "42703"
+        should_retry = Charrington::AlterPostgresTable.call(connection, event, schema, table_name, columns)
+      else
+        raise InsertFailed, "Charrington: Rescue from SQLException #{e.message}"
+      end
+      should_retry
+    rescue Java::JavaSql::SQLException => e
+      puts "catching SQLException #{e.message}"
+      case e.getSQLState()
+      when "42P01"
+        should_retry = Charrington::CreateRedshiftTable.call(connection, event, schema, table_name, columns)
       when "42703"
-        should_retry = Charrington::AlterTable.call(connection, event, table_name, columns)
+        should_retry = Charrington::AlterRedshiftTable.call(connection, event, schema, table_name, columns)
       else
         raise InsertFailed, "Charrington: Rescue from SQLException #{e.message}"
       end
       should_retry
+
     rescue => e
       raise InsertFailed, "Charrington: Rescue from SQLException #{e.message}"
     ensure
@@ -77,7 +93,7 @@ module Charrington
     end
 
     def insert_statement
-      "INSERT INTO #{table_name} #{columns_text} VALUES #{insert_values}"
+      "INSERT INTO #{schema}#{table_name} #{columns_text} VALUES #{insert_values}"
     end
 
     def prepared_statement
@@ -134,6 +150,8 @@ module Charrington
       statement.execute()
     rescue Java::OrgPostgresqlUtil::PSQLException => e
       puts "PSQLException: #{e.message}"
+    rescue Java::JavaSql::SQLException => e
+      puts "Redshift SQLException: #{e.message}"
     ensure
       statement.close unless statement.nil?
     end