logstash-output-charrington 0.3.13 → 0.3.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9bb02ce087011aef5d96e4a5ea26e7151ff87d185e24b2812a24c95319c322b0
-  data.tar.gz: 6a5ea166ed113d948d766339f35f049e68bd2309e23899e25c7ae5e9345dce33
+  metadata.gz: 7bfd649565a175e374554e824e34e48bc7d3a343a6bf4c3dd3343153ef783c9e
+  data.tar.gz: 6e57f8174acea5086848184d69b24a98b7416f694ab4d44a8fab18077aa912cb
 SHA512:
-  metadata.gz: 1a963e2a2c2a6e182fcb63c202b857e55814e7dc4cf7983bc7e34e3f3af7141ecb9c7bd656c52d15559905f978993cc9442e33ce29f927e0140e0fc2553fadf1
-  data.tar.gz: e257a03e1a42b74c14b01b0a4cd5e67ad285e8a8de9e3a3b7f5bd934c8c97c228149808e1d429bf17d3f1ee3658ba9b84bc326a16b163f0212a1a9f9abe14c28
+  metadata.gz: 102b9c2e207472fe275ec8528266779f33962f18091a6a72aa71d149c5b08ec6ea58546872be37b45b06ac110cbcd12501edccdd0c9c399349cabd679d6dda35
+  data.tar.gz: bd2a4119733e409a0ca980acb5662688ecdc2b6e6255f101f263caea39ae0429fcc7224cd449af390f918a2ec483dcca421e7d227cafdf5c804ae6a278b38b9a

data/README.md

@@ -85,6 +85,43 @@ Sometimes the embedded postgres JAR can leave dangling open processes. This is o
 and the test output contains a stacktrace that starts with `[2019-06-21T15:18:06,734][ERROR][ru.yandex.qatools.embed.postgresql.PostgresProcess] Failed to read PID file (File '/var/folders/...`.
 To resolve this issue, check for dangling processes using `lsof -PiTCP -sTCP:LISTEN | grep postgres`, then you can kill the process ids
 
+#### Tracks Table
+**Create Statment for Driver is Postgres and Transform is Redshift**
+ * See `@@redshift_tracks_columns` in [insert.rb](./lib/logstash/outputs/charrington/insert.rb) for column list
+```sql
+create table <schema>.tracks
+(
+	id varchar(512) not null constraint tracks_pkey primary key,
+	anonymous_id varchar(512),
+	app_name varchar(512),
+	context_campaign_content varchar(512)
+	context_campaign_medium varchar(512),
+	context_campaign_name varchar(512),
+	context_campaign_source varchar(512),
+	context_ip varchar(512),
+	context_library_name varchar(512),
+	context_library_version varchar(512),
+	context_page_path varchar(512),
+	context_page_referrer varchar(512),
+	context_page_search varchar(512),
+	context_page_title varchar(512),
+	context_page_url varchar(512),
+	context_user_agent varchar(512),
+	event varchar(512),
+	event_text varchar(512),
+	original_timestamp timestamp,
+	received_at timestamp,
+	segment_dedupe_id varchar(512),
+	sent_at timestamp,
+	timestamp timestamp,
+	user_id varchar(512),
+	user_uid varchar(512),
+	uuid bigint,
+	uuid_ts timestamp default ('now'::text)::timestamp without time zone,
+)
+
+```
+
 #### Resources
 * [logstash-devutils](https://rubygems.org/gems/logstash-devutils) gem with logstash helper methods and tools
     * [spec_helper.rb source](https://github.com/elastic/logstash-devutils/blob/master/lib/logstash/devutils/rspec/spec_helper.rb)  

data/lib/logstash/outputs/charrington/alter_postgres_table.rb

@@ -70,7 +70,6 @@ module Charrington
       sql = "SELECT * FROM #{schema}#{table_name} LIMIT 1;"
       stmt, rs = executeQuery(prep_sql(sql))
       meta_data = rs.getMetaData()
-      stmt.close unless stmt.nil?
       column_count = meta_data.getColumnCount()
       (1..column_count).map {|i| meta_data.getColumnName(i) }
     ensure
@@ -81,8 +80,7 @@ module Charrington
       stmt = connection.prepareStatement(prep_sql(sql))
       stmt.execute()
     rescue Java::OrgPostgresqlUtil::PSQLException => e
-      puts "PSQLException: #{e.message}"
-      self.logger.info "PSQLException: #{e.message}"
+      self.logger.error "PSQLException: #{e.message}"
     ensure
       stmt.close unless stmt.nil?
     end

data/lib/logstash/outputs/charrington/alter_redshift_table.rb

@@ -91,13 +91,11 @@ module Charrington
       stmt = connection.prepareStatement(prep_sql(sql))
       stmt.execute()
     rescue Java::JavaSql::SQLException => e
-      puts "Alter Redshift SQLException: #{e.message}"
-      self.logger.info "Alter Redshift SQLException: #{e.message}, with SQL: #{sql}"
+      self.logger.error "Alter Redshift SQLException: #{e.message}, with SQL: #{sql}"
     rescue => e
-      puts "Alter Redshift Unknown exception: #{e.message}, with SQL: #{sql}"
-      self.logger.info "Alter Redshift Unknown exception: #{e.message}"
+      self.logger.error "Alter Redshift Unknown exception: #{e.message}, with SQL: #{sql}"
     ensure
-      self.logger.info "Within ensure block of execute in alter_redshift_table.rb and value of stmt.nil?: #{stmt.nil?}"
+      self.logger.error "Within ensure block of execute in alter_redshift_table.rb and value of stmt.nil?: #{stmt.nil?}"
       stmt.close unless stmt.nil?
     end
 

data/lib/logstash/outputs/charrington/create_postgres_table.rb

@@ -85,8 +85,7 @@ module Charrington
       statement = connection.prepareStatement( sql.gsub(/\s+/, " ").strip )
       statement.execute()
     rescue Java::OrgPostgresqlUtil::PSQLException => e
-      puts "PSQLException: #{e.message}"
-      self.logger.info "PSQLException: #{e.message}"
+      self.logger.error "PSQLException: #{e.message} sql=#{sql}"
     ensure
       statement.close unless statement.nil?
     end

data/lib/logstash/outputs/charrington/create_redshift_table.rb

@@ -84,7 +84,6 @@ module Charrington
       statement = connection.prepareStatement( sql.gsub(/\s+/, " ").strip )
       statement.execute()
     rescue Java::JavaSql::SQLException => e
-      puts "Redshift SQLException: #{e.message}"
       self.logger.info "Redshift SQLException: #{e.message}, with SQL: #{sql}"
     ensure
       self.logger.info "Within ensure block of create_redshift_table.rb and value of statement.nil?: #{statement.nil?}"

data/lib/logstash/outputs/charrington/insert.rb

@@ -3,6 +3,7 @@ require File.join(File.dirname(__FILE__), "create_redshift_table")
 require File.join(File.dirname(__FILE__), "alter_postgres_table")
 require File.join(File.dirname(__FILE__), "alter_redshift_table")
 require File.join(File.dirname(__FILE__), "service")
+require 'time'
 
 module Charrington
   class Insert
@@ -12,8 +13,11 @@ module Charrington
     include Service
     include LogStash::Util::Loggable
     attr_accessor :event, :should_retry
-    attr_reader :connection, :schema, :table_name, :columns, :driver, :opts
+    attr_reader :connection, :schema, :table_name, :columns, :driver, :opts, :transformer, :tracks
     attr_reader :event_as_json_keyword, :enable_event_as_json_keyword
+    # TODO create a current_table_columns (alter_postgres_table.rb) query on the tracks table to get the current columns
+    @@redshift_tracks_columns = %w(id app_name received_at uuid uuid_ts anonymous_id context_ip context_library_name context_library_version context_page_path context_page_referrer context_page_title context_page_url context_user_agent event event_text original_timestamp sent_at timestamp user_id user_uid context_campaign_medium context_campaign_name context_page_search context_campaign_source segment_dedupe_id context_campaign_content)
+    @@postgres_tracks_columns = %w(anonymous_user app_name event published_at session_ip session_library_name session_library_version session_page_path session_page_referrer session_page_search session_page_title session_page_url session_user_agent user_id user_uid)
 
     Error = Class.new(StandardError)
     EventNil = Class.new(Error)
@@ -22,7 +26,9 @@ module Charrington
 
     def initialize(connection, event, opts = {})
       raise EventNil, "Table name is nil" if event.nil?
+      @transformer = opts[:transformer]
       @event = event.to_hash
+      @tracks = create_tracks(@event)
       event_name = event["event"].to_s.strip
       raise TableNameNil, "Table name is nil" if event_name.empty?
 
@@ -41,13 +47,16 @@ module Charrington
 
     def call
       self.logger.info "Attempting insert into table name: #{table_name}"
-      insert_stmt = insert_statement
+      insert_sql = insert_event_statement
+      insert_stmt = connection.prepareStatement(insert_sql)
       self.logger.info "Insert statement passed into prepareStatement is: #{insert_stmt}"
+      insert_stmt = add_statement_event_params(insert_stmt, event)
+      self.logger.info "Insert statement to be run is: #{insert_stmt.toString}"
+      insert_stmt.execute
+
+      self.logger.info "Attempting insert into tracks table"
+      do_tracks_insert
 
-      stmt = connection.prepareStatement(insert_stmt)
-      stmt = add_statement_event_params(stmt)
-      self.logger.info "Insert statement to be run is: #{insert_stmt}"
-      stmt.execute
       should_retry
     rescue Java::JavaSql::SQLException => e
       case e.getSQLState()
@@ -58,65 +67,112 @@ module Charrington
         self.logger.info "Received Java::JavaSql::SQLException with error sql state of 42703, moving to alter table"
         should_retry = alter_table
       else
-        raise InsertFailed, "Charrington: Rescue from SQLException #{e.message}"
+        raise InsertFailed, "SQLException (Charrington:Insert) #{e.message} #{insert_sql}"
       end
       should_retry
 
     rescue => e
-      raise InsertFailed, "Charrington: Rescue from SQLException #{e.message}"
+      raise InsertFailed, "SQLException (Charrington:Insert) #{e.message} #{insert_sql}"
     ensure
-      stmt.close unless stmt.nil?
+      insert_stmt.close unless insert_stmt.nil?
       cleanup
     end
 
+    def do_tracks_insert
+      tracks_sql = insert_tracks_statement
+      tracks_stmt = connection.prepareStatement(tracks_sql)
+      tracks_stmt = add_statement_event_params(tracks_stmt, tracks)
+      self.logger.info "Insert tracks statment to be run: #{tracks_stmt.toString}"
+      tracks_stmt.execute
+    rescue Java::JavaSql::SQLException => e
+      self.logger.error("SQLException (Charrington:Insert) Insert tracks entry failed. #{e.message} #{tracks_sql}")
+    ensure
+      tracks_stmt.close unless tracks_stmt.nil?
+    end
+
     private
 
+    def tracks_columns
+      is_redshift_transform? ? @@redshift_tracks_columns : @@postgres_tracks_columns
+    end
+
+    def is_redshift?
+      driver == 'redshift'
+    end
+
+    def is_postgres?
+      driver == 'postgres'
+    end
+
+    def is_postgres_transform?
+      transformer == 'postgres'
+    end
+
+    def is_redshift_transform?
+      transformer == 'redshift'
+    end
+
     def create_table
-      if driver == "postgres"
+      if is_postgres?
         Charrington::CreatePostgresTable.call(connection, event, schema, table_name, columns, opts)
-      elsif driver == "redshift"
+      elsif is_redshift?
         Charrington::CreateRedshiftTable.call(connection, event, schema, table_name, columns, opts)
       end
     end
 
     def alter_table
-      if driver == "postgres"
+      if is_postgres?
         Charrington::AlterPostgresTable.call(connection, event, schema, table_name, columns)
-      elsif driver == "redshift"
+      elsif is_redshift?
         Charrington::AlterRedshiftTable.call(connection, event, schema, table_name, columns)
       end
     end
 
     def cleanup
       @columns.clear if clearable(@columns)
+      @tracks.clear if clearable(@tracks)
     end
 
     ### Set Variables
+    def create_tracks(event)
+      tracks = event.clone
+      tracks.keys.each do |key|
+        tracks.delete(key) unless tracks_columns.include?(key)
+      end
+      tracks
+    end
 
     def columns_text
       @columns_text ||= arr_to_csv(columns)
     end
 
-    def value_placeholders
+    def tracks_columns_text
+      @tracks_columns_text ||= arr_to_csv(tracks.keys)
+    end
+
+
+    def value_placeholders(columns)
       ('?' * columns.length).split('')
     end
 
-    def insert_values
-      arr_to_csv(value_placeholders)
+    def placeholder_text(columns)
+      arr_to_csv(value_placeholders(columns))
     end
 
-    def insert_statement
-      "INSERT INTO #{schema}#{table_name} #{columns_text} VALUES #{insert_values}"
+    def insert_event_statement
+      "INSERT INTO #{schema}#{table_name} #{columns_text} VALUES #{placeholder_text(columns)}"
     end
 
-    def prepared_statement
-      columns.map { |column| "[#{column}]" }
+    def insert_tracks_statement
+      "INSERT INTO #{schema}tracks #{tracks_columns_text} VALUES #{placeholder_text(tracks.keys)}"
     end
 
-    def add_statement_event_params(stmt)
-      columns.each_with_index do |key, idx|
+    def add_statement_event_params(stmt, map)
+      values = []
+      map.keys.each_with_index do |key, idx|
         pos = idx + 1
-        value = event[key]
+        value = map[key]
+        values << value
 
         if key == 'published_at' || key == 'sent_at'
           format = java.text.SimpleDateFormat.new("yyyy-MM-dd'T'HH:mm:ss'Z'")
@@ -181,11 +237,9 @@ module Charrington
       statement = connection.prepareStatement( sql.gsub(/\s+/, " ").strip )
       statement.execute()
     rescue Java::OrgPostgresqlUtil::PSQLException => e
-      puts "PSQLException: #{e.message}"
-      self.logger.info "PSQLException: #{e.message}, with SQL: #{sql}"
+      self.logger.error "PSQLException: #{e.message}, with SQL: #{sql}"
     rescue Java::JavaSql::SQLException => e
-      puts "Redshift SQLException: #{e.message}"
-      self.logger.info "Redshift SQLException: #{e.message}, with SQL: #{sql}"
+      self.logger.error "Redshift SQLException: #{e.message}, with SQL: #{sql}"
     ensure
       statement.close unless statement.nil?
     end

data/logstash-output-charrington.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-charrington'
-  s.version = '0.3.13'
+  s.version = '0.3.14'
 
   s.licenses = ['Apache-2.0']
   s.homepage = 'https://gitlab.podium.com/engineering/analytics/logstash-output-charrington'

data/spec/charrington_spec_helper.rb

@@ -80,6 +80,42 @@ RSpec.shared_context 'postgres' do
     conn.close if !conn.nil? and !conn.isClosed
   end
 
+  def create_tracks_table(schema = '')
+    sql = <<-SQL
+    CREATE TABLE IF NOT EXISTS #{ schema.empty? ? schema : schema + '.'}tracks
+    (
+      id VARCHAR(512) NOT NULL CONSTRAINT tracks_pkey PRIMARY KEY,
+      anonymous_id VARCHAR(512),
+      app_name VARCHAR(512),
+      context_campaign_content VARCHAR(512),
+      context_campaign_medium VARCHAR(512),
+      context_campaign_name VARCHAR(512),
+      context_campaign_source VARCHAR(512),
+      context_ip VARCHAR(512),
+      context_library_name VARCHAR(512),
+      context_library_version VARCHAR(512),
+      context_page_path VARCHAR(512),
+      context_page_referrer VARCHAR(512),
+      context_page_search VARCHAR(512),
+      context_page_title VARCHAR(512),
+      context_page_url VARCHAR(512),
+      context_user_agent VARCHAR(512),
+      event VARCHAR(512),
+      event_text VARCHAR(512),
+      original_timestamp TIMESTAMP,
+      received_at TIMESTAMP,
+      segment_dedupe_id VARCHAR(512),
+      sent_at TIMESTAMP,
+      timestamp TIMESTAMP,
+      user_id VARCHAR(512),
+      user_uid VARCHAR(512),
+      uuid bigint,
+      uuid_ts TIMESTAMP DEFAULT ('now'::text)::TIMESTAMP without TIME ZONE
+    )
+    SQL
+    create(sql)
+  end
+
   def drop_table(table)
     execute_update("DROP TABLE IF EXISTS #{table}")
   end

data/spec/outputs/charrington_spec.rb

@@ -27,6 +27,8 @@ describe LogStash::Outputs::Charrington do
     end
 
     it 'creates a table and inserts a record' do
+      drop_table('tracks')
+      create_tracks_table
       drop_table('schemaless')
       run_pipeline
       expect(query('SELECT * FROM schemaless')).to match_array([{id: "1", app_name: "Web App", event: "schemaless", inserted_at: a_kind_of(String), :meta_type => "XML", :published_at => "2019-07-29 20:09:18"}])
@@ -90,6 +92,8 @@ describe LogStash::Outputs::Charrington do
     end
 
     it 'creates a table and inserts the first record and alters the table for the second record' do
+      drop_table('tracks')
+      create_tracks_table
       drop_table('metadata')
       run_pipeline
       expect(query('SELECT * FROM metadata')).to match_array([
@@ -123,6 +127,8 @@ describe LogStash::Outputs::Charrington do
     end
 
     it 'can insert into an existing table' do
+      drop_table('tracks')
+      create_tracks_table
       # setup pre-existing data
       drop_table('existing')
       create_table('CREATE TABLE existing (id SERIAL PRIMARY KEY, inserted_at TIMESTAMP DEFAULT NOW(), app_name VARCHAR(255))')
@@ -164,6 +170,8 @@ describe LogStash::Outputs::Charrington do
     end
 
     it 'creates a table and inserts the first record and alters the table for the second record' do
+      drop_table('tracks')
+      create_tracks_table('dea_test')
       create("CREATE SCHEMA IF NOT EXISTS #{schema}")
       drop_table("#{schema}.from_agent")
       run_pipeline
@@ -210,7 +218,7 @@ describe LogStash::Outputs::Charrington do
       <<-CONFIG
       input {
         generator {
-          message => '{"app_name": "Web App", "event": "schemaless", "meta": { "type": "XML" }, "published_at": "2019-07-29T20:09:18Z" }'
+          message => '{"app_name": "Web App", "event": "schemaless", "meta": { "type": "XML" }, "published_at": "2019-07-29T20:09:18Z", "user_id": "123", "user_uid": "456" }'
           codec => 'json'
           count => 1
         }
@@ -228,12 +236,17 @@ describe LogStash::Outputs::Charrington do
     end
 
     it 'creates a table and inserts a record with data the shape of redshift' do
+      drop_table('tracks')
+      create_tracks_table
       drop_table('schemaless')
       run_pipeline
       expect(query('SELECT * FROM schemaless')).to match_array([
-        {:anonymous_id=>"", :app_name=>"Web App", :event=>"schemaless", :event_text=>"schemaless", :id=>a_kind_of(String), :original_timestamp=>a_kind_of(String), :received_at=>a_kind_of(String), :sent_at=>"2019-07-29 20:09:18", :timestamp=>a_kind_of(String), :type=>"XML", :uuid_ts=>a_kind_of(String)}
+        {:anonymous_id=>"", :app_name=>"Web App", :event=>"schemaless", :event_text=>"schemaless", :id=>a_kind_of(String), :original_timestamp=>a_kind_of(String), :received_at=>a_kind_of(String), :sent_at=>"2019-07-29 20:09:18", :timestamp=>a_kind_of(String), :type=>"XML", :user_id=>"123", :user_uid=>"456", :uuid_ts=>a_kind_of(String)}
       ])
       expect(query('SELECT COUNT(1) FROM schemaless').first[:count]).to eq("1")
+      expect(query("SELECT * FROM tracks")).to match_array([
+        {:id=>a_kind_of(String), :anonymous_id=>"", :app_name=>"Web App", :context_campaign_content=>nil, :context_campaign_medium=>nil, :context_campaign_name=>nil, :context_campaign_source=>nil, :context_ip=>nil, :context_library_name=>nil, :context_library_version=>nil, :context_page_path=>nil, :context_page_referrer=>nil, :context_page_search=>nil, :context_page_title=>nil, :context_page_url=>nil, :context_user_agent=>nil, :event=>"schemaless", :event_text=>"schemaless", :original_timestamp=>nil, :received_at=>nil, :segment_dedupe_id=>nil, :sent_at=>a_kind_of(String), :timestamp=>nil, :user_id=>"123", :user_uid=>"456", :uuid=>nil, :uuid_ts=>a_kind_of(String)}
+      ])
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-charrington
 version: !ruby/object:Gem::Version
-  version: 0.3.13
+  version: 0.3.14
 platform: ruby
 authors:
 - dconger
@@ -182,6 +182,8 @@ files:
 - README.md
 - THANKS.md
 - lib/com/zaxxer/HikariCP/2.7.2/HikariCP-2.7.2.jar
+- lib/commons-io/commons-io/2.4/commons-io-2.4.jar
+- lib/de/flapdoodle/embed/de.flapdoodle.embed.process/2.0.2/de.flapdoodle.embed.process-2.0.2.jar
 - lib/logstash-output-charrington_jars.rb
 - lib/logstash/outputs/charrington.rb
 - lib/logstash/outputs/charrington/alter_postgres_table.rb
@@ -193,9 +195,16 @@ files:
 - lib/logstash/outputs/charrington/service.rb
 - lib/logstash/outputs/charrington/transform_postgres.rb
 - lib/logstash/outputs/charrington/transform_redshift.rb
+- lib/net/java/dev/jna/jna-platform/4.0.0/jna-platform-4.0.0.jar
+- lib/net/java/dev/jna/jna/4.0.0/jna-4.0.0.jar
+- lib/org/apache/commons/commons-compress/1.10/commons-compress-1.10.jar
+- lib/org/apache/commons/commons-lang3/3.1/commons-lang3-3.1.jar
 - lib/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar
 - lib/org/apache/logging/log4j/log4j-slf4j-impl/2.6.2/log4j-slf4j-impl-2.6.2.jar
+- lib/org/postgresql/postgresql/42.2.5/postgresql-42.2.5.jar
 - lib/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar
+- lib/ru/yandex/qatools/embed/postgresql-embedded/2.10/postgresql-embedded-2.10.jar
+- lib/ru/yandex/qatools/embed/postgresql-embedded/2.8/postgresql-embedded-2.8.jar
 - logstash-output-charrington.gemspec
 - spec/charrington_spec_helper.rb
 - spec/logstash-output-charrington_test_jars.rb