logstash-codec-snowplow 0.4 → 0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 735f80f0427aa930ac4d0a1222cdfb8466513011
-  data.tar.gz: a1d55fd670e17f55ca995a84fd5bc9d5207bbd66
+  metadata.gz: 90da0028f8f79c9b026078966eef30d3930196b7
+  data.tar.gz: 21dd115d541c8efef29524b306fef1124b5d5345
 SHA512:
-  metadata.gz: 25a549cc5cfe8299dd82c637736fd3c5220946b2c480089881003395d33e200dd5ac1dc2543e6063d5b511d7b9f9a3a295ead519a9e36394ce755f0da4650bf7
-  data.tar.gz: c561c0b8d91ac8878616335b62a8ec0c6a316cff77a6edcaf2164e6709b79a1d4310d9bea30f3ec413c2714fa53f989e6e1a0adecf2a09efc67d3cee723c817f
+  metadata.gz: fc7c35086824551cadbcaa3f6eeda78f3c80706ed659278bf1c3505f84383cae494c1c25c0703fcc0b34d2aad733eb90bbc0ef70c138bb17a553029aca2f9a43
+  data.tar.gz: 8eaf58bd27921ccb2d86c273914394f1e7526c7bc58459e8907b9f09ff8b42781975e9153dc899a42cbe7612e2ce4bd83c04b8b428dd4e42fd6e35549cb9a09f

data/Gemfile

@@ -1,2 +1,4 @@
 source 'https://rubygems.org'
 gemspec
+gem 'rspec', :require => false, :group => :test
+gem 'simplecov', :require => false, :group => :test

data/lib/logstash/codecs/snowplow.rb

@@ -4,6 +4,7 @@ require 'json'
 require 'logger'
 require 'logstash/codecs/base'
 require 'logstash/namespace'
+require 'logstash/serializer'
 
 # Read serialized Thrift Snowplow enriched-event records as Logstash events
 #
@@ -39,156 +40,20 @@ require 'logstash/namespace'
 # }
 # ----------------------------------
 
-ENRICHED_EVENT = %w[
-  app_id
-  platform
-  etl_tstamp
-  collector_tstamp
-  dvce_created_tstamp
-  event
-  event_id
-  txn_id
-  name_tracker
-  v_tracker
-  v_collector
-  v_etl
-  user_id
-  user_ipaddress
-  user_fingerprint
-  domain_userid
-  domain_sessionidx
-  network_userid
-  geo_country
-  geo_region
-  geo_city
-  geo_zipcode
-  geo_latitude
-  geo_longitude
-  geo_region_name
-  ip_isp
-  ip_organization
-  ip_domain
-  ip_netspeed
-  page_url
-  page_title
-  page_referrer
-  page_urlscheme
-  page_urlhost
-  page_urlport
-  page_urlpath
-  page_urlquery
-  page_urlfragment
-  refr_urlscheme
-  refr_urlhost
-  refr_urlport
-  refr_urlpath
-  refr_urlquery
-  refr_urlfragment
-  refr_medium
-  refr_source
-  refr_term
-  mkt_medium
-  mkt_source
-  mkt_term
-  mkt_content
-  mkt_campaign
-  contexts
-  se_category
-  se_action
-  se_label
-  se_property
-  se_value
-  unstruct_event
-  tr_orderid
-  tr_affiliation
-  tr_total
-  tr_tax
-  tr_shipping
-  tr_city
-  tr_state
-  tr_country
-  ti_orderid
-  ti_sku
-  ti_name
-  ti_category
-  ti_price
-  ti_quantity
-  pp_xoffset_min
-  pp_xoffset_max
-  pp_yoffset_min
-  pp_yoffset_max
-  useragent
-  br_name
-  br_family
-  br_version
-  br_type
-  br_renderengine
-  br_lang
-  br_features_pdf
-  br_features_flash
-  br_features_java
-  br_features_director
-  br_features_quicktime
-  br_features_realplayer
-  br_features_windowsmedia
-  br_features_gears
-  br_features_silverlight
-  br_cookies
-  br_colordepth
-  br_viewwidth
-  br_viewheight
-  os_name
-  os_family
-  os_manufacturer
-  os_timezone
-  dvce_type
-  dvce_ismobile
-  dvce_screenwidth
-  dvce_screenheight
-  doc_charset
-  doc_width
-  doc_height
-  tr_currency
-  tr_total_base
-  tr_tax_base
-  tr_shipping_base
-  ti_currency
-  ti_price_base
-  base_currency
-  geo_timezone
-  mkt_clickid
-  mkt_network
-  etl_tags
-  dvce_sent_tstamp
-  refr_domain_userid
-  refr_dvce_tstamp
-  derived_contexts
-  domain_sessionid
-  derived_tstamp
-  event_vendor
-  event_name
-  event_format
-  event_version
-  event_fingerprint
-].freeze
-
 class LogStash::Codecs::Snowplow < LogStash::Codecs::Base
   config_name 'snowplow'
+  config :json_fields, validate: :array, default: %w[contexts derived_contexts se_property unstruct_event], required: true
 
   public
-
   def register
-    @logger.info('Initializing logstash Snowplow enriched-event codec')
+    @logger.info("Initializing logstash snowplow codec: json fields: #{@json_fields}!")
   end
 
   public
   def decode(data)
-    begin
-      values = data.to_s.split("\t")
-      hash = Hash[*ENRICHED_EVENT.zip(values).flatten] if values.length == ENRICHED_EVENT.length
-      yield LogStash::Event.new(hash)
-    rescue
-      @logger.error("Fail to decode: #{data.to_s}")
-    end
+    event = Serializer.deserialize(data, @json_fields)
+    yield LogStash::Event.new(event)
+  rescue
+    @logger.error("Fail to decode: #{data}")
   end
-end
+end

data/lib/logstash/serializer.rb

@@ -0,0 +1,161 @@
+# encoding: utf-8
+
+require 'json'
+
+ENRICHED_EVENT = %w[
+  app_id
+  platform
+  etl_tstamp
+  collector_tstamp
+  dvce_created_tstamp
+  event
+  event_id
+  txn_id
+  name_tracker
+  v_tracker
+  v_collector
+  v_etl
+  user_id
+  user_ipaddress
+  user_fingerprint
+  domain_userid
+  domain_sessionidx
+  network_userid
+  geo_country
+  geo_region
+  geo_city
+  geo_zipcode
+  geo_latitude
+  geo_longitude
+  geo_region_name
+  ip_isp
+  ip_organization
+  ip_domain
+  ip_netspeed
+  page_url
+  page_title
+  page_referrer
+  page_urlscheme
+  page_urlhost
+  page_urlport
+  page_urlpath
+  page_urlquery
+  page_urlfragment
+  refr_urlscheme
+  refr_urlhost
+  refr_urlport
+  refr_urlpath
+  refr_urlquery
+  refr_urlfragment
+  refr_medium
+  refr_source
+  refr_term
+  mkt_medium
+  mkt_source
+  mkt_term
+  mkt_content
+  mkt_campaign
+  contexts
+  se_category
+  se_action
+  se_label
+  se_property
+  se_value
+  unstruct_event
+  tr_orderid
+  tr_affiliation
+  tr_total
+  tr_tax
+  tr_shipping
+  tr_city
+  tr_state
+  tr_country
+  ti_orderid
+  ti_sku
+  ti_name
+  ti_category
+  ti_price
+  ti_quantity
+  pp_xoffset_min
+  pp_xoffset_max
+  pp_yoffset_min
+  pp_yoffset_max
+  useragent
+  br_name
+  br_family
+  br_version
+  br_type
+  br_renderengine
+  br_lang
+  br_features_pdf
+  br_features_flash
+  br_features_java
+  br_features_director
+  br_features_quicktime
+  br_features_realplayer
+  br_features_windowsmedia
+  br_features_gears
+  br_features_silverlight
+  br_cookies
+  br_colordepth
+  br_viewwidth
+  br_viewheight
+  os_name
+  os_family
+  os_manufacturer
+  os_timezone
+  dvce_type
+  dvce_ismobile
+  dvce_screenwidth
+  dvce_screenheight
+  doc_charset
+  doc_width
+  doc_height
+  tr_currency
+  tr_total_base
+  tr_tax_base
+  tr_shipping_base
+  ti_currency
+  ti_price_base
+  base_currency
+  geo_timezone
+  mkt_clickid
+  mkt_network
+  etl_tags
+  dvce_sent_tstamp
+  refr_domain_userid
+  refr_dvce_tstamp
+  derived_contexts
+  domain_sessionid
+  derived_tstamp
+  event_vendor
+  event_name
+  event_format
+  event_version
+  event_fingerprint
+  true_tstamp
+].freeze
+
+class Serializer
+  # @param [String] event
+  # @param [Array] json_fields
+  # @return [Hash]
+  def self.deserialize(event, json_fields)
+    event_values = event.to_s.split("\t")
+    event = Hash[*ENRICHED_EVENT.zip(event_values).flatten]
+    json_fields.each do |field|
+      event = parse(event, field) if event.has_key? field
+    end
+    event
+  end
+
+  # @param [Hash] event
+  # @param [String] field
+  # @return [Hash]
+  def self.parse(event, field)
+    event[field] = JSON.parse(event[field])
+    return event
+  rescue JSON::ParserError => _
+    return event
+  end
+end

data/logstash-codec-snowplow.gemspec

@@ -1,8 +1,8 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-codec-snowplow'
-  s.version = '0.4'
+  s.version = '0.5'
   s.licenses = ['Apache-2.0']
-  s.summary = 'This example input streams a string at a definable interval.'
+  s.summary = 'This codec gets a enrich snowplow event and generates a logstash event.'
   s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program'
   s.authors = ['Robson Júnior']
   s.email = 'bsao@cerebello.co'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-snowplow
 version: !ruby/object:Gem::Version
-  version: '0.4'
+  version: '0.5'
 platform: ruby
 authors:
 - Robson Júnior
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-16 00:00:00.000000000 Z
+date: 2017-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -62,6 +62,7 @@ files:
 - LICENSE.md
 - README.md
 - lib/logstash/codecs/snowplow.rb
+- lib/logstash/serializer.rb
 - lib/thrift/event.thrift
 - lib/thrift/snowplow.thrift
 - logstash-codec-snowplow.gemspec
@@ -90,5 +91,5 @@ rubyforge_project:
 rubygems_version: 2.6.8
 signing_key:
 specification_version: 4
-summary: This example input streams a string at a definable interval.
+summary: This codec gets a enrich snowplow event and generates a logstash event.
 test_files: []