logstash-codec-snowplow 0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: cbabf39fbd655f440e460086dafdb072f5adf009c20d688cb104edcb0f9c70d2
+  data.tar.gz: c71c78e4fa66d8ab25ff673deb8d3c9a66725dbfc0edae0b52c6b682b0f16725
+SHA512:
+  metadata.gz: 56cb875ad7d37040599c891ea2b4211fc5a7d89dff0d3d634611c8e61c8cb8abc560eafbb828bda61b8b03d3a97dcb397a91f844b73b4cce9990e1ac12b2892d
+  data.tar.gz: b4c5f79d1974f3cf1980b2795d2e276a06839aa2f09128bb5af8a614cfa199ffabeb3c209ff64d25a0235774619f6406711daffc95eb13e065048b56aaa10fb5

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE.md

@@ -0,0 +1,13 @@
+Copyright 2016 Active Agent AG
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+		http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,41 @@
+# Logstash Codec snowplow
+
+[![Gem Version](https://badge.fury.io/rb/logstash-codec-snowplow.svg)](http://badge.fury.io/rb/logstash-codec-snowplow) [![Build Status](https://travis-ci.org/active-agent/logstash-codec-snowplow.svg?branch=master)](https://travis-ci.org/active-agent/logstash-codec-snowplow)
+
+## Install
+
+    $ ./bin/plugin install logstash-codec-snowplow
+
+## Usage
+
+```ruby
+input {
+  kafka {
+    codec => snowplow {}
+  }
+}
+```
+
+## Install dependencies
+
+Install jruby:
+
+    $ brew install jruby
+
+Install jruby bundler:
+
+    $ jruby -S gem install bundler
+
+Install plugin dependencies:
+
+    $ jruby -S bundler install
+
+## Build & Install
+
+Build Gem:
+
+    $ gem build logstash-codec-snowplow.gemspec
+
+Install Gem:
+
+    $ ./bin/plugin install /your/local/plugin/logstash-codec-snowplow.gem

data/lib/logstash/codecs/snowplow.rb

@@ -0,0 +1,191 @@
+# encoding: utf-8
+
+require 'json'
+require 'logger'
+require 'logstash/codecs/base'
+
+# Read serialized Thrift Snowplow enriched-event records as Logstash events
+#
+# This plugin is used to serialize Logstash events as
+# Snowplow enriched-event objects.
+#
+# ==== Encoding
+#
+# This codec is for serializing individual Logstash events as Snowplow enriched-event.
+#
+#
+# ==== Decoding
+#
+# This codec is for deserializing individual Snowplow enriched-event serialized objects as Logstash events.
+#
+#
+# ==== Usage
+# Example usage with kafka input.
+#
+# [source,ruby]
+# ----------------------------------
+# input {
+#   kafka {
+#     codec => snowplow {
+#     }
+#   }
+# }
+# filter {
+#   ...
+# }
+# output {
+#   ...
+# }
+# ----------------------------------
+
+ENRICHED_EVENT = %w[
+  app_id
+  platform
+  etl_tstamp
+  collector_tstamp
+  dvce_created_tstamp
+  event
+  event_id
+  txn_id
+  name_tracker
+  v_tracker
+  v_collector
+  v_etl
+  user_id
+  user_ipaddress
+  user_fingerprint
+  domain_userid
+  domain_sessionidx
+  network_userid
+  geo_country
+  geo_region
+  geo_city
+  geo_zipcode
+  geo_latitude
+  geo_longitude
+  geo_region_name
+  ip_isp
+  ip_organization
+  ip_domain
+  ip_netspeed
+  page_url
+  page_title
+  page_referrer
+  page_urlscheme
+  page_urlhost
+  page_urlport
+  page_urlpath
+  page_urlquery
+  page_urlfragment
+  refr_urlscheme
+  refr_urlhost
+  refr_urlport
+  refr_urlpath
+  refr_urlquery
+  refr_urlfragment
+  refr_medium
+  refr_source
+  refr_term
+  mkt_medium
+  mkt_source
+  mkt_term
+  mkt_content
+  mkt_campaign
+  contexts
+  se_category
+  se_action
+  se_label
+  se_property
+  se_value
+  unstruct_event
+  tr_orderid
+  tr_affiliation
+  tr_total
+  tr_tax
+  tr_shipping
+  tr_city
+  tr_state
+  tr_country
+  ti_orderid
+  ti_sku
+  ti_name
+  ti_category
+  ti_price
+  ti_quantity
+  pp_xoffset_min
+  pp_xoffset_max
+  pp_yoffset_min
+  pp_yoffset_max
+  useragent
+  br_name
+  br_family
+  br_version
+  br_type
+  br_renderengine
+  br_lang
+  br_features_pdf
+  br_features_flash
+  br_features_java
+  br_features_director
+  br_features_quicktime
+  br_features_realplayer
+  br_features_windowsmedia
+  br_features_gears
+  br_features_silverlight
+  br_cookies
+  br_colordepth
+  br_viewwidth
+  br_viewheight
+  os_name
+  os_family
+  os_manufacturer
+  os_timezone
+  dvce_type
+  dvce_ismobile
+  dvce_screenwidth
+  dvce_screenheight
+  doc_charset
+  doc_width
+  doc_height
+  tr_currency
+  tr_total_base
+  tr_tax_base
+  tr_shipping_base
+  ti_currency
+  ti_price_base
+  base_currency
+  geo_timezone
+  mkt_clickid
+  mkt_network
+  etl_tags
+  dvce_sent_tstamp
+  refr_domain_userid
+  refr_dvce_tstamp
+  derived_contexts
+  domain_sessionid
+  derived_tstamp
+  event_vendor
+  event_name
+  event_format
+  event_version
+  event_fingerprint
+].freeze
+
+class LogStash::Codecs::Snowplow < LogStash::Codecs::Base
+  config_name 'snowplow'
+
+  public
+
+  def register
+    @logger.info('Initializing logstash Snowplow enriched-event codec')
+  end
+
+  public
+  def decode(data)
+    values = data.to_s.split("\t")
+    hash = Hash[*ENRICHED_EVENT.zip(values).flatten] if values.length == ENRICHED_EVENT.length
+    yield LogStash::Event.new(hash)
+  rescue
+    @logger.error("Fail to decode: #{data.to_s}")
+  end
+end

data/lib/thrift/event.thrift

@@ -0,0 +1,6 @@
+struct event {
+    1: required string timestamp, #timestamp as iso8601
+    2: required string version,
+    3: optional string host,
+    4: optional string message
+}

data/lib/thrift/snowplow.thrift

@@ -0,0 +1,206 @@
+struct SchemaSniffer {
+
+  // The application (site, game, app etc) this event belongs to, and the tracker platform
+  1: optional string app_id,
+  2: optional string platform,
+
+  // Date/time
+  3: optional string etl_tstamp,
+  4: optional string collector_tstamp,
+  5: optional string dvce_created_tstamp,
+
+  // Transaction (i.e. this logging event)
+  6: optional string event,
+  7: optional string event_id,
+  8: optional string txn_id,
+
+  // Versioning
+  9: optional string name_tracker,
+  10: optional string v_tracker,
+  11: optional string v_collector,
+  12: optional string v_etl,
+
+  // User and visit
+  13: optional string user_id,
+  14: optional string user_ipaddress,
+  15: optional string user_fingerprint,
+  17: optional string domain_userid,
+  18: optional i16 domain_sessionidx,
+  19: optional string network_userid,
+
+  // Location
+  20: optional string geo_country,
+  21: optional string geo_region,
+  22: optional string geo_city,
+  23: optional string geo_zipcode,
+  24: optional double geo_latitude,
+  25: optional double geo_longitude,
+  26: optional string geo_region_name,
+
+  // Other IP lookups
+  27: optional string ip_isp,
+  28: optional string ip_organization,
+  29: optional string ip_domain,
+  30: optional string ip_netspeed,
+
+  // Page
+  31: optional string page_url,
+  32: optional string page_title,
+  33: optional string page_referrer,
+
+  // Page URL components
+  34: optional string page_urlscheme,
+  35: optional string page_urlhost,
+  36: optional string page_urlport,
+  37: optional string page_urlpath,
+  38: optional string page_urlquery,
+  39: optional string page_urlfragment,
+
+  // Referrer URL components
+  40: optional string refr_urlscheme,
+  41: optional string refr_urlhost,
+  42: optional string refr_urlport,
+  43: optional string refr_urlpath,
+  44: optional string refr_urlquery,
+  45: optional string refr_urlfragment,
+
+  // Referrer details
+  46: optional string refr_medium,
+  47: optional string refr_source,
+  48: optional string refr_term,
+
+  // Marketing
+  49: optional string mkt_medium,
+  50: optional string mkt_source,
+  51: optional string mkt_term,
+  52: optional string mkt_content,
+  53: optional string mkt_campaign,
+
+  // Custom Contexts
+  54: optional string contexts,
+
+  // Structured Event
+  55: optional string se_category,
+  56: optional string se_action,
+  57: optional string se_label,
+  58: optional string se_property,
+  59: optional string se_value, // Technically should be a Double but may be rendered incorrectly by Cascading with scientific notification (which Redshift can't process)
+
+  // Unstructured Event
+  60: optional string unstruct_event,
+
+  // Ecommerce transaction (from querystring)
+  61: optional string tr_orderid,
+  62: optional string tr_affiliation,
+  63: optional string tr_total,
+  64: optional string tr_tax,
+  65: optional string tr_shipping,
+  66: optional string tr_city,
+  67: optional string tr_state,
+  68: optional string tr_country,
+
+  // Ecommerce transaction item (from querystring)
+  69: optional string ti_orderid,
+  70: optional string ti_sku,
+  71: optional string ti_name,
+  72: optional string ti_category,
+  73: optional string ti_price,
+  74: optional string ti_quantity,
+
+  // Page Pings
+  75: optional string pp_xoffset_min,
+  76: optional string pp_xoffset_max,
+  77: optional string pp_yoffset_min,
+  78: optional string pp_yoffset_max,
+
+  // User Agent
+  79: optional string useragent,
+
+  // Browser (from user-agent)
+  80: optional string br_name,
+  81: optional string br_family,
+  82: optional string br_version,
+  83: optional string br_type,
+  84: optional string br_renderengine,
+
+  // Browser (from querystring)
+  85: optional string br_lang,
+
+  // Individual feature fields for non-Hive targets (e.g. Infobright)
+  86: optional i8 br_features_pdf,
+  87: optional i8 br_features_flash,
+  88: optional i8 br_features_java,
+  89: optional i8 br_features_director,
+  90: optional i8 br_features_quicktime,
+  91: optional i8 br_features_realplayer,
+  92: optional i8 br_features_windowsmedia,
+  93: optional i8 br_features_gears,
+  94: optional i8 br_features_silverlight,
+  95: optional i8 br_cookies,
+  96: optional string br_colordepth,
+  97: optional i16 br_viewwidth,
+  98: optional i16 br_viewheight,
+
+  // OS (from user-agent)
+  99: optional string os_name,
+  100: optional string os_family,
+  101: optional string os_manufacturer,
+  102: optional string os_timezone,
+
+  // Device/Hardware (from user-agent)
+  103: optional string dvce_type,
+  104: optional string dvce_ismobile,
+
+  // Device (from querystring)
+  105: optional i16 dvce_screenwidth,
+  106: optional i16 dvce_screenheight,
+
+  // Document
+  107: optional string doc_charset,
+  108: optional i16 doc_width,
+  109: optional i16 doc_height,
+
+  // Currency
+  110: optional string tr_currency,
+  111: optional string tr_total_base,
+  112: optional string tr_tax_base,
+  113: optional string tr_shipping_base,
+  114: optional string ti_currency,
+  115: optional string ti_price_base,
+  116: optional string base_currency,
+
+  // Geolocation
+  117: optional string geo_timezone,
+
+  // Click ID
+  118: optional string mkt_clickid,
+  119: optional string mkt_network,
+
+  // ETL tags
+  120: optional string etl_tags,
+
+  // Time event was sent
+  121: optional string dvce_sent_tstamp,
+
+  // Referer
+  122: optional string refr_domain_userid,
+  123: optional string refr_dvce_tstamp,
+
+  // Derived contexts
+  124: optional string derived_contexts,
+
+  // Session ID
+  125: optional string domain_sessionid,
+
+  // Derived timestamp
+  126: optional string derived_tstamp,
+
+  // Derived event vendor/name/format/version
+  127: optional string event_vendor,
+  128: optional string event_name,
+  129: optional string event_format,
+  130: optional string event_version,
+
+  // Event fingerprint
+  131: optional string event_fingerprint
+}

data/logstash-codec-snowplow.gemspec

@@ -0,0 +1,25 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-codec-snowplow'
+  s.version = '0.3'
+  s.licenses = ['Apache-2.0']
+  s.summary = 'This example input streams a string at a definable interval.'
+  s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program'
+  s.authors = ['Robson Júnior']
+  s.email = 'bsao@cerebello.co'
+  s.homepage = 'http://cerebello.co'
+  s.require_paths = ['lib']
+
+  # Files
+  s.files = Dir['lib/**/*', '*.gemspec', '*.md', 'Gemfile']
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = {'logstash_plugin' => 'true', 'logstash_group' => 'codec'}
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
+  s.add_runtime_dependency 'logstash-codec-line'
+  s.add_development_dependency 'logstash-devutils'
+end

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: logstash-codec-snowplow
+version: !ruby/object:Gem::Version
+  version: '0.3'
+platform: ruby
+authors:
+- Robson Júnior
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2017-08-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-line
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not
+  a stand-alone program
+email: bsao@cerebello.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE.md
+- README.md
+- lib/logstash/codecs/snowplow.rb
+- lib/thrift/event.thrift
+- lib/thrift/snowplow.thrift
+- logstash-codec-snowplow.gemspec
+homepage: http://cerebello.co
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: codec
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.11
+signing_key:
+specification_version: 4
+summary: This example input streams a string at a definable interval.
+test_files: []