fluent-plugin-mixpanel-enchanced 0.0.10

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 27467730f8f9dcfa247c004aa9ce3e0ca374f993
+  data.tar.gz: 71c01ebc5f14ee468f6685c420db82bbd2cb3d64
+SHA512:
+  metadata.gz: 12c4377d98a3a4b57d75583fa8689fd7c0a79706a2d6c035bf01f887b9b392d655f4da36fcc445eec2aa1c66eeea86eed52011d26d928cdec55e091399935e9f
+  data.tar.gz: 9be23b64240e8e057885519a656a4cb2bd09f095f09ce8210705282702b23cbcbfa8dc355ebc2b2097d061c0ccb00864bd4516c4de91467c40c1d3fa218dc492

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.idea/

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - 2.1.9
+  - 2.2.5
+  - 2.3.1

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-mixpanel.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright (c) 2014- Kazuyuki Honda
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,172 @@
+# fluent-plugin-mixpanel
+
+[![Build Status](https://travis-ci.org/hakobera/fluent-plugin-mixpanel.png?branch=master)](https://travis-ci.org/hakobera/fluent-plugin-mixpanel)
+
+**CAUTION** This plugin does not support Ruby < 2.0.
+
+## Component
+
+### MixpanelOutput
+
+[Fluentd](http://fluentd.org) plugin to send event track data to [mixpanel](https://mixpanel.com).
+
+### HttpMixpanelInput
+
+[Fluentd](http://fluentd.org) plugin to integrate [mixpanel javascript libraries](https://mixpanel.com/docs/integration-libraries/javascript).
+
+## Installation
+
+Install with gem or fluent-gem command as:
+
+```
+# for fluentd
+$ gem install fluent-plugin-mixpanel
+
+# for td-agent
+$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-mixpanel
+```
+
+## Configuration
+
+### MixpanelOutput
+
+MixpanelOutput needs mixpanel's `project_token`, that can get from your mixpanel project settings.
+
+#### Use distinct_id_key and event_key
+
+You should also specify property key name by `distinct_id_key` and `event_key`.
+
+```
+<match output.mixpanel.*>
+  type mixpanel
+  project_token YOUR_PROJECT_TOKEN
+  distinct_id_key user_id
+  event_key event_name
+</match>
+```
+
+If record like this:
+
+```rb
+{ user_id: "123", event_name: "event1", key1: "value1", key2: "value2" }
+```
+
+above settings send to the following data to mixpanel, using [mixpanel-ruby](https://github.com/mixpanel/mixpanel-ruby) gem.
+
+```rb
+tracker = Mixpanel::Tracker.new(YOUR_PROJECT_TOKEN)
+tracker.track("123", "event1", { key1: "value1", key2: "value2" })
+```
+
+#### Use distinct_id_key and event_map_tag
+
+You can use tag name as event name like this. (see additional tag manipulations options below)
+
+#### Discarding mixpanel errors
+
+When delivering events to Mixpanel, Fluent creates a chunk of messages to send. By default, if one event fails to send to Mixpanel, all messages in that chunk are requeued for delivery. Enabling `discard_event_on_send_error` allows you to ignore single delivery failures. The event is logged via `info`, including the record being dropped.
+
+```
+<match output.mixpanel.*>
+  ...
+  discard_event_on_send_error true
+  ...
+</match>
+```
+
+##PLEASE NOTE (breaking api change in a future release)
+
+The api for remove_tag_prefix will be changing in a future release. There is currently a boolean option,
+use_legacy_prefix_behavior, which will ensure legacy behavior is maintained until that time. Eventually this option will go away
+as well and the new behavior will be the only way. The difference is pretty simple, the '.' in the prefix needs to be specified.
+This change allows this plugin to use Fluet's mixin and unifies syntax across plugins. Currently, use_legacy_prefix_behavior
+defaults to true, which will work either way, but eventually you will need to specify the '.' in your prefix. Again, use_legacy_prefix_behavior simply removes any '.' along with the specified prefix and will behave properly even after you change your configs
+to be current as seen below. You do not need to set this option.
+
+```
+<match output.mixpanel.*>
+  type mixpanel
+  project_token YOUR_PROJECT_TOKEN
+  distinct_id_key user_id
+  remove_tag_prefix output.mixpanel.
+  event_map_tag true
+</match>
+```
+
+If tag name is `output.mixpanel.event1` and record like this:
+
+```rb
+{ user_id: "123", key1: "value1", key2: "value2" }
+```
+
+above settings send to the following data to mixpanel, using [mixpanel-ruby](https://github.com/mixpanel/mixpanel-ruby) gem.
+
+```rb
+tracker = Mixpanel::Tracker.new(YOUR_PROJECT_TOKEN)
+tracker.track("123", "event1", { key1: "value1", key2: "value2" })
+```
+
+#### Use the import method to post instead of track
+
+You can use tag name as event name like this.
+
+```
+<match output.mixpanel.*>
+  type mixpanel
+  project_token YOUR_PROJECT_TOKEN
+  distinct_id_key user_id
+  remove_tag_prefix output.mixpanel.
+  event_map_tag true
+  use_import true
+  api_key YOUR_API_KEY
+</match>
+```
+
+If tag name is `output.mixpanel.event1` and record like this:
+
+```rb
+{ user_id: "123", key1: "value1", key2: "value2" }
+```
+
+above settings send to the following data to mixpanel, using [mixpanel-ruby](https://github.com/mixpanel/mixpanel-ruby) gem.
+
+```rb
+tracker = Mixpanel::Tracker.new(YOUR_PROJECT_TOKEN)
+tracker.import(api_key, "123", "event1", { key1: "value1", key2: "value2" })
+```
+
+---
+
+fluentd-plugin-mixpanel also includes the HandleTagNameMixin mixin which allows the following additional options:
+
+```
+remove_tag_prefix <tag_prefix_to_remove_including_the_dot>
+remove_tag_suffix <tag_suffix_to_remove_including_the_dot>
+add_tag_prefix <tag_prefix_to_add_including_the_dot>
+add_tag_suffix <tag_suffix_to_add_including_the_dot>
+```
+
+### HttpMixpanelInput
+
+HttpMixpanelInput has same configuration as [http Input Plugin](http://docs.fluentd.org/en/articles/in_http).
+
+```
+<source>
+  type http_mixpanel
+  bind 127.0.0.1
+  port 8888
+  body_size_limit 10m
+  keepalive_timeout 5
+  add_http_headers true
+</source>
+```
+
+In example folder, you can see example configuration and HTML.
+
+## Contributing
+
+1. Fork it ( http://github.com/hakobera/fluent-plugin-mixpanel/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/example/http_server.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+ruby -rwebrick -e'WEBrick::HTTPServer.new(:Port => 8000, :DocumentRoot => Dir.pwd).start'

data/example/in_http_mixpanel.conf

@@ -0,0 +1,12 @@
+<source>
+  type http_mixpanel
+  bind 127.0.0.1
+  port 8888
+  body_size_limit 10m
+  keepalive_timeout 5
+  add_http_headers true
+</source>
+
+<match mixpanel.**>
+  type stdout
+</match>

data/example/index.html

@@ -0,0 +1,32 @@
+<!doctype html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <title>in_http_mixpanel example</title>
+</head>
+<body>
+  <ul>
+    <li><button data-event='event1' data-value="1">Event1</button></li>
+    <li><button data-event='event2' data-value="2">Event2</button></li>
+    <li><button data-event='event3' data-value="3">Event3</button></li>
+  </ul>
+<script type="text/javascript">
+  (function(c,a){window.mixpanel=a;var b,d,h,e;b=c.createElement("script");
+  b.type="text/javascript";b.async=!0;b.src=("https:"===c.location.protocol?"https:":"http:")+'//cdn.mxpnl.com/libs/mixpanel-2.2.min.js';d=c.getElementsByTagName("script")[0];d.parentNode.insertBefore(b,d);a._i=[];a.init=function(b,c,f){function d(a,b){var c=b.split(".");2==c.length&&(a=a[c[0]],b=c[1]);a[b]=function(){a.push([b].concat(Array.prototype.slice.call(arguments,0)))}}var g=a;"undefined"!==typeof f?g=a[f]=[]:f="mixpanel";g.people=g.people||[];h=['disable','track','track_pageview','track_links','track_forms','register','register_once','unregister','identify','alias','name_tag','set_config','people.set','people.set_once','people.increment','people.track_charge','people.append'];for(e=0;e<h.length;e++)d(g,h[e]);a._i.push([b,c,f])};a.__SV=1.2;})(document,window.mixpanel||[]);
+  mixpanel.init("dummy", {
+    api_host: '//0.0.0.0:5000',
+    debug: true
+  });
+</script>
+<script src="http://code.jquery.com/jquery-1.11.0.min.js"></script>
+<script>
+$(function () {
+  $('button').on('click', function () {
+    var evt = $(this).data('event');
+    var value = $(this).data('value');
+    mixpanel.track(evt, { value: value });
+  });
+});
+</script>
+</body>
+</html>

data/fluent-plugin-mixpanel-enchanced.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-mixpanel-enchanced"
+  spec.version       = "0.0.10"
+  spec.authors       = ["zhron4x"]
+  spec.email         = ["zhr0n4x@gmail.com"]
+  spec.summary       = %q{Fluentd plugin to input/output event track data to mixpanel}
+  spec.description   = %q{Fluentd plugin to input/output event track data to mixpanel}
+  spec.homepage      = "https://github.com/zhron4x/fluent-plugin-mixpanel-enchanced"
+  spec.license       = "Apache License, Version 2.0"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "fluentd", ">= 0.10.55"
+  spec.add_runtime_dependency "mixpanel-ruby", "~> 2.2.0"
+
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "webmock"
+  spec.add_development_dependency "test-unit", "~> 3.0.2"
+  spec.add_development_dependency "pry"
+end

data/lib/fluent/plugin/in_http_mixpanel.rb

@@ -0,0 +1,33 @@
+require 'fluent/plugin/in_http'
+require 'base64'
+require 'pry'
+
+class Fluent::HttpMixpanelInput < Fluent::HttpInput
+  Fluent::Plugin.register_input('http_mixpanel', self)
+
+  config_param :tag_prefix, :default => 'mixpanel'
+
+  def on_request(path_info, params)
+    data = Base64.decode64(params['data']).force_encoding('utf-8')
+    json = JSON.parse(data)
+    binding.pry
+    props = json['properties']
+    path = "/#{tag_prefix}.#{json['event']}"
+    params['json'] = props.to_json
+    params['time'] = props['time'].to_s if props['time']
+
+    ret = super(path, params)
+    
+    headers = {
+      'Access-Control-Allow-Credentials' => true,
+      'Access-Control-Allow-Headers' => 'X-Requested-With',
+      'Access-Control-Allow-Methods' => 'GET, POST, OPTIONS',
+      'Access-Control-Allow-Origin' => params['HTTP_ORIGIN'],
+      'Access-Control-Max-Age' => 1728000,
+      'Cache-Control' => 'no-cache, no-store',
+      'Content-type' => 'text/plain'
+    }
+
+    [ret[0], headers, (ret[0] == '200 OK' ? '1' : '0')]
+  end
+end

data/lib/fluent/plugin/mixpanel_ruby_error_handler.rb

@@ -0,0 +1,15 @@
+require 'mixpanel-ruby'
+
+class Fluent::MixpanelOutputErrorHandler < Mixpanel::ErrorHandler
+   def initialize(logger)
+     @logger = logger
+   end
+
+   def handle(error)
+     # Default behavior is to not return an error. Mixpanel-ruby gem returns
+     # true/false. If there is an error, an optional error handler is called.
+     # In this case, here, we only want to log the error for future development
+     # of error handling.
+     @logger.error "MixpanelOutputErrorHandler:\n\tClass: #{error.class.to_s}\n\tMessage: #{error.message}\n\tBacktrace: #{error.backtrace}"
+   end
+end

data/lib/fluent/plugin/out_mixpanel.rb

@@ -0,0 +1,127 @@
+require_relative "mixpanel_ruby_error_handler.rb"
+
+class Fluent::MixpanelOutput < Fluent::BufferedOutput
+  Fluent::Plugin.register_output('mixpanel', self)
+
+  include Fluent::HandleTagNameMixin
+
+  config_param :project_token, :string, :secret => true
+  config_param :api_key, :string, :default => '', :secret => true
+  config_param :use_import, :bool, :default => nil
+  config_param :distinct_id_key, :string
+  config_param :event_key, :string, :default => nil
+  config_param :ip_key, :string, :default => nil
+  config_param :event_map_tag, :bool, :default => false
+  #NOTE: This will be removed in a future release. Please specify the '.' on any prefix
+  config_param :use_legacy_prefix_behavior, :default => true
+  config_param :discard_event_on_send_error, :default => false
+
+  class MixpanelError < StandardError
+  end
+
+  def initialize
+    super
+    require 'mixpanel-ruby'
+  end
+
+  def configure(conf)
+    super
+    @project_tokey = conf['project_token']
+    @distinct_id_key = conf['distinct_id_key']
+    @event_key = conf['event_key']
+    @ip_key = conf['ip_key']
+    @event_map_tag = conf['event_map_tag']
+    @api_key = conf['api_key']
+    @use_import = conf['use_import']
+    @use_legacy_prefix_behavior = conf['use_legacy_prefix_behavior']
+    @discard_event_on_send_error = conf['discard_event_on_send_error']
+
+    if @event_key.nil? and !@event_map_tag
+      raise Fluent::ConfigError, "'event_key' must be specifed when event_map_tag == false."
+    end
+  end
+
+  def start
+    super
+    error_handler = Fluent::MixpanelOutputErrorHandler.new(log)
+    @tracker = Mixpanel::Tracker.new(@project_token, error_handler)
+  end
+
+  def shutdown
+    super
+  end
+
+  def format(tag, time, record)
+    time = record['time'] if record['time'] && @use_import
+    [tag, time, record].to_msgpack
+  end
+
+  def write(chunk)
+    records = []
+    chunk.msgpack_each do |tag, time, record|
+      data = {}
+      prop = data['properties'] = record.dup
+
+      # Ignore token in record
+      prop.delete('token')
+
+      if @event_map_tag
+        tag.gsub!(/^\./, '') if @use_legacy_prefix_behavior
+        data['event'] = tag
+      elsif record[@event_key]
+        data['event'] = record[@event_key]
+        prop.delete(@event_key)
+      else
+        log.warn("no event, tag: #{tag}, time: #{time.to_s}, record: #{record.to_json}")
+        next
+      end
+
+      # Ignore browswer only special event
+      next if data['event'].start_with?('mp_')
+
+      if record[@distinct_id_key]
+        data['distinct_id'] = record[@distinct_id_key]
+        prop.delete(@distinct_id_key)
+      else
+        log.warn("no distinct_id, tag: #{tag}, time: #{time.to_s}, record: #{record.to_json}")
+        next
+      end
+
+      if !@ip_key.nil? and record[@ip_key]
+        prop['ip'] = record[@ip_key]
+        prop.delete(@ip_key)
+      end
+
+      prop.select! {|key, _| !key.start_with?('mp_') }
+      prop.merge!('time' => time.to_i)
+
+      records << data
+    end
+
+    send_to_mixpanel(records)
+  end
+
+  def send_to_mixpanel(records)
+    log.debug("sending #{records.length} to mixpanel")
+
+    records.each do |record|
+      success = true
+
+      if @use_import
+        success = @tracker.import(@api_key, record['distinct_id'], record['event'], record['properties'])
+      else
+        success = @tracker.track(record['distinct_id'], record['event'], record['properties'])
+      end
+
+      unless success
+        if @discard_event_on_send_error
+          msg = "Failed to track event to mixpanel:\n"
+          msg += "\tRecord: #{record.to_json}"
+          log.info(msg)
+        else
+          raise MixpanelError.new("Failed to track event to mixpanel")
+        end
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,39 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+require 'webmock/test_unit'
+WebMock.disable_net_connect!(:allow_localhost => true)
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+def unused_port
+  s = TCPServer.open(0)
+  port = s.addr[1]
+  s.close
+  port
+end
+
+require 'fluent/plugin/mixpanel_ruby_error_handler'
+require 'fluent/plugin/out_mixpanel'
+require 'fluent/plugin/in_http_mixpanel'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_in_http_mixpanel.rb

@@ -0,0 +1,129 @@
+require 'helper'
+require 'net/http'
+require 'base64'
+require 'fluent/test'
+require 'net/http'
+
+class HttpMixpanelInputTest < Test::Unit::TestCase
+
+  class << self
+    def startup
+      socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+      @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
+      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+    end
+
+    def shutdown
+      @server.close
+    end
+  end
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  PORT = unused_port
+  CONFIG = %[
+    port #{PORT}
+    bind "127.0.0.1"
+    body_size_limit 10m
+    keepalive_timeout 5
+    respond_with_empty_img true
+  ]
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::HttpMixpanelInput).configure(conf, true)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal PORT, d.instance.port
+    assert_equal '127.0.0.1', d.instance.bind
+    assert_equal 10*1024*1024, d.instance.body_size_limit
+    assert_equal 5, d.instance.keepalive_timeout
+    assert_equal false, d.instance.add_http_headers
+  end
+
+  def test_time
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+
+    d.expect_emit "mixpanel.tag1", time, {"a"=>1}
+    d.expect_emit "mixpanel.tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag, record_time, record|
+        res = track("#{tag}", {"json"=>record})
+        assert_equal "200", res.code
+        assert_equal '1', res.body
+        assert_equal 'true', res['access-control-allow-credentials']
+        assert_equal 'X-Requested-With', res['access-control-allow-headers']
+        assert_equal 'GET, POST, OPTIONS', res['access-control-allow-methods']
+        assert_equal 'http://foo.example', res['access-control-allow-origin']
+        assert_equal '1728000', res['access-control-max-age']
+        assert_equal 'no-cache, no-store', res['cache-control']
+      }
+    end
+  end
+
+  def test_json
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "mixpanel.tag1", time, {"a"=>1}
+    d.expect_emit "mixpanel.tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag, record_time, record|
+        res = track("#{tag}", {"json"=>record, "time"=>record_time.to_s})
+        assert_equal "200", res.code
+      }
+    end
+
+    d.emit_streams.each { |tag, es|
+      assert !include_http_header?(es.first[1])
+    }
+  end
+
+  def test_json_with_add_http_headers
+    d = create_driver(CONFIG + "add_http_headers true")
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    records = [["mixpanel.tag1", time, {"a"=>1}], ["mixpanel.tag2", time, {"a"=>2}]]
+
+    d.run do
+      records.each {|tag, record_time, record|
+        res = track("#{tag}", {"json"=>record, "time"=>record_time.to_s})
+        assert_equal "200", res.code
+      }
+    end
+
+    d.emit_streams.each { |tag, es|
+      assert include_http_header?(es.first[1])
+    }
+  end
+
+  def track(tag, params)
+    event = tag.sub(/^mixpanel\.(.+)$/, '\1')
+    params['json']['time'] = params['time'] if params['time']
+    data = {
+      event: event,
+      properties: params['json']
+    }
+    data = CGI.escape(Base64.encode64(data.to_json))
+    query = "data=#{data}"
+    path = "/track/?#{query}"
+
+    http = Net::HTTP.new("127.0.0.1", PORT)
+    req = Net::HTTP::Get.new(path, { 'origin' => 'http://foo.example' })
+    http.request(req)
+  end
+
+  def include_http_header?(record)
+    record.keys.find { |header| header.start_with?('HTTP_') }
+  end
+end

data/test/plugin/test_mixpanel_ruby_error_handler.rb

@@ -0,0 +1,16 @@
+require 'helper'
+
+class MixpanelRubyErrorHandlerTest < Test::Unit::TestCase
+
+  def test_handle
+    mixpanel_error = Fluent::MixpanelOutput::MixpanelError.new("Foobar failed")
+    @io = StringIO.new
+    logger = Logger.new(@io)
+    error = Fluent::MixpanelOutputErrorHandler.new(logger)
+    error.handle(mixpanel_error)
+
+    output = @io.string
+    expected_output = "MixpanelOutputErrorHandler:\n\tClass: Fluent::MixpanelOutput::MixpanelError\n\tMessage: Foobar failed\n\tBacktrace: \n"
+    assert_match expected_output, output
+  end
+end

data/test/plugin/test_out_mixpanel.rb

@@ -0,0 +1,371 @@
+require 'helper'
+require 'uri'
+require 'msgpack'
+
+class MixpanelOutputTest < Test::Unit::TestCase
+
+  def setup
+    Fluent::Test.setup
+    @out = []
+  end
+
+  CONFIG = %[
+    project_token test_token
+    distinct_id_key user_id
+  ]
+
+  IMPORT_CONFIG = CONFIG + %[ api_key test_api_key
+                              use_import true
+                              use_legacy_prefix_behavior false
+                            ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::MixpanelOutput, 'mixpanel.test').configure(conf)
+  end
+
+  def stub_mixpanel(url="https://api.mixpanel.com/track")
+    stub_request(:post, url).with do |req|
+      body = URI.decode_www_form(req.body)
+      @out << JSON.load(Base64.decode64(body.assoc('data').last))
+    end.to_return(status: 200, body: JSON.generate({ status: 1 }))
+  end
+
+  def stub_mixpanel_import
+    stub_mixpanel("https://api.mixpanel.com/import")
+  end
+
+  def stub_mixpanel_unavailable(url="https://api.mixpanel.com/track")
+    stub_request(:post, url).to_return(status: 503, body: "Service Unavailable")
+  end
+
+  def sample_record
+    { user_id: "123", event: "event1", key1: "value1", key2: "value2" }
+  end
+
+  def test_configure
+    d = create_driver(CONFIG + "event_key event")
+
+    assert_equal 'test_token', d.instance.project_token
+    assert_equal 'user_id', d.instance.distinct_id_key
+    assert_equal 'event', d.instance.event_key
+  end
+
+  def test_configure_with_ip_key
+    d = create_driver(CONFIG + "event_key event\n ip_key ip")
+
+    assert_equal 'test_token', d.instance.project_token
+    assert_equal 'user_id', d.instance.distinct_id_key
+    assert_equal 'event', d.instance.event_key
+    assert_equal 'ip', d.instance.ip_key
+  end
+
+  def test_configure_with_event_map_tag
+    d = create_driver(CONFIG + "event_map_tag true")
+
+    assert_equal 'test_token', d.instance.project_token
+    assert_equal 'user_id', d.instance.distinct_id_key
+    assert_equal nil, d.instance.event_key
+    assert_equal true.to_s, d.instance.event_map_tag
+  end
+
+  def test_write
+    stub_mixpanel
+    d = create_driver(CONFIG + "event_key event")
+    time = Time.new('2014-01-01T01:23:45+00:00').to_i
+    d.emit(sample_record, time)
+    d.run
+
+    assert_equal "test_token", @out[0]['properties']['token']
+    assert_equal "123",     @out[0]['properties']['distinct_id']
+    assert_equal "event1",  @out[0]['event']
+    assert_equal time, @out[0]['properties']['time']
+    assert_equal "value1",  @out[0]['properties']['key1']
+    assert_equal "value2",  @out[0]['properties']['key2']
+  end
+
+  def test_write_setting_time_via_export
+    stub_mixpanel_import
+    d = create_driver(CONFIG + "use_import true\nevent_key event")
+    time = Time.new('2014-01-01T01:23:45+00:00').to_i
+    d.emit(sample_record.merge!('time' => 1435707767), time)
+    d.run
+
+    assert_equal "test_token", @out[0]['properties']['token']
+    assert_equal "123",     @out[0]['properties']['distinct_id']
+    assert_equal "event1",  @out[0]['event']
+    assert_equal 1435707767, @out[0]['properties']['time']
+    assert_equal "value1",  @out[0]['properties']['key1']
+    assert_equal "value2",  @out[0]['properties']['key2']
+  end
+
+  def test_write_multi_request
+    stub_mixpanel_import
+    d = create_driver(IMPORT_CONFIG + "event_key event")
+    time1 = Time.new('2014-01-01T01:23:45+00:00').to_i
+    time2 = Time.new('2014-01-02T01:23:45+00:00').to_i
+
+    d.emit(sample_record, time1)
+    d.emit(sample_record.merge(key3: "value3"), time2)
+    d.run
+
+    assert_equal "123",      @out[0]['properties']['distinct_id']
+    assert_equal "event1",   @out[0]['event']
+    assert_equal time1, @out[0]['properties']['time']
+    assert_equal "value1",   @out[0]['properties']['key1']
+    assert_equal "value2",   @out[0]['properties']['key2']
+
+    assert_equal "123",      @out[1]['properties']['distinct_id']
+    assert_equal "event1",   @out[1]['event']
+    assert_equal time2, @out[1]['properties']['time']
+    assert_equal "value1",   @out[1]['properties']['key1']
+    assert_equal "value2",   @out[1]['properties']['key2']
+    assert_equal "value2",   @out[1]['properties']['key2']
+  end
+
+  def test_write_with_ip_key
+    stub_mixpanel
+    d = create_driver(CONFIG + "event_key event\n ip_key ip_address")
+    time = Time.new('2014-01-01T01:23:45+00:00').to_i
+    d.emit(sample_record.merge('ip_address' => '192.168.0.2'), time)
+    d.run
+
+    assert_equal "123",         @out[0]['properties']['distinct_id']
+    assert_equal "event1",      @out[0]['event']
+    assert_equal time,     @out[0]['properties']['time']
+    assert_equal "192.168.0.2", @out[0]['properties']['ip']
+    assert_equal "value1",      @out[0]['properties']['key1']
+    assert_equal "value2",      @out[0]['properties']['key2']
+  end
+
+  def test_write_with_no_tag_manipulation
+    stub_mixpanel
+    d = create_driver(CONFIG + "event_map_tag true")
+    time = Time.new('2014-01-01T01:23:45+00:00').to_i
+    d.emit(sample_record, time)
+    d.run
+
+    assert_equal "123",           @out[0]['properties']['distinct_id']
+    assert_equal "mixpanel.test", @out[0]['event']
+    assert_equal time,       @out[0]['properties']['time']
+    assert_equal "value1",        @out[0]['properties']['key1']
+    assert_equal "value2",        @out[0]['properties']['key2']
+  end
+
+  def test_write_with_event_map_tag_removing_prefix
+    stub_mixpanel
+    d = create_driver(CONFIG + "remove_tag_prefix mixpanel.\n event_map_tag true")
+    time = Time.new('2014-01-01T01:23:45+00:00').to_i
+    d.emit(sample_record, time)
+    d.run
+
+    assert_equal "123",     @out[0]['properties']['distinct_id']
+    assert_equal "test",    @out[0]['event']
+    assert_equal time, @out[0]['properties']['time']
+    assert_equal "value1",  @out[0]['properties']['key1']
+    assert_equal "value2",  @out[0]['properties']['key2']
+  end
+
+  def test_write_with_event_map_tag_removing_prefix_LEGACY
+    stub_mixpanel
+    d = create_driver(CONFIG + "remove_tag_prefix mixpanel\n event_map_tag true\n use_legacy_prefix_behavior true")
+    time = Time.new('2014-01-01T01:23:45+00:00').to_i
+    d.emit(sample_record, time)
+    d.run
+
+    assert_equal "123",     @out[0]['properties']['distinct_id']
+    assert_equal "test",    @out[0]['event']
+    assert_equal time, @out[0]['properties']['time']
+    assert_equal "value1",  @out[0]['properties']['key1']
+    assert_equal "value2",  @out[0]['properties']['key2']
+  end
+
+  def test_write_with_event_map_tag_removing_prefix_LEGACY_with_dot
+    stub_mixpanel
+    d = create_driver(CONFIG + "remove_tag_prefix mixpanel.\n event_map_tag true\n use_legacy_prefix_behavior true")
+    time = Time.new('2014-01-01T01:23:45+00:00').to_i
+    d.emit(sample_record, time)
+    d.run
+
+    assert_equal "123",     @out[0]['properties']['distinct_id']
+    assert_equal "test",    @out[0]['event']
+    assert_equal time, @out[0]['properties']['time']
+    assert_equal "value1",  @out[0]['properties']['key1']
+    assert_equal "value2",  @out[0]['properties']['key2']
+  end
+
+  def test_write_with_event_map_tag_removing_suffix
+    stub_mixpanel
+    d = create_driver(CONFIG + "remove_tag_suffix .test\n event_map_tag true")
+    time = Time.new('2014-01-01T01:23:45+00:00').to_i
+    d.emit(sample_record, time)
+    d.run
+
+    assert_equal "123",     @out[0]['properties']['distinct_id']
+    assert_equal "mixpanel",    @out[0]['event']
+    assert_equal time, @out[0]['properties']['time']
+    assert_equal "value1",  @out[0]['properties']['key1']
+    assert_equal "value2",  @out[0]['properties']['key2']
+  end
+
+  def test_write_with_event_map_tag_adding_prefix
+    stub_mixpanel
+    d = create_driver(CONFIG + "add_tag_prefix foo.\n event_map_tag true")
+    time = Time.new('2014-01-01T01:23:45+00:00').to_i
+    d.emit(sample_record, time)
+    d.run
+
+    assert_equal "123",     @out[0]['properties']['distinct_id']
+    assert_equal "foo.mixpanel.test",    @out[0]['event']
+    assert_equal time, @out[0]['properties']['time']
+    assert_equal "value1",  @out[0]['properties']['key1']
+    assert_equal "value2",  @out[0]['properties']['key2']
+  end
+
+  def test_write_with_event_map_tag_adding_suffix
+    stub_mixpanel
+    d = create_driver(CONFIG + "add_tag_suffix .foo\n event_map_tag true")
+    time = Time.new('2014-01-01T01:23:45+00:00').to_i
+    d.emit(sample_record, time)
+    d.run
+
+    assert_equal "123",     @out[0]['properties']['distinct_id']
+    assert_equal "mixpanel.test.foo",    @out[0]['event']
+    assert_equal time, @out[0]['properties']['time']
+    assert_equal "value1",  @out[0]['properties']['key1']
+    assert_equal "value2",  @out[0]['properties']['key2']
+  end
+
+  def test_write_ignore_special_event
+    stub_mixpanel
+    d = create_driver(CONFIG + "event_key event")
+    time = Time.new('2014-01-01T01:23:45+00:00').to_i
+    d.emit({ user_id: '123', event: 'mp_page_view' }, time)
+    d.run
+
+    assert_equal 0, @out.length
+  end
+
+  def test_write_ignore_special_property
+    stub_mixpanel
+    d = create_driver(CONFIG + "event_key event")
+    time = Time.new('2014-01-01T01:23:45+00:00').to_i
+    d.emit(sample_record.merge('mp_event' => '3'), time)
+    d.run
+
+    assert_equal "test_token", @out[0]['properties']['token']
+    assert_equal "123",     @out[0]['properties']['distinct_id']
+    assert_equal "event1",  @out[0]['event']
+    assert_equal time, @out[0]['properties']['time']
+    assert_equal "value1",  @out[0]['properties']['key1']
+    assert_equal "value2",  @out[0]['properties']['key2']
+    assert_equal false, @out[0]['properties'].key?('mp_event')
+  end
+
+  def test_write_delete_supried_token
+    stub_mixpanel
+    d = create_driver(CONFIG + "event_key event")
+    time = Time.new('2014-01-01T01:23:45+00:00').to_i
+    d.emit(sample_record.merge('token' => '123'), time)
+    d.run
+
+    assert_equal "test_token", @out[0]['properties']['token']
+    assert_equal "123",     @out[0]['properties']['distinct_id']
+    assert_equal "event1",  @out[0]['event']
+    assert_equal time, @out[0]['properties']['time']
+    assert_equal "value1",  @out[0]['properties']['key1']
+    assert_equal "value2",  @out[0]['properties']['key2']
+    assert_equal false, @out[0]['properties'].key?('mp_event')
+  end
+
+  def test_request_error
+    stub_mixpanel_unavailable
+    d = create_driver(CONFIG + "event_key event")
+    d.emit(sample_record)
+    assert_raise(Fluent::MixpanelOutput::MixpanelError) {
+      d.run
+    }
+  end
+
+  def test_multiple_records_1_missing_event
+    stub_mixpanel
+    d = create_driver(CONFIG + "event_key event")
+    time = Time.new('2014-01-01T01:23:45+00:00').to_i
+    d.emit(sample_record, time)
+
+    broken_record = sample_record.dup.delete(:event)
+    d.emit(broken_record, time)
+
+    d.run
+
+    assert_equal 1, @out.length
+
+    assert_equal "test_token", @out[0]['properties']['token']
+    assert_equal "123",     @out[0]['properties']['distinct_id']
+    assert_equal "event1",  @out[0]['event']
+    assert_equal time, @out[0]['properties']['time']
+    assert_equal "value1",  @out[0]['properties']['key1']
+    assert_equal "value2",  @out[0]['properties']['key2']
+
+  end
+
+  def test_multiple_records_1_missing_distinct_id
+    stub_mixpanel
+    d = create_driver(CONFIG + "event_key event")
+    time = Time.new('2014-01-01T01:23:45+00:00').to_i
+    d.emit(sample_record, time)
+
+    broken_record = sample_record.dup.delete(:user_id)
+    d.emit(broken_record, time)
+
+    d.run
+
+    assert_equal 1, @out.length
+
+    assert_equal "test_token", @out[0]['properties']['token']
+    assert_equal "123",     @out[0]['properties']['distinct_id']
+    assert_equal "event1",  @out[0]['event']
+    assert_equal time, @out[0]['properties']['time']
+    assert_equal "value1",  @out[0]['properties']['key1']
+    assert_equal "value2",  @out[0]['properties']['key2']
+
+  end
+
+  def test_multiple_records_1_having_mp
+    stub_mixpanel
+    d = create_driver(CONFIG + "event_key event")
+    time = Time.new('2014-01-01T01:23:45+00:00').to_i
+    d.emit(sample_record, time)
+
+    broken_record = sample_record.dup.merge({ event: 'mp_foo'})
+    d.emit(broken_record, time)
+
+    d.run
+
+    assert_equal 1, @out.length
+
+    assert_equal "test_token", @out[0]['properties']['token']
+    assert_equal "123",     @out[0]['properties']['distinct_id']
+    assert_equal "event1",  @out[0]['event']
+    assert_equal time, @out[0]['properties']['time']
+    assert_equal "value1",  @out[0]['properties']['key1']
+    assert_equal "value2",  @out[0]['properties']['key2']
+
+  end
+
+  def test_request_error_discard
+    stub_mixpanel_unavailable
+    d = create_driver(CONFIG + "event_key event\ndiscard_event_on_send_error true")
+    time = Time.new('2014-01-01T01:23:45+00:00').to_i
+    d.emit(sample_record, time)
+    d.run
+
+    logs = d.instance.log.logs
+
+    assert_match "MixpanelOutputErrorHandler:", logs[0]
+    assert_match "Class: Mixpanel::ServerError", logs[0]
+    assert_match "Message: Could not write to Mixpanel, server responded with 503 returning: 'Service Unavailable", logs[0]
+    assert_match "Backtrace", logs[0]
+    assert_match "Failed to track event to mixpanel", logs[1]
+    assert_match 'Record: {"properties":{"key1":"value1","key2":"value2","time":' + time.to_s + '},"event":"event1","distinct_id":"123"}', logs[1]
+  end
+end

metadata

@@ -0,0 +1,149 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-mixpanel-enchanced
+version: !ruby/object:Gem::Version
+  version: 0.0.10
+platform: ruby
+authors:
+- zhron4x
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-02-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.55
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.55
+- !ruby/object:Gem::Dependency
+  name: mixpanel-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.2.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.2
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd plugin to input/output event track data to mixpanel
+email:
+- zhr0n4x@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- example/http_server.sh
+- example/in_http_mixpanel.conf
+- example/index.html
+- fluent-plugin-mixpanel-enchanced.gemspec
+- lib/fluent/plugin/in_http_mixpanel.rb
+- lib/fluent/plugin/mixpanel_ruby_error_handler.rb
+- lib/fluent/plugin/out_mixpanel.rb
+- test/helper.rb
+- test/plugin/test_in_http_mixpanel.rb
+- test/plugin/test_mixpanel_ruby_error_handler.rb
+- test/plugin/test_out_mixpanel.rb
+homepage: https://github.com/zhron4x/fluent-plugin-mixpanel-enchanced
+licenses:
+- Apache License, Version 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: Fluentd plugin to input/output event track data to mixpanel
+test_files:
+- test/helper.rb
+- test/plugin/test_in_http_mixpanel.rb
+- test/plugin/test_mixpanel_ruby_error_handler.rb
+- test/plugin/test_out_mixpanel.rb