fluent-plugin-indicative 0.1.3 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 89dfbfce81d05b8bf8b93141d722eab9bbeb88bdd92a88ac1d6e2abfa1dc8ba7
-  data.tar.gz: d71646b0fac4161dbe30ee5e9362b6d959138a30dfaceaaea85cd9dd8984858f
+  metadata.gz: 5156275d2e768b5d94cfb54dacac2edbd1730480dd13b94f02df6b3fb2345fd6
+  data.tar.gz: 8014b67c109af46ddbb9a9085cb8da4c93bc23f82a1c926418074a2832c36f6d
 SHA512:
-  metadata.gz: 22d2abb17750310715710fa2f1847ffb44d7891b50dc9cdd37efffb88f92cb02320e897844349605ef8260ad993562836ed15a41b96130247dd3ba96e905f1e6
-  data.tar.gz: 92c62349f678b48557717c60e669bb503226cb6573b88f2cd2101556808d35f5d42dee03c7965c1c31e6f873c2e1f33a1c4fcdb0fa91ff991dc6660ce04bbe76
+  metadata.gz: cf214eec01b00c32e7928d2c040bc9c0e44b03c4f5fac422e363ca4bed3aaafa2f211e426ba598f63c4d41b8d1edaffa38fa579817d9ecfc27361883220edaa8
+  data.tar.gz: c7c971b0fac7a26fffd47402f1915d8bbdfe79f78da32513f1207ec29e53b6abbde6319027c99873ede53a4c7e15927cbc5964c567eb3024f6472796af9fa74a

data/README.md

@@ -12,5 +12,11 @@ Fluentd output plugin to send events to [Indicative](https://www.indicative.com/
   event_name_key        event_name
   event_time_key        created_at
   event_unique_id_keys  user_id, cookie_id, session_id  # keys to search for unique user ID value, in order of priority
+
+  # Optionally use buffering (recommended for high event volumes)
+  <buffer>
+    path /var/log/td-agent/indicative.buffer
+    chunk_limit_records 1000
+  </buffer>
 </match>
 ```

data/fluent-plugin-indicative.gemspec

@@ -3,10 +3,10 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-indicative"
-  s.version     = "0.1.3"
+  s.version     = "0.1.8"
   s.authors     = ["Sam Millar"]
   s.email       = ["sam@millar.io"]
-  s.homepage    = "https://github.com/millar/fluent-plugin-indicative"
+  s.homepage    = "https://github.com/mixcloud/fluent-plugin-indicative"
   s.summary     = %q{Fluentd output plugin to send events to Indicative}
 
   s.files         = `git ls-files`.split("\n")

data/lib/fluent/plugin/out_indicative.rb

@@ -5,13 +5,21 @@ require 'uri'
 
 require 'fluent/plugin/output'
 
+
 def flatten_hash(hash)
   hash.each_with_object({}) do |(k, v), h|
     if v.is_a? Hash
       flatten_hash(v).map do |h_k, h_v|
         h["#{k}.#{h_k}"] = h_v
       end
-    elsif !v.is_a? Array
+    elsif v.is_a? Array
+      # Indicative doesn't support arrays so we use the value of the array as a key and set it to true
+      v.each do |item|
+        if item.is_a?(Hash) && item.has_key?("key") && item.has_key?("value")
+          h["#{k}.#{item["key"]}"] = item["value"]
+        end
+      end
+    else
       h[k] = v
     end
    end
@@ -22,30 +30,45 @@ class Fluent::Plugin::IndicativeOutput < Fluent::Plugin::Output
   Fluent::Plugin.register_output('indicative', self)
 
   config_param :api_key, :string, secret: true
-  config_param :api_url, :string, default: 'https://api.indicative.com/service/event'
+  config_param :api_url, :string, default: 'https://api.indicative.com/service/event/batch'
+  config_param :batch_size, :integer, default: 15
   config_param :event_name_key, :string
   config_param :event_time_key, :string
   config_param :event_unique_id_keys, :array, value_type: :string
+  config_param :event_filter_key, :string, default: nil
 
   def process(tag, es)
-    es.each do |time, record|
-      send_event(record)
+    es.each_slice(@batch_size) do |events|
+      send_events(events.map {|time, record| record})
+    end
+  end
+
+  def write(chunk)
+    records = []
+    chunk.each do |time, record|
+      records << record
+    end
+    records.each_slice(@batch_size) do |events|
+      send_events(events)
     end
   end
 
-  def send_event(data)
+  def send_events(events)
     uri = URI.parse(@api_url)
 
     headers = {'Content-Type' => 'application/json'}
 
-    unique_id_key = @event_unique_id_keys.find {|k| data[k]}
-
     payload = {
       apiKey: @api_key,
-      eventName: data[@event_name_key],
-      eventUniqueId: unique_id_key && data[unique_id_key],
-      properties: flatten_hash(data),
-      eventTime: DateTime.parse(data[@event_time_key]).rfc3339
+      events: events.filter {|data| !@event_filter_key or data[@event_filter_key] != false}.map do |data|
+        unique_id_key = @event_unique_id_keys.find {|k| data[k]}
+        {
+          eventName: data[@event_name_key],
+          eventUniqueId: unique_id_key && data[unique_id_key],
+          properties: flatten_hash(data),
+          eventTime: DateTime.parse(data[@event_time_key]).rfc3339
+        }
+      end
     }
 
     http = Net::HTTP.new(uri.host, uri.port)

data/test/plugin/test_out_indicative.rb

@@ -5,14 +5,33 @@ class IndicativeOutputTest < Test::Unit::TestCase
     Fluent::Test.setup
   end
 
-  CONFIG = %[
+  STREAM_CONFIG = %[
     api_key               INDICATIVE_API_KEY
     event_name_key        event_name
     event_time_key        created_at
     event_unique_id_keys  user_id, session_id
   ]
 
-  def create_driver(conf=CONFIG)
+  FILTER_CONFIG = %[
+    api_key               INDICATIVE_API_KEY
+    event_name_key        event_name
+    event_time_key        created_at
+    event_unique_id_keys  user_id, session_id
+    event_filter_key      indicative
+  ]
+
+  BUFFER_CONFIG = %[
+    api_key               INDICATIVE_API_KEY
+    event_name_key        event_name
+    event_time_key        created_at
+    event_unique_id_keys  user_id, session_id
+
+    <buffer>
+      chunk_limit_records 50
+    </buffer>
+  ]
+
+  def create_driver(conf=STREAM_CONFIG)
     Fluent::Test::Driver::Output.new(Fluent::Plugin::IndicativeOutput).configure(conf)
   end
 
@@ -20,7 +39,7 @@ class IndicativeOutputTest < Test::Unit::TestCase
     assert_raise(Fluent::ConfigError) {
       d = create_driver('')
     }
-    d = create_driver CONFIG
+    d = create_driver(STREAM_CONFIG)
     assert_equal 'INDICATIVE_API_KEY', d.instance.api_key
     assert_equal 'event_name', d.instance.event_name_key
     assert_equal 'created_at', d.instance.event_time_key
@@ -28,8 +47,8 @@ class IndicativeOutputTest < Test::Unit::TestCase
   end
 
 
-  def test_emit
-    d = create_driver(CONFIG)
+  def test_emit_stream
+    d = create_driver(STREAM_CONFIG)
     stub_request(:any, d.instance.api_url)
     d.run(default_tag: 'test') do
       d.feed({'event_name' => 'screen_view', 'created_at' => '2015-01-01T10:00:00.000Z', 'session_id' => 'a3bd2', 'user_id' => nil, 'screen' => {'id' => 'index'}})
@@ -39,16 +58,93 @@ class IndicativeOutputTest < Test::Unit::TestCase
     assert_requested :post, d.instance.api_url,
       headers: {'Content-Type' => 'application/json'}, body: {
         'apiKey' => 'INDICATIVE_API_KEY',
-        'eventName' => 'screen_view',
-        'eventUniqueId' => 'a3bd2',
-        'properties' => {
-          'event_name' => 'screen_view',
-          'created_at' => '2015-01-01T10:00:00.000Z',
-          'session_id' => 'a3bd2',
-          'user_id' => nil,
-          'screen.id' => 'index'
-        },
-        'eventTime' => '2015-01-01T10:00:00+00:00'
+        'events' => [{
+          'eventName' => 'screen_view',
+          'eventUniqueId' => 'a3bd2',
+          'properties' => {
+            'event_name' => 'screen_view',
+            'created_at' => '2015-01-01T10:00:00.000Z',
+            'session_id' => 'a3bd2',
+            'user_id' => nil,
+            'screen.id' => 'index'
+          },
+          'eventTime' => '2015-01-01T10:00:00+00:00'
+        }]
+      }.to_json, times: 1
+  end
+
+  def test_emit_stream_with_filter
+    d = create_driver(FILTER_CONFIG)
+    stub_request(:any, d.instance.api_url)
+    d.run(default_tag: 'test') do
+      d.feed({'event_name' => 'filter_nil', 'created_at' => '2015-01-01T10:00:00.000Z', 'session_id' => 'a3bd2'})
+      d.feed({'event_name' => 'filter_false', 'created_at' => '2015-01-01T10:00:00.000Z', 'session_id' => 'a3bd2', 'indicative' => false})
+      d.feed({'event_name' => 'filter_true', 'created_at' => '2015-01-01T10:00:00.000Z', 'session_id' => 'a3bd2', 'indicative' => true})
+    end
+    events = d.events
+    assert_equal 0, events.length
+    assert_requested :post, d.instance.api_url,
+      headers: {'Content-Type' => 'application/json'}, body: {
+        'apiKey' => 'INDICATIVE_API_KEY',
+        'events' => [{
+          'eventName' => 'filter_nil',
+          'eventUniqueId' => 'a3bd2',
+          'properties' => {
+            'event_name' => 'filter_nil',
+            'created_at' => '2015-01-01T10:00:00.000Z',
+            'session_id' => 'a3bd2'
+          },
+          'eventTime' => '2015-01-01T10:00:00+00:00'
+        }, {
+          'eventName' => 'filter_true',
+          'eventUniqueId' => 'a3bd2',
+          'properties' => {
+            'event_name' => 'filter_true',
+            'created_at' => '2015-01-01T10:00:00.000Z',
+            'session_id' => 'a3bd2',
+            'indicative' => true
+          },
+          'eventTime' => '2015-01-01T10:00:00+00:00'
+        }]
+      }.to_json, times: 1
+  end
+
+  def test_emit_buffer
+    d = create_driver(BUFFER_CONFIG)
+    stub_request(:any, d.instance.api_url)
+    d.run(default_tag: 'test') do
+      5.times do
+        d.feed({'event_name' => 'screen_view', 'created_at' => '2015-01-01T10:00:00.000Z', 'session_id' => 'a3bd2', 'user_id' => nil, 'screen' => {'id' => 'index'}})
+      end
+    end
+    events = d.events
+    assert_equal 0, events.length
+    assert_requested :post, d.instance.api_url, times: 1
+  end
+
+  def test_key_value_object_transformation
+    d = create_driver(STREAM_CONFIG)
+    stub_request(:any, d.instance.api_url)
+    d.run(default_tag: 'test') do
+      d.feed({'event_name' => 'screen_view', 'created_at' => '2015-01-01T10:00:00.000Z', 'session_id' => 'a3bd2', 'experiments': [{'key': 'a', 'value': 1}, {'key': 'b', 'value': 2}]})
+    end
+    events = d.events
+    assert_equal 0, events.length
+    assert_requested :post, d.instance.api_url,
+      headers: {'Content-Type' => 'application/json'}, body: {
+        'apiKey' => 'INDICATIVE_API_KEY',
+        'events' => [{
+          'eventName' => 'screen_view',
+          'eventUniqueId' => 'a3bd2',
+          'properties' => {
+            'event_name' => 'screen_view',
+            'created_at' => '2015-01-01T10:00:00.000Z',
+            'session_id' => 'a3bd2',
+            'experiments.a' => 1,
+            'experiments.b' => 2
+          },
+          'eventTime' => '2015-01-01T10:00:00+00:00'
+        }]
       }.to_json, times: 1
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-indicative
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.8
 platform: ruby
 authors:
 - Sam Millar
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-12 00:00:00.000000000 Z
+date: 2020-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -95,7 +95,7 @@ files:
 - lib/fluent/plugin/out_indicative.rb
 - test/helper.rb
 - test/plugin/test_out_indicative.rb
-homepage: https://github.com/millar/fluent-plugin-indicative
+homepage: https://github.com/mixcloud/fluent-plugin-indicative
 licenses: []
 metadata: {}
 post_install_message: 
@@ -113,8 +113,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Fluentd output plugin to send events to Indicative