fluent-plugin-indicative 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 89dfbfce81d05b8bf8b93141d722eab9bbeb88bdd92a88ac1d6e2abfa1dc8ba7
-  data.tar.gz: d71646b0fac4161dbe30ee5e9362b6d959138a30dfaceaaea85cd9dd8984858f
+  metadata.gz: 77a70ff5f9f75d916912313e9ad5570700461e7c1ad32d425d8424fddff54a99
+  data.tar.gz: b10e2ba2fa650fb0670d9e75023ae8e640a5759dc217d229abb7a3221e494cf8
 SHA512:
-  metadata.gz: 22d2abb17750310715710fa2f1847ffb44d7891b50dc9cdd37efffb88f92cb02320e897844349605ef8260ad993562836ed15a41b96130247dd3ba96e905f1e6
-  data.tar.gz: 92c62349f678b48557717c60e669bb503226cb6573b88f2cd2101556808d35f5d42dee03c7965c1c31e6f873c2e1f33a1c4fcdb0fa91ff991dc6660ce04bbe76
+  metadata.gz: 9895f1104b602c410cbbe022e2704de3b74bc8ff5cc52e6111d35840ea57b56bd03e88ecca0356b3a7c63edd209ce3def48cf6ecca3e12da8d0d87cbb438fa18
+  data.tar.gz: ae64a4fb95fb6d2fb219d691ae9dd6ab10245ba85a84584a06bae4fc2b7c630ddf0f239d5e6c527f01cfa1525d18cb6e66d603ff4854ec7aa62d703a3841773f

data/README.md

@@ -12,5 +12,11 @@ Fluentd output plugin to send events to [Indicative](https://www.indicative.com/
   event_name_key        event_name
   event_time_key        created_at
   event_unique_id_keys  user_id, cookie_id, session_id  # keys to search for unique user ID value, in order of priority
+
+  # Optionally use buffering (recommended for high event volumes)
+  <buffer>
+    path /var/log/td-agent/indicative.buffer
+    chunk_limit_records 1000
+  </buffer>
 </match>
 ```

data/fluent-plugin-indicative.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-indicative"
-  s.version     = "0.1.3"
+  s.version     = "0.1.4"
   s.authors     = ["Sam Millar"]
   s.email       = ["sam@millar.io"]
   s.homepage    = "https://github.com/millar/fluent-plugin-indicative"

data/lib/fluent/plugin/out_indicative.rb

@@ -5,6 +5,9 @@ require 'uri'
 
 require 'fluent/plugin/output'
 
+BATCH_SIZE = 100
+
+
 def flatten_hash(hash)
   hash.each_with_object({}) do |(k, v), h|
     if v.is_a? Hash
@@ -28,24 +31,37 @@ class Fluent::Plugin::IndicativeOutput < Fluent::Plugin::Output
   config_param :event_unique_id_keys, :array, value_type: :string
 
   def process(tag, es)
-    es.each do |time, record|
-      send_event(record)
+    es.each_slice(BATCH_SIZE) do |events|
+      send_events(events.map {|time, record| record})
+    end
+  end
+
+  def write(chunk)
+    records = []
+    chunk.each do |time, record|
+      records << record
+    end
+    records.each_slice(BATCH_SIZE) do |events|
+      send_events(events)
     end
   end
 
-  def send_event(data)
+  def send_events(events)
     uri = URI.parse(@api_url)
 
     headers = {'Content-Type' => 'application/json'}
 
-    unique_id_key = @event_unique_id_keys.find {|k| data[k]}
-
     payload = {
       apiKey: @api_key,
-      eventName: data[@event_name_key],
-      eventUniqueId: unique_id_key && data[unique_id_key],
-      properties: flatten_hash(data),
-      eventTime: DateTime.parse(data[@event_time_key]).rfc3339
+      events: events.map do |data|
+        unique_id_key = @event_unique_id_keys.find {|k| data[k]}
+        {
+          eventName: data[@event_name_key],
+          eventUniqueId: unique_id_key && data[unique_id_key],
+          properties: flatten_hash(data),
+          eventTime: DateTime.parse(data[@event_time_key]).rfc3339
+        }
+      end
     }
 
     http = Net::HTTP.new(uri.host, uri.port)

data/test/plugin/test_out_indicative.rb

@@ -5,14 +5,29 @@ class IndicativeOutputTest < Test::Unit::TestCase
     Fluent::Test.setup
   end
 
-  CONFIG = %[
+  STREAM_CONFIG = %[
     api_key               INDICATIVE_API_KEY
     event_name_key        event_name
     event_time_key        created_at
     event_unique_id_keys  user_id, session_id
   ]
 
-  def create_driver(conf=CONFIG)
+  BUFFER_CONFIG = %[
+    api_key               INDICATIVE_API_KEY
+    event_name_key        event_name
+    event_time_key        created_at
+    event_unique_id_keys  user_id, session_id
+
+    <buffer>
+      chunk_limit_records 50
+    </buffer>
+
+    <format>
+      @type json
+    </format>
+  ]
+
+  def create_driver(conf=STREAM_CONFIG)
     Fluent::Test::Driver::Output.new(Fluent::Plugin::IndicativeOutput).configure(conf)
   end
 
@@ -20,7 +35,7 @@ class IndicativeOutputTest < Test::Unit::TestCase
     assert_raise(Fluent::ConfigError) {
       d = create_driver('')
     }
-    d = create_driver CONFIG
+    d = create_driver(STREAM_CONFIG)
     assert_equal 'INDICATIVE_API_KEY', d.instance.api_key
     assert_equal 'event_name', d.instance.event_name_key
     assert_equal 'created_at', d.instance.event_time_key
@@ -28,8 +43,8 @@ class IndicativeOutputTest < Test::Unit::TestCase
   end
 
 
-  def test_emit
-    d = create_driver(CONFIG)
+  def test_emit_stream
+    d = create_driver(STREAM_CONFIG)
     stub_request(:any, d.instance.api_url)
     d.run(default_tag: 'test') do
       d.feed({'event_name' => 'screen_view', 'created_at' => '2015-01-01T10:00:00.000Z', 'session_id' => 'a3bd2', 'user_id' => nil, 'screen' => {'id' => 'index'}})
@@ -39,16 +54,31 @@ class IndicativeOutputTest < Test::Unit::TestCase
     assert_requested :post, d.instance.api_url,
       headers: {'Content-Type' => 'application/json'}, body: {
         'apiKey' => 'INDICATIVE_API_KEY',
-        'eventName' => 'screen_view',
-        'eventUniqueId' => 'a3bd2',
-        'properties' => {
-          'event_name' => 'screen_view',
-          'created_at' => '2015-01-01T10:00:00.000Z',
-          'session_id' => 'a3bd2',
-          'user_id' => nil,
-          'screen.id' => 'index'
-        },
-        'eventTime' => '2015-01-01T10:00:00+00:00'
+        'events' => [{
+          'eventName' => 'screen_view',
+          'eventUniqueId' => 'a3bd2',
+          'properties' => {
+            'event_name' => 'screen_view',
+            'created_at' => '2015-01-01T10:00:00.000Z',
+            'session_id' => 'a3bd2',
+            'user_id' => nil,
+            'screen.id' => 'index'
+          },
+          'eventTime' => '2015-01-01T10:00:00+00:00'
+        }]
       }.to_json, times: 1
   end
+
+  def test_emit_buffer
+    d = create_driver(BUFFER_CONFIG)
+    stub_request(:any, d.instance.api_url)
+    d.run(default_tag: 'test') do
+      20.times do
+        d.feed({'event_name' => 'screen_view', 'created_at' => '2015-01-01T10:00:00.000Z', 'session_id' => 'a3bd2', 'user_id' => nil, 'screen' => {'id' => 'index'}})
+      end
+    end
+    events = d.events
+    assert_equal 0, events.length
+    assert_requested :post, d.instance.api_url, times: 1
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-indicative
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Sam Millar
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-12 00:00:00.000000000 Z
+date: 2019-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake