fluent-plugin-indicative 0.1.3 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 89dfbfce81d05b8bf8b93141d722eab9bbeb88bdd92a88ac1d6e2abfa1dc8ba7
4
- data.tar.gz: d71646b0fac4161dbe30ee5e9362b6d959138a30dfaceaaea85cd9dd8984858f
3
+ metadata.gz: 5156275d2e768b5d94cfb54dacac2edbd1730480dd13b94f02df6b3fb2345fd6
4
+ data.tar.gz: 8014b67c109af46ddbb9a9085cb8da4c93bc23f82a1c926418074a2832c36f6d
5
5
  SHA512:
6
- metadata.gz: 22d2abb17750310715710fa2f1847ffb44d7891b50dc9cdd37efffb88f92cb02320e897844349605ef8260ad993562836ed15a41b96130247dd3ba96e905f1e6
7
- data.tar.gz: 92c62349f678b48557717c60e669bb503226cb6573b88f2cd2101556808d35f5d42dee03c7965c1c31e6f873c2e1f33a1c4fcdb0fa91ff991dc6660ce04bbe76
6
+ metadata.gz: cf214eec01b00c32e7928d2c040bc9c0e44b03c4f5fac422e363ca4bed3aaafa2f211e426ba598f63c4d41b8d1edaffa38fa579817d9ecfc27361883220edaa8
7
+ data.tar.gz: c7c971b0fac7a26fffd47402f1915d8bbdfe79f78da32513f1207ec29e53b6abbde6319027c99873ede53a4c7e15927cbc5964c567eb3024f6472796af9fa74a
data/README.md CHANGED
@@ -12,5 +12,11 @@ Fluentd output plugin to send events to [Indicative](https://www.indicative.com/
12
12
  event_name_key event_name
13
13
  event_time_key created_at
14
14
  event_unique_id_keys user_id, cookie_id, session_id # keys to search for unique user ID value, in order of priority
15
+
16
+ # Optionally use buffering (recommended for high event volumes)
17
+ <buffer>
18
+ path /var/log/td-agent/indicative.buffer
19
+ chunk_limit_records 1000
20
+ </buffer>
15
21
  </match>
16
22
  ```
@@ -3,10 +3,10 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "fluent-plugin-indicative"
6
- s.version = "0.1.3"
6
+ s.version = "0.1.8"
7
7
  s.authors = ["Sam Millar"]
8
8
  s.email = ["sam@millar.io"]
9
- s.homepage = "https://github.com/millar/fluent-plugin-indicative"
9
+ s.homepage = "https://github.com/mixcloud/fluent-plugin-indicative"
10
10
  s.summary = %q{Fluentd output plugin to send events to Indicative}
11
11
 
12
12
  s.files = `git ls-files`.split("\n")
@@ -5,13 +5,21 @@ require 'uri'
5
5
 
6
6
  require 'fluent/plugin/output'
7
7
 
8
+
8
9
  def flatten_hash(hash)
9
10
  hash.each_with_object({}) do |(k, v), h|
10
11
  if v.is_a? Hash
11
12
  flatten_hash(v).map do |h_k, h_v|
12
13
  h["#{k}.#{h_k}"] = h_v
13
14
  end
14
- elsif !v.is_a? Array
15
+ elsif v.is_a? Array
16
+ # Indicative doesn't support arrays so we use the value of the array as a key and set it to true
17
+ v.each do |item|
18
+ if item.is_a?(Hash) && item.has_key?("key") && item.has_key?("value")
19
+ h["#{k}.#{item["key"]}"] = item["value"]
20
+ end
21
+ end
22
+ else
15
23
  h[k] = v
16
24
  end
17
25
  end
@@ -22,30 +30,45 @@ class Fluent::Plugin::IndicativeOutput < Fluent::Plugin::Output
22
30
  Fluent::Plugin.register_output('indicative', self)
23
31
 
24
32
  config_param :api_key, :string, secret: true
25
- config_param :api_url, :string, default: 'https://api.indicative.com/service/event'
33
+ config_param :api_url, :string, default: 'https://api.indicative.com/service/event/batch'
34
+ config_param :batch_size, :integer, default: 15
26
35
  config_param :event_name_key, :string
27
36
  config_param :event_time_key, :string
28
37
  config_param :event_unique_id_keys, :array, value_type: :string
38
+ config_param :event_filter_key, :string, default: nil
29
39
 
30
40
  def process(tag, es)
31
- es.each do |time, record|
32
- send_event(record)
41
+ es.each_slice(@batch_size) do |events|
42
+ send_events(events.map {|time, record| record})
43
+ end
44
+ end
45
+
46
+ def write(chunk)
47
+ records = []
48
+ chunk.each do |time, record|
49
+ records << record
50
+ end
51
+ records.each_slice(@batch_size) do |events|
52
+ send_events(events)
33
53
  end
34
54
  end
35
55
 
36
- def send_event(data)
56
+ def send_events(events)
37
57
  uri = URI.parse(@api_url)
38
58
 
39
59
  headers = {'Content-Type' => 'application/json'}
40
60
 
41
- unique_id_key = @event_unique_id_keys.find {|k| data[k]}
42
-
43
61
  payload = {
44
62
  apiKey: @api_key,
45
- eventName: data[@event_name_key],
46
- eventUniqueId: unique_id_key && data[unique_id_key],
47
- properties: flatten_hash(data),
48
- eventTime: DateTime.parse(data[@event_time_key]).rfc3339
63
+ events: events.filter {|data| !@event_filter_key or data[@event_filter_key] != false}.map do |data|
64
+ unique_id_key = @event_unique_id_keys.find {|k| data[k]}
65
+ {
66
+ eventName: data[@event_name_key],
67
+ eventUniqueId: unique_id_key && data[unique_id_key],
68
+ properties: flatten_hash(data),
69
+ eventTime: DateTime.parse(data[@event_time_key]).rfc3339
70
+ }
71
+ end
49
72
  }
50
73
 
51
74
  http = Net::HTTP.new(uri.host, uri.port)
@@ -5,14 +5,33 @@ class IndicativeOutputTest < Test::Unit::TestCase
5
5
  Fluent::Test.setup
6
6
  end
7
7
 
8
- CONFIG = %[
8
+ STREAM_CONFIG = %[
9
9
  api_key INDICATIVE_API_KEY
10
10
  event_name_key event_name
11
11
  event_time_key created_at
12
12
  event_unique_id_keys user_id, session_id
13
13
  ]
14
14
 
15
- def create_driver(conf=CONFIG)
15
+ FILTER_CONFIG = %[
16
+ api_key INDICATIVE_API_KEY
17
+ event_name_key event_name
18
+ event_time_key created_at
19
+ event_unique_id_keys user_id, session_id
20
+ event_filter_key indicative
21
+ ]
22
+
23
+ BUFFER_CONFIG = %[
24
+ api_key INDICATIVE_API_KEY
25
+ event_name_key event_name
26
+ event_time_key created_at
27
+ event_unique_id_keys user_id, session_id
28
+
29
+ <buffer>
30
+ chunk_limit_records 50
31
+ </buffer>
32
+ ]
33
+
34
+ def create_driver(conf=STREAM_CONFIG)
16
35
  Fluent::Test::Driver::Output.new(Fluent::Plugin::IndicativeOutput).configure(conf)
17
36
  end
18
37
 
@@ -20,7 +39,7 @@ class IndicativeOutputTest < Test::Unit::TestCase
20
39
  assert_raise(Fluent::ConfigError) {
21
40
  d = create_driver('')
22
41
  }
23
- d = create_driver CONFIG
42
+ d = create_driver(STREAM_CONFIG)
24
43
  assert_equal 'INDICATIVE_API_KEY', d.instance.api_key
25
44
  assert_equal 'event_name', d.instance.event_name_key
26
45
  assert_equal 'created_at', d.instance.event_time_key
@@ -28,8 +47,8 @@ class IndicativeOutputTest < Test::Unit::TestCase
28
47
  end
29
48
 
30
49
 
31
- def test_emit
32
- d = create_driver(CONFIG)
50
+ def test_emit_stream
51
+ d = create_driver(STREAM_CONFIG)
33
52
  stub_request(:any, d.instance.api_url)
34
53
  d.run(default_tag: 'test') do
35
54
  d.feed({'event_name' => 'screen_view', 'created_at' => '2015-01-01T10:00:00.000Z', 'session_id' => 'a3bd2', 'user_id' => nil, 'screen' => {'id' => 'index'}})
@@ -39,16 +58,93 @@ class IndicativeOutputTest < Test::Unit::TestCase
39
58
  assert_requested :post, d.instance.api_url,
40
59
  headers: {'Content-Type' => 'application/json'}, body: {
41
60
  'apiKey' => 'INDICATIVE_API_KEY',
42
- 'eventName' => 'screen_view',
43
- 'eventUniqueId' => 'a3bd2',
44
- 'properties' => {
45
- 'event_name' => 'screen_view',
46
- 'created_at' => '2015-01-01T10:00:00.000Z',
47
- 'session_id' => 'a3bd2',
48
- 'user_id' => nil,
49
- 'screen.id' => 'index'
50
- },
51
- 'eventTime' => '2015-01-01T10:00:00+00:00'
61
+ 'events' => [{
62
+ 'eventName' => 'screen_view',
63
+ 'eventUniqueId' => 'a3bd2',
64
+ 'properties' => {
65
+ 'event_name' => 'screen_view',
66
+ 'created_at' => '2015-01-01T10:00:00.000Z',
67
+ 'session_id' => 'a3bd2',
68
+ 'user_id' => nil,
69
+ 'screen.id' => 'index'
70
+ },
71
+ 'eventTime' => '2015-01-01T10:00:00+00:00'
72
+ }]
73
+ }.to_json, times: 1
74
+ end
75
+
76
+ def test_emit_stream_with_filter
77
+ d = create_driver(FILTER_CONFIG)
78
+ stub_request(:any, d.instance.api_url)
79
+ d.run(default_tag: 'test') do
80
+ d.feed({'event_name' => 'filter_nil', 'created_at' => '2015-01-01T10:00:00.000Z', 'session_id' => 'a3bd2'})
81
+ d.feed({'event_name' => 'filter_false', 'created_at' => '2015-01-01T10:00:00.000Z', 'session_id' => 'a3bd2', 'indicative' => false})
82
+ d.feed({'event_name' => 'filter_true', 'created_at' => '2015-01-01T10:00:00.000Z', 'session_id' => 'a3bd2', 'indicative' => true})
83
+ end
84
+ events = d.events
85
+ assert_equal 0, events.length
86
+ assert_requested :post, d.instance.api_url,
87
+ headers: {'Content-Type' => 'application/json'}, body: {
88
+ 'apiKey' => 'INDICATIVE_API_KEY',
89
+ 'events' => [{
90
+ 'eventName' => 'filter_nil',
91
+ 'eventUniqueId' => 'a3bd2',
92
+ 'properties' => {
93
+ 'event_name' => 'filter_nil',
94
+ 'created_at' => '2015-01-01T10:00:00.000Z',
95
+ 'session_id' => 'a3bd2'
96
+ },
97
+ 'eventTime' => '2015-01-01T10:00:00+00:00'
98
+ }, {
99
+ 'eventName' => 'filter_true',
100
+ 'eventUniqueId' => 'a3bd2',
101
+ 'properties' => {
102
+ 'event_name' => 'filter_true',
103
+ 'created_at' => '2015-01-01T10:00:00.000Z',
104
+ 'session_id' => 'a3bd2',
105
+ 'indicative' => true
106
+ },
107
+ 'eventTime' => '2015-01-01T10:00:00+00:00'
108
+ }]
109
+ }.to_json, times: 1
110
+ end
111
+
112
+ def test_emit_buffer
113
+ d = create_driver(BUFFER_CONFIG)
114
+ stub_request(:any, d.instance.api_url)
115
+ d.run(default_tag: 'test') do
116
+ 5.times do
117
+ d.feed({'event_name' => 'screen_view', 'created_at' => '2015-01-01T10:00:00.000Z', 'session_id' => 'a3bd2', 'user_id' => nil, 'screen' => {'id' => 'index'}})
118
+ end
119
+ end
120
+ events = d.events
121
+ assert_equal 0, events.length
122
+ assert_requested :post, d.instance.api_url, times: 1
123
+ end
124
+
125
+ def test_key_value_object_transformation
126
+ d = create_driver(STREAM_CONFIG)
127
+ stub_request(:any, d.instance.api_url)
128
+ d.run(default_tag: 'test') do
129
+ d.feed({'event_name' => 'screen_view', 'created_at' => '2015-01-01T10:00:00.000Z', 'session_id' => 'a3bd2', 'experiments': [{'key': 'a', 'value': 1}, {'key': 'b', 'value': 2}]})
130
+ end
131
+ events = d.events
132
+ assert_equal 0, events.length
133
+ assert_requested :post, d.instance.api_url,
134
+ headers: {'Content-Type' => 'application/json'}, body: {
135
+ 'apiKey' => 'INDICATIVE_API_KEY',
136
+ 'events' => [{
137
+ 'eventName' => 'screen_view',
138
+ 'eventUniqueId' => 'a3bd2',
139
+ 'properties' => {
140
+ 'event_name' => 'screen_view',
141
+ 'created_at' => '2015-01-01T10:00:00.000Z',
142
+ 'session_id' => 'a3bd2',
143
+ 'experiments.a' => 1,
144
+ 'experiments.b' => 2
145
+ },
146
+ 'eventTime' => '2015-01-01T10:00:00+00:00'
147
+ }]
52
148
  }.to_json, times: 1
53
149
  end
54
150
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-indicative
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.3
4
+ version: 0.1.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sam Millar
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-09-12 00:00:00.000000000 Z
11
+ date: 2020-09-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -95,7 +95,7 @@ files:
95
95
  - lib/fluent/plugin/out_indicative.rb
96
96
  - test/helper.rb
97
97
  - test/plugin/test_out_indicative.rb
98
- homepage: https://github.com/millar/fluent-plugin-indicative
98
+ homepage: https://github.com/mixcloud/fluent-plugin-indicative
99
99
  licenses: []
100
100
  metadata: {}
101
101
  post_install_message:
@@ -113,8 +113,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
113
113
  - !ruby/object:Gem::Version
114
114
  version: '0'
115
115
  requirements: []
116
- rubyforge_project:
117
- rubygems_version: 2.7.3
116
+ rubygems_version: 3.0.3
118
117
  signing_key:
119
118
  specification_version: 4
120
119
  summary: Fluentd output plugin to send events to Indicative