fluent-plugin-indicative 0.1.3 → 0.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +6 -0
- data/fluent-plugin-indicative.gemspec +1 -1
- data/lib/fluent/plugin/out_indicative.rb +25 -9
- data/test/plugin/test_out_indicative.rb +45 -15
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 77a70ff5f9f75d916912313e9ad5570700461e7c1ad32d425d8424fddff54a99
|
4
|
+
data.tar.gz: b10e2ba2fa650fb0670d9e75023ae8e640a5759dc217d229abb7a3221e494cf8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9895f1104b602c410cbbe022e2704de3b74bc8ff5cc52e6111d35840ea57b56bd03e88ecca0356b3a7c63edd209ce3def48cf6ecca3e12da8d0d87cbb438fa18
|
7
|
+
data.tar.gz: ae64a4fb95fb6d2fb219d691ae9dd6ab10245ba85a84584a06bae4fc2b7c630ddf0f239d5e6c527f01cfa1525d18cb6e66d603ff4854ec7aa62d703a3841773f
|
data/README.md
CHANGED
@@ -12,5 +12,11 @@ Fluentd output plugin to send events to [Indicative](https://www.indicative.com/
|
|
12
12
|
event_name_key event_name
|
13
13
|
event_time_key created_at
|
14
14
|
event_unique_id_keys user_id, cookie_id, session_id # keys to search for unique user ID value, in order of priority
|
15
|
+
|
16
|
+
# Optionally use buffering (recommended for high event volumes)
|
17
|
+
<buffer>
|
18
|
+
path /var/log/td-agent/indicative.buffer
|
19
|
+
chunk_limit_records 1000
|
20
|
+
</buffer>
|
15
21
|
</match>
|
16
22
|
```
|
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-indicative"
|
6
|
-
s.version = "0.1.
|
6
|
+
s.version = "0.1.4"
|
7
7
|
s.authors = ["Sam Millar"]
|
8
8
|
s.email = ["sam@millar.io"]
|
9
9
|
s.homepage = "https://github.com/millar/fluent-plugin-indicative"
|
@@ -5,6 +5,9 @@ require 'uri'
|
|
5
5
|
|
6
6
|
require 'fluent/plugin/output'
|
7
7
|
|
8
|
+
BATCH_SIZE = 100
|
9
|
+
|
10
|
+
|
8
11
|
def flatten_hash(hash)
|
9
12
|
hash.each_with_object({}) do |(k, v), h|
|
10
13
|
if v.is_a? Hash
|
@@ -28,24 +31,37 @@ class Fluent::Plugin::IndicativeOutput < Fluent::Plugin::Output
|
|
28
31
|
config_param :event_unique_id_keys, :array, value_type: :string
|
29
32
|
|
30
33
|
def process(tag, es)
|
31
|
-
es.
|
32
|
-
|
34
|
+
es.each_slice(BATCH_SIZE) do |events|
|
35
|
+
send_events(events.map {|time, record| record})
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def write(chunk)
|
40
|
+
records = []
|
41
|
+
chunk.each do |time, record|
|
42
|
+
records << record
|
43
|
+
end
|
44
|
+
records.each_slice(BATCH_SIZE) do |events|
|
45
|
+
send_events(events)
|
33
46
|
end
|
34
47
|
end
|
35
48
|
|
36
|
-
def
|
49
|
+
def send_events(events)
|
37
50
|
uri = URI.parse(@api_url)
|
38
51
|
|
39
52
|
headers = {'Content-Type' => 'application/json'}
|
40
53
|
|
41
|
-
unique_id_key = @event_unique_id_keys.find {|k| data[k]}
|
42
|
-
|
43
54
|
payload = {
|
44
55
|
apiKey: @api_key,
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
56
|
+
events: events.map do |data|
|
57
|
+
unique_id_key = @event_unique_id_keys.find {|k| data[k]}
|
58
|
+
{
|
59
|
+
eventName: data[@event_name_key],
|
60
|
+
eventUniqueId: unique_id_key && data[unique_id_key],
|
61
|
+
properties: flatten_hash(data),
|
62
|
+
eventTime: DateTime.parse(data[@event_time_key]).rfc3339
|
63
|
+
}
|
64
|
+
end
|
49
65
|
}
|
50
66
|
|
51
67
|
http = Net::HTTP.new(uri.host, uri.port)
|
@@ -5,14 +5,29 @@ class IndicativeOutputTest < Test::Unit::TestCase
|
|
5
5
|
Fluent::Test.setup
|
6
6
|
end
|
7
7
|
|
8
|
-
|
8
|
+
STREAM_CONFIG = %[
|
9
9
|
api_key INDICATIVE_API_KEY
|
10
10
|
event_name_key event_name
|
11
11
|
event_time_key created_at
|
12
12
|
event_unique_id_keys user_id, session_id
|
13
13
|
]
|
14
14
|
|
15
|
-
|
15
|
+
BUFFER_CONFIG = %[
|
16
|
+
api_key INDICATIVE_API_KEY
|
17
|
+
event_name_key event_name
|
18
|
+
event_time_key created_at
|
19
|
+
event_unique_id_keys user_id, session_id
|
20
|
+
|
21
|
+
<buffer>
|
22
|
+
chunk_limit_records 50
|
23
|
+
</buffer>
|
24
|
+
|
25
|
+
<format>
|
26
|
+
@type json
|
27
|
+
</format>
|
28
|
+
]
|
29
|
+
|
30
|
+
def create_driver(conf=STREAM_CONFIG)
|
16
31
|
Fluent::Test::Driver::Output.new(Fluent::Plugin::IndicativeOutput).configure(conf)
|
17
32
|
end
|
18
33
|
|
@@ -20,7 +35,7 @@ class IndicativeOutputTest < Test::Unit::TestCase
|
|
20
35
|
assert_raise(Fluent::ConfigError) {
|
21
36
|
d = create_driver('')
|
22
37
|
}
|
23
|
-
d = create_driver
|
38
|
+
d = create_driver(STREAM_CONFIG)
|
24
39
|
assert_equal 'INDICATIVE_API_KEY', d.instance.api_key
|
25
40
|
assert_equal 'event_name', d.instance.event_name_key
|
26
41
|
assert_equal 'created_at', d.instance.event_time_key
|
@@ -28,8 +43,8 @@ class IndicativeOutputTest < Test::Unit::TestCase
|
|
28
43
|
end
|
29
44
|
|
30
45
|
|
31
|
-
def
|
32
|
-
d = create_driver(
|
46
|
+
def test_emit_stream
|
47
|
+
d = create_driver(STREAM_CONFIG)
|
33
48
|
stub_request(:any, d.instance.api_url)
|
34
49
|
d.run(default_tag: 'test') do
|
35
50
|
d.feed({'event_name' => 'screen_view', 'created_at' => '2015-01-01T10:00:00.000Z', 'session_id' => 'a3bd2', 'user_id' => nil, 'screen' => {'id' => 'index'}})
|
@@ -39,16 +54,31 @@ class IndicativeOutputTest < Test::Unit::TestCase
|
|
39
54
|
assert_requested :post, d.instance.api_url,
|
40
55
|
headers: {'Content-Type' => 'application/json'}, body: {
|
41
56
|
'apiKey' => 'INDICATIVE_API_KEY',
|
42
|
-
'
|
43
|
-
|
44
|
-
|
45
|
-
'
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
57
|
+
'events' => [{
|
58
|
+
'eventName' => 'screen_view',
|
59
|
+
'eventUniqueId' => 'a3bd2',
|
60
|
+
'properties' => {
|
61
|
+
'event_name' => 'screen_view',
|
62
|
+
'created_at' => '2015-01-01T10:00:00.000Z',
|
63
|
+
'session_id' => 'a3bd2',
|
64
|
+
'user_id' => nil,
|
65
|
+
'screen.id' => 'index'
|
66
|
+
},
|
67
|
+
'eventTime' => '2015-01-01T10:00:00+00:00'
|
68
|
+
}]
|
52
69
|
}.to_json, times: 1
|
53
70
|
end
|
71
|
+
|
72
|
+
def test_emit_buffer
|
73
|
+
d = create_driver(BUFFER_CONFIG)
|
74
|
+
stub_request(:any, d.instance.api_url)
|
75
|
+
d.run(default_tag: 'test') do
|
76
|
+
20.times do
|
77
|
+
d.feed({'event_name' => 'screen_view', 'created_at' => '2015-01-01T10:00:00.000Z', 'session_id' => 'a3bd2', 'user_id' => nil, 'screen' => {'id' => 'index'}})
|
78
|
+
end
|
79
|
+
end
|
80
|
+
events = d.events
|
81
|
+
assert_equal 0, events.length
|
82
|
+
assert_requested :post, d.instance.api_url, times: 1
|
83
|
+
end
|
54
84
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-indicative
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sam Millar
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-11-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|