fluent-plugin-eventcounter 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +93 -7
- data/fluent-plugin-eventcounter.gemspec +2 -2
- data/lib/fluent/plugin/out_eventcounter.rb +1 -2
- metadata +3 -3
data/README.md
CHANGED
@@ -1,12 +1,49 @@
|
|
1
1
|
fluent-plugin-eventcounter
|
2
2
|
==========================
|
3
3
|
|
4
|
-
This plugin is designed to count occurrences of unique values in a specified key and pass them through as counts either as a re-emit or by directly incrementing a specified redis hash.
|
4
|
+
This plugin is designed to count occurrences of unique values in a specified key and pass them through as counts either as a re-emit or by directly incrementing a specified redis hash. As it's a buffered plugin it will write or re-emit at a (tunable) sane pace.
|
5
5
|
|
6
|
-
|
7
|
-
------------
|
6
|
+
##Example
|
8
7
|
|
9
|
-
|
8
|
+
Given a set of input like
|
9
|
+
|
10
|
+
```
|
11
|
+
important.thing.12086 { 'time': 1413544800, 'event': 'seen', 'user_id': 12345 }
|
12
|
+
important.thing.1337 { 'time': 1413544890, 'event': 'seen', 'user_id': 1337 }
|
13
|
+
important.thing.12086 { 'time': 1413544830, 'event': 'liked', 'user_id': 33864 }
|
14
|
+
important.thing.12086 { 'time': 1413544860, 'event': 'clicked', 'user_id': 12345, url: 'http://example.com/promote?someParam=something' }
|
15
|
+
important.thing.12086 { 'time': 1413544890, 'event': 'seen', 'user_id': 40555 }
|
16
|
+
important.thing.12086 { 'time': 1413544860, 'event': 'clicked', 'user_id': 12345, url: 'http://example.com/promote?someParam=somethingElse' }
|
17
|
+
```
|
18
|
+
|
19
|
+
With a conf like
|
20
|
+
|
21
|
+
```
|
22
|
+
<match important.thing.*>
|
23
|
+
type eventcounter
|
24
|
+
count_key event
|
25
|
+
input_tag_exclude important.thing.
|
26
|
+
|
27
|
+
capture_extra_if url
|
28
|
+
capture_extra_replace \?.*$
|
29
|
+
|
30
|
+
emit_only true
|
31
|
+
emit_tag event.counts
|
32
|
+
</match>
|
33
|
+
```
|
34
|
+
|
35
|
+
You would get
|
36
|
+
|
37
|
+
```
|
38
|
+
event.counts { 12086: { 'seen': 2, 'liked': 1, 'clicked:http://example.com/12086': 1 } }
|
39
|
+
event.counts { 1337: { 'seen': 1 } }
|
40
|
+
```
|
41
|
+
|
42
|
+
If the plugin is not set to emit only, and redis is properly configured each of those counts increment a key in a hash specified by `redis_output_key`:`tag` `count_key`
|
43
|
+
|
44
|
+
##Installation
|
45
|
+
|
46
|
+
OSX
|
10
47
|
|
11
48
|
/opt/td-agent/embedded/bin/gem install fluent-plugin-eventcounter
|
12
49
|
|
@@ -15,8 +52,57 @@ or
|
|
15
52
|
fluent-gem install fluent-plugin-eventcounter
|
16
53
|
|
17
54
|
|
18
|
-
Configuration
|
19
|
-
|
55
|
+
##Configuration
|
56
|
+
|
57
|
+
###Parameters
|
58
|
+
|
59
|
+
- **count_key** (**required**)
|
60
|
+
- The key within the record to count unique instances of
|
61
|
+
- *eg. event*
|
62
|
+
|
63
|
+
- **input_tag_exclude** (optional)
|
64
|
+
- A pattern to exclude from the incoming tag
|
65
|
+
- *default: ''*
|
66
|
+
|
67
|
+
- **emit_only** (optional) - *boolean*
|
68
|
+
- Skip redis and re-emit using emit_to
|
69
|
+
- *default: false*
|
70
|
+
|
71
|
+
- **emit_to** (optional) - *string*
|
72
|
+
- Tag to re-emit with
|
73
|
+
- *default: debug.events*
|
74
|
+
|
75
|
+
- **redis_host** (optional) - *string*
|
76
|
+
- Host address of the redis server
|
77
|
+
- *default: localhost*
|
78
|
+
|
79
|
+
- **redis_port** (optional)
|
80
|
+
- Port of the redis server
|
81
|
+
- *default: 6379*
|
82
|
+
|
83
|
+
- **redis_password** (optional)
|
84
|
+
- Password for the redis server
|
85
|
+
- *default: nil*
|
86
|
+
|
87
|
+
- **redis_db_number** (optional)
|
88
|
+
- Redis DB
|
89
|
+
- *default: 0*
|
90
|
+
|
91
|
+
- **redis_output_key** (optional)
|
92
|
+
- The key to prefix against the tag
|
93
|
+
- *default: ''*
|
94
|
+
|
95
|
+
- **capture_extra_if** (optional)
|
96
|
+
- An additional field to attach to the captured key
|
97
|
+
- *default: nil*
|
98
|
+
|
99
|
+
- **capture_extra_replace** (optional)
|
100
|
+
- A regular expression to replace a portion of the extra capture
|
101
|
+
- *default: ''*
|
102
|
+
|
103
|
+
- **flush_interval** (optional)
|
104
|
+
- Provided from **Fluent::BufferedOutput** time in seconds between flushes
|
105
|
+
- *default: 60*
|
106
|
+
|
20
107
|
|
21
108
|
|
22
|
-
|
@@ -1,10 +1,10 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
2
|
Gem::Specification.new do |gem|
|
3
3
|
gem.name = "fluent-plugin-eventcounter"
|
4
|
-
gem.version = "0.0.
|
4
|
+
gem.version = "0.0.3"
|
5
5
|
gem.authors = ["Sean Dick", "Change.org"]
|
6
6
|
gem.email = ["sean@change.org"]
|
7
|
-
gem.homepage = "https://github.com/
|
7
|
+
gem.homepage = "https://github.com/change/fluent-plugin-eventcounter"
|
8
8
|
gem.summary = %q{Fluentd plugin to count occurences of values in a field and emit them or write them to redis}
|
9
9
|
gem.description = %q{Fluentd plugin to count occurences of values in a field and emit them or write them to redis}
|
10
10
|
gem.license = "MIT"
|
@@ -18,8 +18,7 @@ class Fluent::EventCounterOutput < Fluent::BufferedOutput
|
|
18
18
|
config_param :capture_extra_if, :string, :default => nil
|
19
19
|
config_param :capture_extra_replace, :string, :default => ''
|
20
20
|
|
21
|
-
config_param :count_key, :string
|
22
|
-
config_param :flush_interval, :time, :default => 10
|
21
|
+
config_param :count_key, :string # REQUIRED
|
23
22
|
|
24
23
|
attr_accessor :counts
|
25
24
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-eventcounter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2014-10-
|
13
|
+
date: 2014-10-17 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: fluentd
|
@@ -108,7 +108,7 @@ files:
|
|
108
108
|
- lib/fluent/plugin/out_eventcounter.rb
|
109
109
|
- spec/out_eventcounter_spec.rb
|
110
110
|
- spec/spec_helper.rb
|
111
|
-
homepage: https://github.com/
|
111
|
+
homepage: https://github.com/change/fluent-plugin-eventcounter
|
112
112
|
licenses:
|
113
113
|
- MIT
|
114
114
|
post_install_message:
|