fluent-plugin-chriak2 0.0.7

Sign up to get free protection for your applications and to get access to all the features.
Files changed (12) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +19 -0
  3. data/Gemfile +6 -0
  4. data/README.md +83 -0
  5. data/Rakefile +17 -0
  6. data/VERSION +1 -0
  7. data/browser.html +178 -0
  8. data/fluent-plugin-riak.gemspec +25 -0
  9. data/lib/fluent/plugin/out_chriak2.rb +80 -0
  10. data/listkeys.rb +36 -0
  11. data/setup_index.sh +11 -0
  12. metadata +137 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: a9c2af38a4e19d22efa8d9d22c0cf7e272613b16
4
+ data.tar.gz: 765e4bdda704d21403f018dd3542a8f2b417f8b2
5
+ SHA512:
6
+ metadata.gz: 0ee5c145ab9e91fd52f1e3b09e2a1842abe33764f3c9a4da1a401b75ba9c6761349e5cbd32a19c133c28685ed76951a7686f59d879f9ded05cc66d2f67d1d9be
7
+ data.tar.gz: 5a84647c9c6ae53bd2b95b03363f12aaeacf73a11088ec352552ba5b92e415cca354c95f1c0140b481c415e7d08a7b15d044ad45c5a220b7cc5718ac73a18cb7
data/.gitignore ADDED
@@ -0,0 +1,19 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ coverage
6
+ InstalledFiles
7
+ lib/bundler/man
8
+ pkg
9
+ rdoc
10
+ spec/reports
11
+ test/tmp
12
+ test/version_tmp
13
+ tmp
14
+ *~
15
+ Gemfile.lock
16
+ # YARD artifacts
17
+ .yardoc
18
+ _yardoc
19
+ doc/
data/Gemfile ADDED
@@ -0,0 +1,6 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gemspec
4
+
5
+ gem "simplecov", :require => false
6
+ gem "uuidtools", :require => ">= 2.1.3"
data/README.md ADDED
@@ -0,0 +1,83 @@
1
+ fluent-plugin-riak, a plugin for [Fluentd](http://fluentd.org)
2
+ ==================
3
+
4
+ fluent-plugin-riak is a alternative for people who are not sufficient with mongo or webhdfs. Riak ( http://github.com/basho/riak ) is an open-source distributed KVS focused on availability. It also has a strong query system with secondary index (2i): see docs ( http://docs.basho.com/riak/latest/tutorials/querying/ ) for details.
5
+
6
+ Current status is still proof-of-concept: index setting and its configuration are to be decided. Also performance optimization is required. Another idea is in_tail_riak by using riak post-commit.
7
+
8
+ installation
9
+ ------------
10
+
11
+ ```bash
12
+ $ sudo gem install fluent-plugin-riak
13
+ ```
14
+
15
+ Notice: you need Riak configured using eleveldb as backend.
16
+
17
+
18
+ fluent.conf example
19
+ -------------------
20
+
21
+ ```
22
+ <match riak.**>
23
+ type riak
24
+
25
+ buffer_type memory
26
+ flush_interval 10s
27
+ retry_limit 5
28
+ retry_wait 1s
29
+ buffer_chunk_limit 256m
30
+ buffer_queue_limit 8096
31
+
32
+ # pb port
33
+ nodes 127.0.0.1:8087
34
+ #for cluster, define multiple machines
35
+ #nodes 192.168.100.128:10018 129.168.100.128:10028
36
+ </match>
37
+
38
+ ```
39
+
40
+ - key format -> 2013-02-<uuid>
41
+ - value format -> [records] in JSON
42
+ - index:
43
+
44
+ - year_int -> year
45
+ - month_bin -> <year>-<month>
46
+ - tag_bin -> tags
47
+
48
+ easy querying log
49
+ -----------------
50
+
51
+ ```bash
52
+ $ curl -X PUT http://localhost:8098/buckets/static/keys/browser.html -H 'Content-type: text/html' -d @browser.html
53
+ $ open http://localhost:8098/buckets/static/keys/browser.html
54
+ ```
55
+
56
+ Pros
57
+ ----
58
+
59
+ - easy operations
60
+ - high availability
61
+ - horizontal scalability (esp. write performance)
62
+ - good night sleep
63
+
64
+ Cons
65
+ ----
66
+
67
+ - no capped table, TTL objects
68
+
69
+ TODOs
70
+ -----
71
+
72
+ - refine browser.html query interface with cool features
73
+ - rething index structures
74
+
75
+
76
+ License
77
+ =======
78
+
79
+ Apache 2.0
80
+
81
+ Copyright Kota UENISHI
82
+
83
+ Many Thanks to fluent-plugin-mongo
data/Rakefile ADDED
@@ -0,0 +1,17 @@
1
+ require 'bundler'
2
+ Bundler::GemHelper.install_tasks
3
+
4
+ require 'rake/testtask'
5
+
6
+ Rake::TestTask.new(:test) do |test|
7
+ test.libs << 'lib' << 'test'
8
+ test.test_files = FileList['test/plugin/*.rb']
9
+ test.verbose = true
10
+ end
11
+
12
+ task :coverage do |t|
13
+ ENV['SIMPLE_COV'] = '1'
14
+ Rake::Task["test"].invoke
15
+ end
16
+
17
+ task :default => [:build]
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.0.7
data/browser.html ADDED
@@ -0,0 +1,178 @@
1
+ <html>
2
+ <head>
3
+ <title>Riak mapreducer</title>
4
+ <style type="text/css">
5
+ body {
6
+ background: #678; text-align: center; width: 100%; font-family: 'Titillium', Monospace;
7
+ }
8
+ body div {
9
+ width: 80%; background: #ddd;
10
+ padding: 20px; text-align: left; margin: 0px auto;
11
+ }
12
+ h2 {
13
+ padding: 0px;
14
+ margin: 0px auto;
15
+ }
16
+ a {
17
+ text-shadow: 1px 1px #888888;
18
+ text-decoration:underline;
19
+ }
20
+ </style>
21
+ <script lang="JavaScript">
22
+ function get_map(){ return document.getElementById("map").value; }
23
+ function get_reduce(){ return document.getElementById("reduce").value; }
24
+ function get_bucketname(){ return document.getElementById("bucket").value; }
25
+ function exec_query(m, r){
26
+ var q = { inputs : get_bucketname(),
27
+ query : [
28
+ { map: { language: "javascript", source: m}},
29
+ { reduce: { language: "javascript", source: r}}
30
+ ]};
31
+ var xmlhttp = new XMLHttpRequest();
32
+ xmlhttp.open("POST", "/mapred", false);
33
+ xmlhttp.setRequestHeader("Content-Type", "application/json");
34
+ xmlhttp.send(JSON.stringify(q, null, ""));
35
+ var result = xmlhttp.responseText;
36
+
37
+ var resultBox = document.getElementById("result");
38
+ resultBox.innerText = result;
39
+ }
40
+ </script>
41
+ </head>
42
+
43
+ <body>
44
+ <div id="content">
45
+ <div style="text-align:right">
46
+ <script lang="JavaScript">
47
+ </script>
48
+ bucket name:<input type="text" id="bucket" value="fluentlog"/>
49
+ </div>
50
+
51
+ <h2>Run mapreduce on fluentd log</h2>
52
+ <table style="border: 1px solid black;">
53
+ <tr>
54
+ <th>map</th> <th>reduce</th>
55
+ </tr>
56
+ <tr>
57
+ <td>
58
+ <textarea name="map" id="map" rows="16" cols="50">
59
+ function(v){
60
+ var data = v.values[0].data;
61
+ var c = JSON.parse(data);
62
+ return [c.count];
63
+ }
64
+ </textarea>
65
+ </td><td>
66
+ <textarea name="reduce" id="reduce" rows="16" cols="50" >
67
+ function(v){
68
+ var sum = 0;
69
+ for( var i=0; i < v.length; ++i){
70
+ sum += v[i];
71
+ }
72
+ return [sum];
73
+ }
74
+ </textarea>
75
+ </td>
76
+ </tr>
77
+ <tr>
78
+ <th colspan="2"> <a onClick="exec_query(get_map(), get_reduce());">query</a> </th>
79
+ </tr>
80
+ <tr>
81
+ <td width="200px" colspan="2">
82
+ <div id="result" style="font: bold;"></div>
83
+ </td>
84
+ </tr>
85
+ <tr><td colspan="3">
86
+ <script lang="JavaScript">
87
+ function grep(){
88
+
89
+ }
90
+ </script>
91
+ <script lang="JavaScript" type="text/javascript">
92
+ function count_rank(){
93
+ var m = "function(v){
94
+ var data = v.values[0].data;
95
+ var c = JSON.parse(data);
96
+ var tags = {};
97
+ for(var i=0; i<c.length; ++i){
98
+ count = tags[c[i].tag];
99
+ if(count == null){ tags[c[i].tag] = 1; }
100
+ else{ tags[c[i].tag] = count+1; }
101
+ }
102
+ return [tags];
103
+ }";
104
+ var r = "function(v){
105
+ var tags = {};
106
+ for(var i=0; i<v.length; ++i){
107
+ for (var prop in v[i]) {
108
+ count = tags[prop];
109
+ if(count == null){ tags[prop] = v[i][prop]; }
110
+ else{ tags[prop] = count+v[i][prop]; }
111
+ }
112
+ }
113
+ return [tags];
114
+ }";
115
+ exec_query(m, r);
116
+ }
117
+ </script>
118
+
119
+ <!-- <a onClick="grep();">grep</a> <input type="text" id="grep" value="debug"/> -->
120
+ <a onClick="count_rank();">count&amp;rank</a> <input type="text" id="count_rank" value="tag"/>
121
+ </td></tr>
122
+ </table>
123
+
124
+ <hr/>
125
+ <h2>Riak object viewer</h2>
126
+ <script lang="JavaScript">
127
+ function get(){
128
+ var key = document.getElementById("key").value;
129
+ var xmlhttp = new XMLHttpRequest();
130
+ xmlhttp.open("GET", "/buckets/"+ get_bucketname() +"/keys/"+key, false);
131
+ xmlhttp.send();
132
+ var result = xmlhttp.responseText;
133
+ console.log(result);
134
+ document.getElementById("object").innerText = result;
135
+ }
136
+ </script>
137
+ <a onClick="get();" >get</a> <input type="text" id="key" />
138
+ <div id="object"></div>
139
+
140
+
141
+ <h2>Riak keylister</h2>
142
+ <script lang="JavaScript">
143
+ function listkeys(){
144
+ var xmlhttp = new XMLHttpRequest();
145
+ xmlhttp.open("GET", "/buckets/"+ get_bucketname() +"/keys?keys=true", false);
146
+ xmlhttp.send();
147
+ var result = JSON.parse(xmlhttp.responseText);
148
+ console.log(result);
149
+
150
+ document.getElementById("keylist").innerText = result.keys.sort().join("\n");
151
+ }
152
+ </script>
153
+ <a onClick="listkeys();" >listkeys</a>
154
+ <div id="keylist"></div>
155
+ <hr/>
156
+ (C)@kuenishi distributed under Apache 2 license
157
+ </div>
158
+ <!--
159
+ <h2>Riak object putter</h2>
160
+ <script lang="JavaScript">
161
+ function put(){
162
+ var key = document.getElementById("key2").value;
163
+ var value = document.getElementById("value").value;
164
+ var xmlhttp = new XMLHttpRequest();
165
+ xmlhttp.open("PUT", "/buckets/fluentlog/keys/"+key, false);
166
+ xmlhttp.setRequestHeader("Content-Type", "application/json");
167
+ xmlhttp.send(value);
168
+ var result = xmlhttp.responseText;
169
+ console.log(result);
170
+ document.getElementById("object2").innerText = result;
171
+ }
172
+ </script>
173
+ <a onClick="put();" >put</a> <input type="text" id="key2" /> <input type="text" id="value" />
174
+ <div id="object2"></div>
175
+ -->
176
+
177
+ </body>
178
+ </html>
data/fluent-plugin-riak.gemspec ADDED
@@ -0,0 +1,25 @@
1
+ # encoding: utf-8
2
+ $:.push File.expand_path('../lib', __FILE__)
3
+
4
+ Gem::Specification.new do |gem|
5
+ gem.name = "fluent-plugin-chriak2"
6
+ gem.description = "Riak 2 plugin for Fluent event collector"
7
+ gem.homepage = "https://github.com/collectivehealth/fluent-plugin-riak"
8
+ gem.summary = gem.description
9
+ gem.version = File.read("VERSION").strip
10
+ gem.authors = ["Kota UENISHI", "Matt Nunogawa"]
11
+ gem.email = "matt@collectivehealth.com"
12
+ gem.has_rdoc = false
13
+ #gem.platform = Gem::Platform::RUBY
14
+ gem.files = `git ls-files`.split("\n")
15
+ gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
16
+ gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
17
+ gem.require_paths = ['lib']
18
+
19
+ gem.add_dependency "fluentd", "~> 0.10"
20
+ gem.add_dependency "riak-client", "~> 2.1.0"
21
+ gem.add_dependency "uuidtools", ">= 2.1.3"
22
+ gem.add_development_dependency "rake", ">= 0.9.2"
23
+ gem.add_development_dependency "simplecov", ">= 0.5.4"
24
+ gem.add_development_dependency "rr", ">= 1.0.0"
25
+ end
data/lib/fluent/plugin/out_chriak2.rb ADDED
@@ -0,0 +1,80 @@
1
+ module Fluent
2
+
3
+ class Chriak2Output < BufferedOutput
4
+
5
+ Fluent::Plugin.register_output('chriak2', self)
6
+ include SetTimeKeyMixin
7
+ config_set_default :include_tag_key, true
8
+ include SetTagKeyMixin
9
+ config_set_default :include_time_key, true
10
+
11
+ config_param :bucket_name, :string, :default => "fluentlog"
12
+ config_param :nodes, :string, :default => "localhost:8087"
13
+
14
+ def initialize
15
+ super
16
+ require 'riak'
17
+ require 'msgpack'
18
+ require 'uuidtools'
19
+ end
20
+
21
+ def configure(conf)
22
+ super
23
+
24
+ @nodes = @nodes.split(',').map{ |s|
25
+ ip,port = s.split(':')
26
+ {:host => ip, :pb_port => port.to_i}
27
+ }
28
+ $log.info "riak nodes=#{@nodes}"
29
+ end
30
+
31
+ def start
32
+ $log.debug " => #{@buffer.chunk_limit} #{@buffer.queue_limit} "
33
+ @conn = Riak::Client.new(:nodes => @nodes, :protocol => "pbc")
34
+ @bucket = @conn.bucket(@bucket_name)
35
+ @buf = {}
36
+
37
+ super
38
+ end
39
+
40
+ def format(tag, time, record)
41
+ [time, tag, record].to_msgpack
42
+ end
43
+
44
+ def write(chunk)
45
+ $log.debug " <<<<<===========\n"
46
+
47
+ records = []
48
+ tags = []
49
+ chunk.msgpack_each { |time, tag, record|
50
+ record[@tag_key] = tag
51
+ tags << tag
52
+ records << record
53
+ $log.debug record
54
+ }
55
+ put_now(records, tags)
56
+ end
57
+
58
+ private
59
+
60
+ # TODO: add index for some analysis
61
+ def put_now(records, tags)
62
+ if not records.empty? then
63
+ today = Date.today
64
+ key = "#{today.to_s}-#{UUIDTools::UUID.timestamp_create.to_s}"
65
+ robj = Riak::RObject.new(@bucket, key)
66
+ robj.raw_data = records.to_json
67
+ robj.indexes['year_int'] << today.year
68
+ robj.indexes['month_bin'] << "#{today.year}-#{"%02d" % today.month}"
69
+ tags.each do |tag|
70
+ robj.indexes['tag_bin'] << tag
71
+ end
72
+ robj.content_type = 'application/json'
73
+ robj.store
74
+ robj
75
+ end
76
+ end
77
+
78
+ end
79
+
80
+ end
data/listkeys.rb ADDED
@@ -0,0 +1,36 @@
1
+ require 'riak'
2
+
3
+ nodes = [{ :host => "127.0.0.1",
4
+ :pb_port => 8087 }]
5
+
6
+
7
+ c = Riak::Client.new( :nodes => nodes, :protocol => "pbc" )
8
+ #c.list_buckets
9
+ b = c.bucket("fluentlog")
10
+
11
+ def put(bucket)
12
+ o = Riak::RObject.new(bucket, "hoge")
13
+ o.content_type = "text/plain"
14
+ o.raw_data = "hogehoge"
15
+ o.store
16
+ end
17
+
18
+ def get(bucket)
19
+ o = bucket.get("hoge")
20
+ p o.content_type
21
+ p o.raw_data
22
+ end
23
+
24
+ def keys(bucket)
25
+ bucket.keys { |ks|
26
+ ks.each { |k|
27
+ o = bucket.get(k)
28
+ print "#{o.bucket.name}/#{o.key} (#{o.content_type})\n"
29
+ print " => #{o.raw_data}\n"
30
+ }
31
+ }
32
+ end
33
+
34
+ #put b
35
+ #get b
36
+ keys b
data/setup_index.sh ADDED
@@ -0,0 +1,11 @@
1
+ #!/bin/sh
2
+
3
+ HOST=$1
4
+
5
+ echo setting up fluentlog_index at $HOST
6
+ curl -X PUT http://${HOST}/yz/index/fluentlog_index
7
+ echo "setting fluentlog_index to fluentlog bucket"
8
+ curl -X PUT http://${HOST}/buckets/fluentlog/props -H 'Content-type:application/json' -d '{"props":{"yz_index":"fluentlog_index"}}'
9
+
10
+ curl http://${HOST}/yz/index?index=true
11
+ curl http://${HOST}/buckets/fluentlog/props
metadata ADDED
@@ -0,0 +1,137 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-chriak2
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.7
5
+ platform: ruby
6
+ authors:
7
+ - Kota UENISHI
8
+ - Matt Nunogawa
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2014-11-29 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: fluentd
16
+ requirement: !ruby/object:Gem::Requirement
17
+ requirements:
18
+ - - ~>
19
+ - !ruby/object:Gem::Version
20
+ version: '0.10'
21
+ type: :runtime
22
+ prerelease: false
23
+ version_requirements: !ruby/object:Gem::Requirement
24
+ requirements:
25
+ - - ~>
26
+ - !ruby/object:Gem::Version
27
+ version: '0.10'
28
+ - !ruby/object:Gem::Dependency
29
+ name: riak-client
30
+ requirement: !ruby/object:Gem::Requirement
31
+ requirements:
32
+ - - ~>
33
+ - !ruby/object:Gem::Version
34
+ version: 2.1.0
35
+ type: :runtime
36
+ prerelease: false
37
+ version_requirements: !ruby/object:Gem::Requirement
38
+ requirements:
39
+ - - ~>
40
+ - !ruby/object:Gem::Version
41
+ version: 2.1.0
42
+ - !ruby/object:Gem::Dependency
43
+ name: uuidtools
44
+ requirement: !ruby/object:Gem::Requirement
45
+ requirements:
46
+ - - '>='
47
+ - !ruby/object:Gem::Version
48
+ version: 2.1.3
49
+ type: :runtime
50
+ prerelease: false
51
+ version_requirements: !ruby/object:Gem::Requirement
52
+ requirements:
53
+ - - '>='
54
+ - !ruby/object:Gem::Version
55
+ version: 2.1.3
56
+ - !ruby/object:Gem::Dependency
57
+ name: rake
58
+ requirement: !ruby/object:Gem::Requirement
59
+ requirements:
60
+ - - '>='
61
+ - !ruby/object:Gem::Version
62
+ version: 0.9.2
63
+ type: :development
64
+ prerelease: false
65
+ version_requirements: !ruby/object:Gem::Requirement
66
+ requirements:
67
+ - - '>='
68
+ - !ruby/object:Gem::Version
69
+ version: 0.9.2
70
+ - !ruby/object:Gem::Dependency
71
+ name: simplecov
72
+ requirement: !ruby/object:Gem::Requirement
73
+ requirements:
74
+ - - '>='
75
+ - !ruby/object:Gem::Version
76
+ version: 0.5.4
77
+ type: :development
78
+ prerelease: false
79
+ version_requirements: !ruby/object:Gem::Requirement
80
+ requirements:
81
+ - - '>='
82
+ - !ruby/object:Gem::Version
83
+ version: 0.5.4
84
+ - !ruby/object:Gem::Dependency
85
+ name: rr
86
+ requirement: !ruby/object:Gem::Requirement
87
+ requirements:
88
+ - - '>='
89
+ - !ruby/object:Gem::Version
90
+ version: 1.0.0
91
+ type: :development
92
+ prerelease: false
93
+ version_requirements: !ruby/object:Gem::Requirement
94
+ requirements:
95
+ - - '>='
96
+ - !ruby/object:Gem::Version
97
+ version: 1.0.0
98
+ description: Riak 2 plugin for Fluent event collector
99
+ email: matt@collectivehealth.com
100
+ executables: []
101
+ extensions: []
102
+ extra_rdoc_files: []
103
+ files:
104
+ - .gitignore
105
+ - Gemfile
106
+ - README.md
107
+ - Rakefile
108
+ - VERSION
109
+ - browser.html
110
+ - fluent-plugin-riak.gemspec
111
+ - lib/fluent/plugin/out_chriak2.rb
112
+ - listkeys.rb
113
+ - setup_index.sh
114
+ homepage: https://github.com/collectivehealth/fluent-plugin-riak
115
+ licenses: []
116
+ metadata: {}
117
+ post_install_message:
118
+ rdoc_options: []
119
+ require_paths:
120
+ - lib
121
+ required_ruby_version: !ruby/object:Gem::Requirement
122
+ requirements:
123
+ - - '>='
124
+ - !ruby/object:Gem::Version
125
+ version: '0'
126
+ required_rubygems_version: !ruby/object:Gem::Requirement
127
+ requirements:
128
+ - - '>='
129
+ - !ruby/object:Gem::Version
130
+ version: '0'
131
+ requirements: []
132
+ rubyforge_project:
133
+ rubygems_version: 2.0.14
134
+ signing_key:
135
+ specification_version: 4
136
+ summary: Riak 2 plugin for Fluent event collector
137
+ test_files: []