fluent-plugin-riak 0.0.2 → 0.0.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (9) hide show
  1. checksums.yaml +7 -0
  2. data/Gemfile +1 -1
  3. data/README.md +31 -7
  4. data/VERSION +1 -1
  5. data/browser.html +178 -0
  6. data/lib/fluent/plugin/out_riak.rb +15 -12
  7. data/setup_index.sh +11 -0
  8. metadata +22 -41
  9. data/loader.rb +0 -21
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 6013e6377c7ac209a511398f26574ff9044a5f6b
4
+ data.tar.gz: 79a4bd4445aba19bdcf2700ba3693b83d34be945
5
+ SHA512:
6
+ metadata.gz: 716b604d26eba39bbf19398614e823cacb9cb6e899bd08bac69da6e132639d25b381ce7fb10581bd70c8441844c654c3599c154a3a1ac2cc899df102cfe3fbb4
7
+ data.tar.gz: 5482f01385ad0bb914ae48289e025f6158e223ce21a75139097b3ec43ecdeedc0c1313d47adbff7337ebc7632a84ea6d8a9d4f1618d0e2ef3045ec10bec92643
data/Gemfile CHANGED
@@ -1,4 +1,4 @@
1
- source :rubygems
1
+ source 'https://rubygems.org'
2
2
 
3
3
  gemspec
4
4
 
data/README.md CHANGED
@@ -1,10 +1,19 @@
1
- fluent-plugin-riak
1
+ fluent-plugin-riak, a plugin for [Fluentd](http://fluentd.org)
2
2
  ==================
3
3
 
4
4
  fluent-plugin-riak is a alternative for people who are not sufficient with mongo or webhdfs. Riak ( http://github.com/basho/riak ) is an open-source distributed KVS focused on availability. It also has a strong query system with secondary index (2i): see docs ( http://docs.basho.com/riak/latest/tutorials/querying/ ) for details.
5
5
 
6
6
  Current status is still proof-of-concept: index setting and its configuration are to be decided. Also performance optimization is required. Another idea is in_tail_riak by using riak post-commit.
7
7
 
8
+ installation
9
+ ------------
10
+
11
+ ```bash
12
+ $ sudo gem install fluent-plugin-riak
13
+ ```
14
+
15
+ Notice: you need Riak configured using eleveldb as backend.
16
+
8
17
 
9
18
  fluent.conf example
10
19
  -------------------
@@ -28,12 +37,21 @@ fluent.conf example
28
37
 
29
38
  ```
30
39
 
31
- key format -> 2013-02-<uuid>
32
- value format -> [records] in JSON
33
- index:
34
- year_int -> year
35
- month_bin -> <year>-<month>
36
- tag_bin -> tags
40
+ - key format -> 2013-02-<uuid>
41
+ - value format -> [records] in JSON
42
+ - index:
43
+
44
+ - year_int -> year
45
+ - month_bin -> <year>-<month>
46
+ - tag_bin -> tags
47
+
48
+ easy querying log
49
+ -----------------
50
+
51
+ ```bash
52
+ $ curl -X PUT http://localhost:8098/buckets/static/keys/browser.html -H 'Content-type: text/html' -d @browser.html
53
+ $ open http://localhost:8098/buckets/static/keys/browser.html
54
+ ```
37
55
 
38
56
  Pros
39
57
  ----
@@ -48,6 +66,12 @@ Cons
48
66
 
49
67
  - no capped table, TTL objects
50
68
 
69
+ TODOs
70
+ -----
71
+
72
+ - refine browser.html query interface with cool features
73
+ - rething index structures
74
+
51
75
 
52
76
  License
53
77
  =======
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.0.2
1
+ 0.0.3
data/browser.html ADDED
@@ -0,0 +1,178 @@
1
+ <html>
2
+ <head>
3
+ <title>Riak mapreducer</title>
4
+ <style type="text/css">
5
+ body {
6
+ background: #678; text-align: center; width: 100%; font-family: 'Titillium', Monospace;
7
+ }
8
+ body div {
9
+ width: 80%; background: #ddd;
10
+ padding: 20px; text-align: left; margin: 0px auto;
11
+ }
12
+ h2 {
13
+ padding: 0px;
14
+ margin: 0px auto;
15
+ }
16
+ a {
17
+ text-shadow: 1px 1px #888888;
18
+ text-decoration:underline;
19
+ }
20
+ </style>
21
+ <script lang="JavaScript">
22
+ function get_map(){ return document.getElementById("map").value; }
23
+ function get_reduce(){ return document.getElementById("reduce").value; }
24
+ function get_bucketname(){ return document.getElementById("bucket").value; }
25
+ function exec_query(m, r){
26
+ var q = { inputs : get_bucketname(),
27
+ query : [
28
+ { map: { language: "javascript", source: m}},
29
+ { reduce: { language: "javascript", source: r}}
30
+ ]};
31
+ var xmlhttp = new XMLHttpRequest();
32
+ xmlhttp.open("POST", "/mapred", false);
33
+ xmlhttp.setRequestHeader("Content-Type", "application/json");
34
+ xmlhttp.send(JSON.stringify(q, null, ""));
35
+ var result = xmlhttp.responseText;
36
+
37
+ var resultBox = document.getElementById("result");
38
+ resultBox.innerText = result;
39
+ }
40
+ </script>
41
+ </head>
42
+
43
+ <body>
44
+ <div id="content">
45
+ <div style="text-align:right">
46
+ <script lang="JavaScript">
47
+ </script>
48
+ bucket name:<input type="text" id="bucket" value="fluentlog"/>
49
+ </div>
50
+
51
+ <h2>Run mapreduce on fluentd log</h2>
52
+ <table style="border: 1px solid black;">
53
+ <tr>
54
+ <th>map</th> <th>reduce</th>
55
+ </tr>
56
+ <tr>
57
+ <td>
58
+ <textarea name="map" id="map" rows="16" cols="50">
59
+ function(v){
60
+ var data = v.values[0].data;
61
+ var c = JSON.parse(data);
62
+ return [c.count];
63
+ }
64
+ </textarea>
65
+ </td><td>
66
+ <textarea name="reduce" id="reduce" rows="16" cols="50" >
67
+ function(v){
68
+ var sum = 0;
69
+ for( var i=0; i < v.length; ++i){
70
+ sum += v[i];
71
+ }
72
+ return [sum];
73
+ }
74
+ </textarea>
75
+ </td>
76
+ </tr>
77
+ <tr>
78
+ <th colspan="2"> <a onClick="exec_query(get_map(), get_reduce());">query</a> </th>
79
+ </tr>
80
+ <tr>
81
+ <td width="200px" colspan="2">
82
+ <div id="result" style="font: bold;"></div>
83
+ </td>
84
+ </tr>
85
+ <tr><td colspan="3">
86
+ <script lang="JavaScript">
87
+ function grep(){
88
+
89
+ }
90
+ </script>
91
+ <script lang="JavaScript" type="text/javascript">
92
+ function count_rank(){
93
+ var m = "function(v){
94
+ var data = v.values[0].data;
95
+ var c = JSON.parse(data);
96
+ var tags = {};
97
+ for(var i=0; i<c.length; ++i){
98
+ count = tags[c[i].tag];
99
+ if(count == null){ tags[c[i].tag] = 1; }
100
+ else{ tags[c[i].tag] = count+1; }
101
+ }
102
+ return [tags];
103
+ }";
104
+ var r = "function(v){
105
+ var tags = {};
106
+ for(var i=0; i<v.length; ++i){
107
+ for (var prop in v[i]) {
108
+ count = tags[prop];
109
+ if(count == null){ tags[prop] = v[i][prop]; }
110
+ else{ tags[prop] = count+v[i][prop]; }
111
+ }
112
+ }
113
+ return [tags];
114
+ }";
115
+ exec_query(m, r);
116
+ }
117
+ </script>
118
+
119
+ <!-- <a onClick="grep();">grep</a> <input type="text" id="grep" value="debug"/> -->
120
+ <a onClick="count_rank();">count&amp;rank</a> <input type="text" id="count_rank" value="tag"/>
121
+ </td></tr>
122
+ </table>
123
+
124
+ <hr/>
125
+ <h2>Riak object viewer</h2>
126
+ <script lang="JavaScript">
127
+ function get(){
128
+ var key = document.getElementById("key").value;
129
+ var xmlhttp = new XMLHttpRequest();
130
+ xmlhttp.open("GET", "/buckets/"+ get_bucketname() +"/keys/"+key, false);
131
+ xmlhttp.send();
132
+ var result = xmlhttp.responseText;
133
+ console.log(result);
134
+ document.getElementById("object").innerText = result;
135
+ }
136
+ </script>
137
+ <a onClick="get();" >get</a> <input type="text" id="key" />
138
+ <div id="object"></div>
139
+
140
+
141
+ <h2>Riak keylister</h2>
142
+ <script lang="JavaScript">
143
+ function listkeys(){
144
+ var xmlhttp = new XMLHttpRequest();
145
+ xmlhttp.open("GET", "/buckets/"+ get_bucketname() +"/keys?keys=true", false);
146
+ xmlhttp.send();
147
+ var result = JSON.parse(xmlhttp.responseText);
148
+ console.log(result);
149
+
150
+ document.getElementById("keylist").innerText = result.keys.sort().join("\n");
151
+ }
152
+ </script>
153
+ <a onClick="listkeys();" >listkeys</a>
154
+ <div id="keylist"></div>
155
+ <hr/>
156
+ (C)@kuenishi distributed under Apache 2 license
157
+ </div>
158
+ <!--
159
+ <h2>Riak object putter</h2>
160
+ <script lang="JavaScript">
161
+ function put(){
162
+ var key = document.getElementById("key2").value;
163
+ var value = document.getElementById("value").value;
164
+ var xmlhttp = new XMLHttpRequest();
165
+ xmlhttp.open("PUT", "/buckets/fluentlog/keys/"+key, false);
166
+ xmlhttp.setRequestHeader("Content-Type", "application/json");
167
+ xmlhttp.send(value);
168
+ var result = xmlhttp.responseText;
169
+ console.log(result);
170
+ document.getElementById("object2").innerText = result;
171
+ }
172
+ </script>
173
+ <a onClick="put();" >put</a> <input type="text" id="key2" /> <input type="text" id="value" />
174
+ <div id="object2"></div>
175
+ -->
176
+
177
+ </body>
178
+ </html>
data/lib/fluent/plugin/out_riak.rb CHANGED
@@ -8,6 +8,7 @@ class RiakOutput < BufferedOutput
8
8
  include SetTagKeyMixin
9
9
  config_set_default :include_time_key, true
10
10
 
11
+ config_param :bucket_name, :string, :default => "fluentlog"
11
12
  config_param :nodes, :string, :default => "localhost:8087"
12
13
 
13
14
  def initialize
@@ -30,7 +31,7 @@ class RiakOutput < BufferedOutput
30
31
  def start
31
32
  $log.debug " => #{@buffer.chunk_limit} #{@buffer.queue_limit} "
32
33
  @conn = Riak::Client.new(:nodes => @nodes, :protocol => "pbc")
33
- @bucket = @conn.bucket("fluentlog")
34
+ @bucket = @conn.bucket(@bucket_name)
34
35
  @buf = {}
35
36
 
36
37
  super
@@ -58,18 +59,20 @@ class RiakOutput < BufferedOutput
58
59
 
59
60
  # TODO: add index for some analysis
60
61
  def put_now(records, tags)
61
- today = Date.today
62
- key = "#{today.to_s}-#{UUIDTools::UUID.random_create.to_s}"
63
- robj = Riak::RObject.new(@bucket, key)
64
- robj.raw_data = records.to_json
65
- robj.indexes['year_int'] << today.year
66
- robj.indexes['month_bin'] << "#{today.year}-#{today.month}"
67
- tags.each do |tag|
68
- robj.indexes['tag_bin'] << tag
62
+ if not records.empty? then
63
+ today = Date.today
64
+ key = "#{today.to_s}-#{UUIDTools::UUID.random_create.to_s}"
65
+ robj = Riak::RObject.new(@bucket, key)
66
+ robj.raw_data = records.to_json
67
+ robj.indexes['year_int'] << today.year
68
+ robj.indexes['month_bin'] << "#{today.year}-#{"%02d" % today.month}"
69
+ tags.each do |tag|
70
+ robj.indexes['tag_bin'] << tag
71
+ end
72
+ robj.content_type = 'application/json'
73
+ robj.store
74
+ robj
69
75
  end
70
- robj.content_type = 'application/json'
71
- robj.store
72
- robj
73
76
  end
74
77
 
75
78
  end
data/setup_index.sh ADDED
@@ -0,0 +1,11 @@
1
+ #!/bin/sh
2
+
3
+ HOST=$1
4
+
5
+ echo setting up fluentlog_index at $HOST
6
+ curl -X PUT http://${HOST}/yz/index/fluentlog_index
7
+ echo "setting fluentlog_index to fluentlog bucket"
8
+ curl -X PUT http://${HOST}/buckets/fluentlog/props -H 'Content-type:application/json' -d '{"props":{"yz_index":"fluentlog_index"}}'
9
+
10
+ curl http://${HOST}/yz/index?index=true
11
+ curl http://${HOST}/buckets/fluentlog/props
metadata CHANGED
@@ -1,110 +1,97 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-riak
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
5
- prerelease:
4
+ version: 0.0.3
6
5
  platform: ruby
7
6
  authors:
8
7
  - Kota UENISHI
9
8
  autorequire:
10
9
  bindir: bin
11
10
  cert_chain: []
12
- date: 2013-02-12 00:00:00.000000000 Z
11
+ date: 2014-04-25 00:00:00.000000000 Z
13
12
  dependencies:
14
13
  - !ruby/object:Gem::Dependency
15
14
  name: fluentd
16
15
  requirement: !ruby/object:Gem::Requirement
17
- none: false
18
16
  requirements:
19
- - - ~>
17
+ - - "~>"
20
18
  - !ruby/object:Gem::Version
21
19
  version: 0.10.7
22
20
  type: :runtime
23
21
  prerelease: false
24
22
  version_requirements: !ruby/object:Gem::Requirement
25
- none: false
26
23
  requirements:
27
- - - ~>
24
+ - - "~>"
28
25
  - !ruby/object:Gem::Version
29
26
  version: 0.10.7
30
27
  - !ruby/object:Gem::Dependency
31
28
  name: riak-client
32
29
  requirement: !ruby/object:Gem::Requirement
33
- none: false
34
30
  requirements:
35
- - - ~>
31
+ - - "~>"
36
32
  - !ruby/object:Gem::Version
37
33
  version: 1.0.0
38
34
  type: :runtime
39
35
  prerelease: false
40
36
  version_requirements: !ruby/object:Gem::Requirement
41
- none: false
42
37
  requirements:
43
- - - ~>
38
+ - - "~>"
44
39
  - !ruby/object:Gem::Version
45
40
  version: 1.0.0
46
41
  - !ruby/object:Gem::Dependency
47
42
  name: uuidtools
48
43
  requirement: !ruby/object:Gem::Requirement
49
- none: false
50
44
  requirements:
51
- - - ! '>='
45
+ - - ">="
52
46
  - !ruby/object:Gem::Version
53
47
  version: 2.1.3
54
48
  type: :runtime
55
49
  prerelease: false
56
50
  version_requirements: !ruby/object:Gem::Requirement
57
- none: false
58
51
  requirements:
59
- - - ! '>='
52
+ - - ">="
60
53
  - !ruby/object:Gem::Version
61
54
  version: 2.1.3
62
55
  - !ruby/object:Gem::Dependency
63
56
  name: rake
64
57
  requirement: !ruby/object:Gem::Requirement
65
- none: false
66
58
  requirements:
67
- - - ! '>='
59
+ - - ">="
68
60
  - !ruby/object:Gem::Version
69
61
  version: 0.9.2
70
62
  type: :development
71
63
  prerelease: false
72
64
  version_requirements: !ruby/object:Gem::Requirement
73
- none: false
74
65
  requirements:
75
- - - ! '>='
66
+ - - ">="
76
67
  - !ruby/object:Gem::Version
77
68
  version: 0.9.2
78
69
  - !ruby/object:Gem::Dependency
79
70
  name: simplecov
80
71
  requirement: !ruby/object:Gem::Requirement
81
- none: false
82
72
  requirements:
83
- - - ! '>='
73
+ - - ">="
84
74
  - !ruby/object:Gem::Version
85
75
  version: 0.5.4
86
76
  type: :development
87
77
  prerelease: false
88
78
  version_requirements: !ruby/object:Gem::Requirement
89
- none: false
90
79
  requirements:
91
- - - ! '>='
80
+ - - ">="
92
81
  - !ruby/object:Gem::Version
93
82
  version: 0.5.4
94
83
  - !ruby/object:Gem::Dependency
95
84
  name: rr
96
85
  requirement: !ruby/object:Gem::Requirement
97
- none: false
98
86
  requirements:
99
- - - ! '>='
87
+ - - ">="
100
88
  - !ruby/object:Gem::Version
101
89
  version: 1.0.0
102
90
  type: :development
103
91
  prerelease: false
104
92
  version_requirements: !ruby/object:Gem::Requirement
105
- none: false
106
93
  requirements:
107
- - - ! '>='
94
+ - - ">="
108
95
  - !ruby/object:Gem::Version
109
96
  version: 1.0.0
110
97
  description: Riak plugin for Fluent event collector
@@ -113,43 +100,37 @@ executables: []
113
100
  extensions: []
114
101
  extra_rdoc_files: []
115
102
  files:
116
- - .gitignore
103
+ - ".gitignore"
117
104
  - Gemfile
118
105
  - README.md
119
106
  - Rakefile
120
107
  - VERSION
108
+ - browser.html
121
109
  - fluent-plugin-riak.gemspec
122
110
  - lib/fluent/plugin/out_riak.rb
123
111
  - listkeys.rb
124
- - loader.rb
112
+ - setup_index.sh
125
113
  homepage: https://github.com/kuenishi/fluent-plugin-riak
126
114
  licenses: []
115
+ metadata: {}
127
116
  post_install_message:
128
117
  rdoc_options: []
129
118
  require_paths:
130
119
  - lib
131
120
  required_ruby_version: !ruby/object:Gem::Requirement
132
- none: false
133
121
  requirements:
134
- - - ! '>='
122
+ - - ">="
135
123
  - !ruby/object:Gem::Version
136
124
  version: '0'
137
- segments:
138
- - 0
139
- hash: 2812051443307104688
140
125
  required_rubygems_version: !ruby/object:Gem::Requirement
141
- none: false
142
126
  requirements:
143
- - - ! '>='
127
+ - - ">="
144
128
  - !ruby/object:Gem::Version
145
129
  version: '0'
146
- segments:
147
- - 0
148
- hash: 2812051443307104688
149
130
  requirements: []
150
131
  rubyforge_project:
151
- rubygems_version: 1.8.23
132
+ rubygems_version: 2.2.0
152
133
  signing_key:
153
- specification_version: 3
134
+ specification_version: 4
154
135
  summary: Riak plugin for Fluent event collector
155
136
  test_files: []
data/loader.rb DELETED
@@ -1,21 +0,0 @@
1
- require 'net/http'
2
-
3
- # uri = URI("http://192.168.62.128:8888/riak.test")
4
-
5
- def run(n, uri)
6
- (0..n).each do |i|
7
- json = "{\"hoge\":\"valueoooooooo ???? special? \",\"id\":#{i}}"
8
- res = Net::HTTP.post_form(uri, 'json' => json)
9
- if (i % 1000) == 0 then
10
- print "#{i}: #{res.code}\n"
11
- end
12
- end
13
- end
14
-
15
- uri = URI("http://192.168.100.128:8888/riak.test")
16
- n = 1024*1024
17
- start = Time.now
18
- run(n, uri)
19
- duration = (Time.now - start).to_i
20
- mps = n * 1.0 / duration
21
- print "#{n} messages in #{duration} secs: #{mps} mps.\n"