fluent-plugin-riak 0.0.2 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. checksums.yaml +7 -0
  2. data/Gemfile +1 -1
  3. data/README.md +31 -7
  4. data/VERSION +1 -1
  5. data/browser.html +178 -0
  6. data/lib/fluent/plugin/out_riak.rb +15 -12
  7. data/setup_index.sh +11 -0
  8. metadata +22 -41
  9. data/loader.rb +0 -21
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 6013e6377c7ac209a511398f26574ff9044a5f6b
4
+ data.tar.gz: 79a4bd4445aba19bdcf2700ba3693b83d34be945
5
+ SHA512:
6
+ metadata.gz: 716b604d26eba39bbf19398614e823cacb9cb6e899bd08bac69da6e132639d25b381ce7fb10581bd70c8441844c654c3599c154a3a1ac2cc899df102cfe3fbb4
7
+ data.tar.gz: 5482f01385ad0bb914ae48289e025f6158e223ce21a75139097b3ec43ecdeedc0c1313d47adbff7337ebc7632a84ea6d8a9d4f1618d0e2ef3045ec10bec92643
data/Gemfile CHANGED
@@ -1,4 +1,4 @@
1
- source :rubygems
1
+ source 'https://rubygems.org'
2
2
 
3
3
  gemspec
4
4
 
data/README.md CHANGED
@@ -1,10 +1,19 @@
1
- fluent-plugin-riak
1
+ fluent-plugin-riak, a plugin for [Fluentd](http://fluentd.org)
2
2
  ==================
3
3
 
4
4
  fluent-plugin-riak is a alternative for people who are not sufficient with mongo or webhdfs. Riak ( http://github.com/basho/riak ) is an open-source distributed KVS focused on availability. It also has a strong query system with secondary index (2i): see docs ( http://docs.basho.com/riak/latest/tutorials/querying/ ) for details.
5
5
 
6
6
  Current status is still proof-of-concept: index setting and its configuration are to be decided. Also performance optimization is required. Another idea is in_tail_riak by using riak post-commit.
7
7
 
8
+ installation
9
+ ------------
10
+
11
+ ```bash
12
+ $ sudo gem install fluent-plugin-riak
13
+ ```
14
+
15
+ Notice: you need Riak configured using eleveldb as backend.
16
+
8
17
 
9
18
  fluent.conf example
10
19
  -------------------
@@ -28,12 +37,21 @@ fluent.conf example
28
37
 
29
38
  ```
30
39
 
31
- key format -> 2013-02-<uuid>
32
- value format -> [records] in JSON
33
- index:
34
- year_int -> year
35
- month_bin -> <year>-<month>
36
- tag_bin -> tags
40
+ - key format -> 2013-02-<uuid>
41
+ - value format -> [records] in JSON
42
+ - index:
43
+
44
+ - year_int -> year
45
+ - month_bin -> <year>-<month>
46
+ - tag_bin -> tags
47
+
48
+ easy querying log
49
+ -----------------
50
+
51
+ ```bash
52
+ $ curl -X PUT http://localhost:8098/buckets/static/keys/browser.html -H 'Content-type: text/html' -d @browser.html
53
+ $ open http://localhost:8098/buckets/static/keys/browser.html
54
+ ```
37
55
 
38
56
  Pros
39
57
  ----
@@ -48,6 +66,12 @@ Cons
48
66
 
49
67
  - no capped table, TTL objects
50
68
 
69
+ TODOs
70
+ -----
71
+
72
+ - refine browser.html query interface with cool features
73
+ - rething index structures
74
+
51
75
 
52
76
  License
53
77
  =======
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.0.2
1
+ 0.0.3
data/browser.html ADDED
@@ -0,0 +1,178 @@
1
+ <html>
2
+ <head>
3
+ <title>Riak mapreducer</title>
4
+ <style type="text/css">
5
+ body {
6
+ background: #678; text-align: center; width: 100%; font-family: 'Titillium', Monospace;
7
+ }
8
+ body div {
9
+ width: 80%; background: #ddd;
10
+ padding: 20px; text-align: left; margin: 0px auto;
11
+ }
12
+ h2 {
13
+ padding: 0px;
14
+ margin: 0px auto;
15
+ }
16
+ a {
17
+ text-shadow: 1px 1px #888888;
18
+ text-decoration:underline;
19
+ }
20
+ </style>
21
+ <script lang="JavaScript">
22
+ function get_map(){ return document.getElementById("map").value; }
23
+ function get_reduce(){ return document.getElementById("reduce").value; }
24
+ function get_bucketname(){ return document.getElementById("bucket").value; }
25
+ function exec_query(m, r){
26
+ var q = { inputs : get_bucketname(),
27
+ query : [
28
+ { map: { language: "javascript", source: m}},
29
+ { reduce: { language: "javascript", source: r}}
30
+ ]};
31
+ var xmlhttp = new XMLHttpRequest();
32
+ xmlhttp.open("POST", "/mapred", false);
33
+ xmlhttp.setRequestHeader("Content-Type", "application/json");
34
+ xmlhttp.send(JSON.stringify(q, null, ""));
35
+ var result = xmlhttp.responseText;
36
+
37
+ var resultBox = document.getElementById("result");
38
+ resultBox.innerText = result;
39
+ }
40
+ </script>
41
+ </head>
42
+
43
+ <body>
44
+ <div id="content">
45
+ <div style="text-align:right">
46
+ <script lang="JavaScript">
47
+ </script>
48
+ bucket name:<input type="text" id="bucket" value="fluentlog"/>
49
+ </div>
50
+
51
+ <h2>Run mapreduce on fluentd log</h2>
52
+ <table style="border: 1px solid black;">
53
+ <tr>
54
+ <th>map</th> <th>reduce</th>
55
+ </tr>
56
+ <tr>
57
+ <td>
58
+ <textarea name="map" id="map" rows="16" cols="50">
59
+ function(v){
60
+ var data = v.values[0].data;
61
+ var c = JSON.parse(data);
62
+ return [c.count];
63
+ }
64
+ </textarea>
65
+ </td><td>
66
+ <textarea name="reduce" id="reduce" rows="16" cols="50" >
67
+ function(v){
68
+ var sum = 0;
69
+ for( var i=0; i < v.length; ++i){
70
+ sum += v[i];
71
+ }
72
+ return [sum];
73
+ }
74
+ </textarea>
75
+ </td>
76
+ </tr>
77
+ <tr>
78
+ <th colspan="2"> <a onClick="exec_query(get_map(), get_reduce());">query</a> </th>
79
+ </tr>
80
+ <tr>
81
+ <td width="200px" colspan="2">
82
+ <div id="result" style="font: bold;"></div>
83
+ </td>
84
+ </tr>
85
+ <tr><td colspan="3">
86
+ <script lang="JavaScript">
87
+ function grep(){
88
+
89
+ }
90
+ </script>
91
+ <script lang="JavaScript" type="text/javascript">
92
+ function count_rank(){
93
+ var m = "function(v){
94
+ var data = v.values[0].data;
95
+ var c = JSON.parse(data);
96
+ var tags = {};
97
+ for(var i=0; i<c.length; ++i){
98
+ count = tags[c[i].tag];
99
+ if(count == null){ tags[c[i].tag] = 1; }
100
+ else{ tags[c[i].tag] = count+1; }
101
+ }
102
+ return [tags];
103
+ }";
104
+ var r = "function(v){
105
+ var tags = {};
106
+ for(var i=0; i<v.length; ++i){
107
+ for (var prop in v[i]) {
108
+ count = tags[prop];
109
+ if(count == null){ tags[prop] = v[i][prop]; }
110
+ else{ tags[prop] = count+v[i][prop]; }
111
+ }
112
+ }
113
+ return [tags];
114
+ }";
115
+ exec_query(m, r);
116
+ }
117
+ </script>
118
+
119
+ <!-- <a onClick="grep();">grep</a> <input type="text" id="grep" value="debug"/> -->
120
+ <a onClick="count_rank();">count&amp;rank</a> <input type="text" id="count_rank" value="tag"/>
121
+ </td></tr>
122
+ </table>
123
+
124
+ <hr/>
125
+ <h2>Riak object viewer</h2>
126
+ <script lang="JavaScript">
127
+ function get(){
128
+ var key = document.getElementById("key").value;
129
+ var xmlhttp = new XMLHttpRequest();
130
+ xmlhttp.open("GET", "/buckets/"+ get_bucketname() +"/keys/"+key, false);
131
+ xmlhttp.send();
132
+ var result = xmlhttp.responseText;
133
+ console.log(result);
134
+ document.getElementById("object").innerText = result;
135
+ }
136
+ </script>
137
+ <a onClick="get();" >get</a> <input type="text" id="key" />
138
+ <div id="object"></div>
139
+
140
+
141
+ <h2>Riak keylister</h2>
142
+ <script lang="JavaScript">
143
+ function listkeys(){
144
+ var xmlhttp = new XMLHttpRequest();
145
+ xmlhttp.open("GET", "/buckets/"+ get_bucketname() +"/keys?keys=true", false);
146
+ xmlhttp.send();
147
+ var result = JSON.parse(xmlhttp.responseText);
148
+ console.log(result);
149
+
150
+ document.getElementById("keylist").innerText = result.keys.sort().join("\n");
151
+ }
152
+ </script>
153
+ <a onClick="listkeys();" >listkeys</a>
154
+ <div id="keylist"></div>
155
+ <hr/>
156
+ (C)@kuenishi distributed under Apache 2 license
157
+ </div>
158
+ <!--
159
+ <h2>Riak object putter</h2>
160
+ <script lang="JavaScript">
161
+ function put(){
162
+ var key = document.getElementById("key2").value;
163
+ var value = document.getElementById("value").value;
164
+ var xmlhttp = new XMLHttpRequest();
165
+ xmlhttp.open("PUT", "/buckets/fluentlog/keys/"+key, false);
166
+ xmlhttp.setRequestHeader("Content-Type", "application/json");
167
+ xmlhttp.send(value);
168
+ var result = xmlhttp.responseText;
169
+ console.log(result);
170
+ document.getElementById("object2").innerText = result;
171
+ }
172
+ </script>
173
+ <a onClick="put();" >put</a> <input type="text" id="key2" /> <input type="text" id="value" />
174
+ <div id="object2"></div>
175
+ -->
176
+
177
+ </body>
178
+ </html>
data/lib/fluent/plugin/out_riak.rb CHANGED
@@ -8,6 +8,7 @@ class RiakOutput < BufferedOutput
8
8
  include SetTagKeyMixin
9
9
  config_set_default :include_time_key, true
10
10
 
11
+ config_param :bucket_name, :string, :default => "fluentlog"
11
12
  config_param :nodes, :string, :default => "localhost:8087"
12
13
 
13
14
  def initialize
@@ -30,7 +31,7 @@ class RiakOutput < BufferedOutput
30
31
  def start
31
32
  $log.debug " => #{@buffer.chunk_limit} #{@buffer.queue_limit} "
32
33
  @conn = Riak::Client.new(:nodes => @nodes, :protocol => "pbc")
33
- @bucket = @conn.bucket("fluentlog")
34
+ @bucket = @conn.bucket(@bucket_name)
34
35
  @buf = {}
35
36
 
36
37
  super
@@ -58,18 +59,20 @@ class RiakOutput < BufferedOutput
58
59
 
59
60
  # TODO: add index for some analysis
60
61
  def put_now(records, tags)
61
- today = Date.today
62
- key = "#{today.to_s}-#{UUIDTools::UUID.random_create.to_s}"
63
- robj = Riak::RObject.new(@bucket, key)
64
- robj.raw_data = records.to_json
65
- robj.indexes['year_int'] << today.year
66
- robj.indexes['month_bin'] << "#{today.year}-#{today.month}"
67
- tags.each do |tag|
68
- robj.indexes['tag_bin'] << tag
62
+ if not records.empty? then
63
+ today = Date.today
64
+ key = "#{today.to_s}-#{UUIDTools::UUID.random_create.to_s}"
65
+ robj = Riak::RObject.new(@bucket, key)
66
+ robj.raw_data = records.to_json
67
+ robj.indexes['year_int'] << today.year
68
+ robj.indexes['month_bin'] << "#{today.year}-#{"%02d" % today.month}"
69
+ tags.each do |tag|
70
+ robj.indexes['tag_bin'] << tag
71
+ end
72
+ robj.content_type = 'application/json'
73
+ robj.store
74
+ robj
69
75
  end
70
- robj.content_type = 'application/json'
71
- robj.store
72
- robj
73
76
  end
74
77
 
75
78
  end
data/setup_index.sh ADDED
@@ -0,0 +1,11 @@
1
+ #!/bin/sh
2
+
3
+ HOST=$1
4
+
5
+ echo setting up fluentlog_index at $HOST
6
+ curl -X PUT http://${HOST}/yz/index/fluentlog_index
7
+ echo "setting fluentlog_index to fluentlog bucket"
8
+ curl -X PUT http://${HOST}/buckets/fluentlog/props -H 'Content-type:application/json' -d '{"props":{"yz_index":"fluentlog_index"}}'
9
+
10
+ curl http://${HOST}/yz/index?index=true
11
+ curl http://${HOST}/buckets/fluentlog/props
metadata CHANGED
@@ -1,110 +1,97 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-riak
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
5
- prerelease:
4
+ version: 0.0.3
6
5
  platform: ruby
7
6
  authors:
8
7
  - Kota UENISHI
9
8
  autorequire:
10
9
  bindir: bin
11
10
  cert_chain: []
12
- date: 2013-02-12 00:00:00.000000000 Z
11
+ date: 2014-04-25 00:00:00.000000000 Z
13
12
  dependencies:
14
13
  - !ruby/object:Gem::Dependency
15
14
  name: fluentd
16
15
  requirement: !ruby/object:Gem::Requirement
17
- none: false
18
16
  requirements:
19
- - - ~>
17
+ - - "~>"
20
18
  - !ruby/object:Gem::Version
21
19
  version: 0.10.7
22
20
  type: :runtime
23
21
  prerelease: false
24
22
  version_requirements: !ruby/object:Gem::Requirement
25
- none: false
26
23
  requirements:
27
- - - ~>
24
+ - - "~>"
28
25
  - !ruby/object:Gem::Version
29
26
  version: 0.10.7
30
27
  - !ruby/object:Gem::Dependency
31
28
  name: riak-client
32
29
  requirement: !ruby/object:Gem::Requirement
33
- none: false
34
30
  requirements:
35
- - - ~>
31
+ - - "~>"
36
32
  - !ruby/object:Gem::Version
37
33
  version: 1.0.0
38
34
  type: :runtime
39
35
  prerelease: false
40
36
  version_requirements: !ruby/object:Gem::Requirement
41
- none: false
42
37
  requirements:
43
- - - ~>
38
+ - - "~>"
44
39
  - !ruby/object:Gem::Version
45
40
  version: 1.0.0
46
41
  - !ruby/object:Gem::Dependency
47
42
  name: uuidtools
48
43
  requirement: !ruby/object:Gem::Requirement
49
- none: false
50
44
  requirements:
51
- - - ! '>='
45
+ - - ">="
52
46
  - !ruby/object:Gem::Version
53
47
  version: 2.1.3
54
48
  type: :runtime
55
49
  prerelease: false
56
50
  version_requirements: !ruby/object:Gem::Requirement
57
- none: false
58
51
  requirements:
59
- - - ! '>='
52
+ - - ">="
60
53
  - !ruby/object:Gem::Version
61
54
  version: 2.1.3
62
55
  - !ruby/object:Gem::Dependency
63
56
  name: rake
64
57
  requirement: !ruby/object:Gem::Requirement
65
- none: false
66
58
  requirements:
67
- - - ! '>='
59
+ - - ">="
68
60
  - !ruby/object:Gem::Version
69
61
  version: 0.9.2
70
62
  type: :development
71
63
  prerelease: false
72
64
  version_requirements: !ruby/object:Gem::Requirement
73
- none: false
74
65
  requirements:
75
- - - ! '>='
66
+ - - ">="
76
67
  - !ruby/object:Gem::Version
77
68
  version: 0.9.2
78
69
  - !ruby/object:Gem::Dependency
79
70
  name: simplecov
80
71
  requirement: !ruby/object:Gem::Requirement
81
- none: false
82
72
  requirements:
83
- - - ! '>='
73
+ - - ">="
84
74
  - !ruby/object:Gem::Version
85
75
  version: 0.5.4
86
76
  type: :development
87
77
  prerelease: false
88
78
  version_requirements: !ruby/object:Gem::Requirement
89
- none: false
90
79
  requirements:
91
- - - ! '>='
80
+ - - ">="
92
81
  - !ruby/object:Gem::Version
93
82
  version: 0.5.4
94
83
  - !ruby/object:Gem::Dependency
95
84
  name: rr
96
85
  requirement: !ruby/object:Gem::Requirement
97
- none: false
98
86
  requirements:
99
- - - ! '>='
87
+ - - ">="
100
88
  - !ruby/object:Gem::Version
101
89
  version: 1.0.0
102
90
  type: :development
103
91
  prerelease: false
104
92
  version_requirements: !ruby/object:Gem::Requirement
105
- none: false
106
93
  requirements:
107
- - - ! '>='
94
+ - - ">="
108
95
  - !ruby/object:Gem::Version
109
96
  version: 1.0.0
110
97
  description: Riak plugin for Fluent event collector
@@ -113,43 +100,37 @@ executables: []
113
100
  extensions: []
114
101
  extra_rdoc_files: []
115
102
  files:
116
- - .gitignore
103
+ - ".gitignore"
117
104
  - Gemfile
118
105
  - README.md
119
106
  - Rakefile
120
107
  - VERSION
108
+ - browser.html
121
109
  - fluent-plugin-riak.gemspec
122
110
  - lib/fluent/plugin/out_riak.rb
123
111
  - listkeys.rb
124
- - loader.rb
112
+ - setup_index.sh
125
113
  homepage: https://github.com/kuenishi/fluent-plugin-riak
126
114
  licenses: []
115
+ metadata: {}
127
116
  post_install_message:
128
117
  rdoc_options: []
129
118
  require_paths:
130
119
  - lib
131
120
  required_ruby_version: !ruby/object:Gem::Requirement
132
- none: false
133
121
  requirements:
134
- - - ! '>='
122
+ - - ">="
135
123
  - !ruby/object:Gem::Version
136
124
  version: '0'
137
- segments:
138
- - 0
139
- hash: 2812051443307104688
140
125
  required_rubygems_version: !ruby/object:Gem::Requirement
141
- none: false
142
126
  requirements:
143
- - - ! '>='
127
+ - - ">="
144
128
  - !ruby/object:Gem::Version
145
129
  version: '0'
146
- segments:
147
- - 0
148
- hash: 2812051443307104688
149
130
  requirements: []
150
131
  rubyforge_project:
151
- rubygems_version: 1.8.23
132
+ rubygems_version: 2.2.0
152
133
  signing_key:
153
- specification_version: 3
134
+ specification_version: 4
154
135
  summary: Riak plugin for Fluent event collector
155
136
  test_files: []
data/loader.rb DELETED
@@ -1,21 +0,0 @@
1
- require 'net/http'
2
-
3
- # uri = URI("http://192.168.62.128:8888/riak.test")
4
-
5
- def run(n, uri)
6
- (0..n).each do |i|
7
- json = "{\"hoge\":\"valueoooooooo ???? special? \",\"id\":#{i}}"
8
- res = Net::HTTP.post_form(uri, 'json' => json)
9
- if (i % 1000) == 0 then
10
- print "#{i}: #{res.code}\n"
11
- end
12
- end
13
- end
14
-
15
- uri = URI("http://192.168.100.128:8888/riak.test")
16
- n = 1024*1024
17
- start = Time.now
18
- run(n, uri)
19
- duration = (Time.now - start).to_i
20
- mps = n * 1.0 / duration
21
- print "#{n} messages in #{duration} secs: #{mps} mps.\n"