fluent-plugin-riak 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/Gemfile +1 -1
- data/README.md +31 -7
- data/VERSION +1 -1
- data/browser.html +178 -0
- data/lib/fluent/plugin/out_riak.rb +15 -12
- data/setup_index.sh +11 -0
- metadata +22 -41
- data/loader.rb +0 -21
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 6013e6377c7ac209a511398f26574ff9044a5f6b
|
4
|
+
data.tar.gz: 79a4bd4445aba19bdcf2700ba3693b83d34be945
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 716b604d26eba39bbf19398614e823cacb9cb6e899bd08bac69da6e132639d25b381ce7fb10581bd70c8441844c654c3599c154a3a1ac2cc899df102cfe3fbb4
|
7
|
+
data.tar.gz: 5482f01385ad0bb914ae48289e025f6158e223ce21a75139097b3ec43ecdeedc0c1313d47adbff7337ebc7632a84ea6d8a9d4f1618d0e2ef3045ec10bec92643
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,10 +1,19 @@
|
|
1
|
-
fluent-plugin-riak
|
1
|
+
fluent-plugin-riak, a plugin for [Fluentd](http://fluentd.org)
|
2
2
|
==================
|
3
3
|
|
4
4
|
fluent-plugin-riak is a alternative for people who are not sufficient with mongo or webhdfs. Riak ( http://github.com/basho/riak ) is an open-source distributed KVS focused on availability. It also has a strong query system with secondary index (2i): see docs ( http://docs.basho.com/riak/latest/tutorials/querying/ ) for details.
|
5
5
|
|
6
6
|
Current status is still proof-of-concept: index setting and its configuration are to be decided. Also performance optimization is required. Another idea is in_tail_riak by using riak post-commit.
|
7
7
|
|
8
|
+
installation
|
9
|
+
------------
|
10
|
+
|
11
|
+
```bash
|
12
|
+
$ sudo gem install fluent-plugin-riak
|
13
|
+
```
|
14
|
+
|
15
|
+
Notice: you need Riak configured using eleveldb as backend.
|
16
|
+
|
8
17
|
|
9
18
|
fluent.conf example
|
10
19
|
-------------------
|
@@ -28,12 +37,21 @@ fluent.conf example
|
|
28
37
|
|
29
38
|
```
|
30
39
|
|
31
|
-
key format -> 2013-02-<uuid>
|
32
|
-
value format -> [records] in JSON
|
33
|
-
index:
|
34
|
-
|
35
|
-
|
36
|
-
|
40
|
+
- key format -> 2013-02-<uuid>
|
41
|
+
- value format -> [records] in JSON
|
42
|
+
- index:
|
43
|
+
|
44
|
+
- year_int -> year
|
45
|
+
- month_bin -> <year>-<month>
|
46
|
+
- tag_bin -> tags
|
47
|
+
|
48
|
+
easy querying log
|
49
|
+
-----------------
|
50
|
+
|
51
|
+
```bash
|
52
|
+
$ curl -X PUT http://localhost:8098/buckets/static/keys/browser.html -H 'Content-type: text/html' -d @browser.html
|
53
|
+
$ open http://localhost:8098/buckets/static/keys/browser.html
|
54
|
+
```
|
37
55
|
|
38
56
|
Pros
|
39
57
|
----
|
@@ -48,6 +66,12 @@ Cons
|
|
48
66
|
|
49
67
|
- no capped table, TTL objects
|
50
68
|
|
69
|
+
TODOs
|
70
|
+
-----
|
71
|
+
|
72
|
+
- refine browser.html query interface with cool features
|
73
|
+
- rething index structures
|
74
|
+
|
51
75
|
|
52
76
|
License
|
53
77
|
=======
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.3
|
data/browser.html
ADDED
@@ -0,0 +1,178 @@
|
|
1
|
+
<html>
|
2
|
+
<head>
|
3
|
+
<title>Riak mapreducer</title>
|
4
|
+
<style type="text/css">
|
5
|
+
body {
|
6
|
+
background: #678; text-align: center; width: 100%; font-family: 'Titillium', Monospace;
|
7
|
+
}
|
8
|
+
body div {
|
9
|
+
width: 80%; background: #ddd;
|
10
|
+
padding: 20px; text-align: left; margin: 0px auto;
|
11
|
+
}
|
12
|
+
h2 {
|
13
|
+
padding: 0px;
|
14
|
+
margin: 0px auto;
|
15
|
+
}
|
16
|
+
a {
|
17
|
+
text-shadow: 1px 1px #888888;
|
18
|
+
text-decoration:underline;
|
19
|
+
}
|
20
|
+
</style>
|
21
|
+
<script lang="JavaScript">
|
22
|
+
function get_map(){ return document.getElementById("map").value; }
|
23
|
+
function get_reduce(){ return document.getElementById("reduce").value; }
|
24
|
+
function get_bucketname(){ return document.getElementById("bucket").value; }
|
25
|
+
function exec_query(m, r){
|
26
|
+
var q = { inputs : get_bucketname(),
|
27
|
+
query : [
|
28
|
+
{ map: { language: "javascript", source: m}},
|
29
|
+
{ reduce: { language: "javascript", source: r}}
|
30
|
+
]};
|
31
|
+
var xmlhttp = new XMLHttpRequest();
|
32
|
+
xmlhttp.open("POST", "/mapred", false);
|
33
|
+
xmlhttp.setRequestHeader("Content-Type", "application/json");
|
34
|
+
xmlhttp.send(JSON.stringify(q, null, ""));
|
35
|
+
var result = xmlhttp.responseText;
|
36
|
+
|
37
|
+
var resultBox = document.getElementById("result");
|
38
|
+
resultBox.innerText = result;
|
39
|
+
}
|
40
|
+
</script>
|
41
|
+
</head>
|
42
|
+
|
43
|
+
<body>
|
44
|
+
<div id="content">
|
45
|
+
<div style="text-align:right">
|
46
|
+
<script lang="JavaScript">
|
47
|
+
</script>
|
48
|
+
bucket name:<input type="text" id="bucket" value="fluentlog"/>
|
49
|
+
</div>
|
50
|
+
|
51
|
+
<h2>Run mapreduce on fluentd log</h2>
|
52
|
+
<table style="border: 1px solid black;">
|
53
|
+
<tr>
|
54
|
+
<th>map</th> <th>reduce</th>
|
55
|
+
</tr>
|
56
|
+
<tr>
|
57
|
+
<td>
|
58
|
+
<textarea name="map" id="map" rows="16" cols="50">
|
59
|
+
function(v){
|
60
|
+
var data = v.values[0].data;
|
61
|
+
var c = JSON.parse(data);
|
62
|
+
return [c.count];
|
63
|
+
}
|
64
|
+
</textarea>
|
65
|
+
</td><td>
|
66
|
+
<textarea name="reduce" id="reduce" rows="16" cols="50" >
|
67
|
+
function(v){
|
68
|
+
var sum = 0;
|
69
|
+
for( var i=0; i < v.length; ++i){
|
70
|
+
sum += v[i];
|
71
|
+
}
|
72
|
+
return [sum];
|
73
|
+
}
|
74
|
+
</textarea>
|
75
|
+
</td>
|
76
|
+
</tr>
|
77
|
+
<tr>
|
78
|
+
<th colspan="2"> <a onClick="exec_query(get_map(), get_reduce());">query</a> </th>
|
79
|
+
</tr>
|
80
|
+
<tr>
|
81
|
+
<td width="200px" colspan="2">
|
82
|
+
<div id="result" style="font: bold;"></div>
|
83
|
+
</td>
|
84
|
+
</tr>
|
85
|
+
<tr><td colspan="3">
|
86
|
+
<script lang="JavaScript">
|
87
|
+
function grep(){
|
88
|
+
|
89
|
+
}
|
90
|
+
</script>
|
91
|
+
<script lang="JavaScript" type="text/javascript">
|
92
|
+
function count_rank(){
|
93
|
+
var m = "function(v){
|
94
|
+
var data = v.values[0].data;
|
95
|
+
var c = JSON.parse(data);
|
96
|
+
var tags = {};
|
97
|
+
for(var i=0; i<c.length; ++i){
|
98
|
+
count = tags[c[i].tag];
|
99
|
+
if(count == null){ tags[c[i].tag] = 1; }
|
100
|
+
else{ tags[c[i].tag] = count+1; }
|
101
|
+
}
|
102
|
+
return [tags];
|
103
|
+
}";
|
104
|
+
var r = "function(v){
|
105
|
+
var tags = {};
|
106
|
+
for(var i=0; i<v.length; ++i){
|
107
|
+
for (var prop in v[i]) {
|
108
|
+
count = tags[prop];
|
109
|
+
if(count == null){ tags[prop] = v[i][prop]; }
|
110
|
+
else{ tags[prop] = count+v[i][prop]; }
|
111
|
+
}
|
112
|
+
}
|
113
|
+
return [tags];
|
114
|
+
}";
|
115
|
+
exec_query(m, r);
|
116
|
+
}
|
117
|
+
</script>
|
118
|
+
|
119
|
+
<!-- <a onClick="grep();">grep</a> <input type="text" id="grep" value="debug"/> -->
|
120
|
+
<a onClick="count_rank();">count&rank</a> <input type="text" id="count_rank" value="tag"/>
|
121
|
+
</td></tr>
|
122
|
+
</table>
|
123
|
+
|
124
|
+
<hr/>
|
125
|
+
<h2>Riak object viewer</h2>
|
126
|
+
<script lang="JavaScript">
|
127
|
+
function get(){
|
128
|
+
var key = document.getElementById("key").value;
|
129
|
+
var xmlhttp = new XMLHttpRequest();
|
130
|
+
xmlhttp.open("GET", "/buckets/"+ get_bucketname() +"/keys/"+key, false);
|
131
|
+
xmlhttp.send();
|
132
|
+
var result = xmlhttp.responseText;
|
133
|
+
console.log(result);
|
134
|
+
document.getElementById("object").innerText = result;
|
135
|
+
}
|
136
|
+
</script>
|
137
|
+
<a onClick="get();" >get</a> <input type="text" id="key" />
|
138
|
+
<div id="object"></div>
|
139
|
+
|
140
|
+
|
141
|
+
<h2>Riak keylister</h2>
|
142
|
+
<script lang="JavaScript">
|
143
|
+
function listkeys(){
|
144
|
+
var xmlhttp = new XMLHttpRequest();
|
145
|
+
xmlhttp.open("GET", "/buckets/"+ get_bucketname() +"/keys?keys=true", false);
|
146
|
+
xmlhttp.send();
|
147
|
+
var result = JSON.parse(xmlhttp.responseText);
|
148
|
+
console.log(result);
|
149
|
+
|
150
|
+
document.getElementById("keylist").innerText = result.keys.sort().join("\n");
|
151
|
+
}
|
152
|
+
</script>
|
153
|
+
<a onClick="listkeys();" >listkeys</a>
|
154
|
+
<div id="keylist"></div>
|
155
|
+
<hr/>
|
156
|
+
(C)@kuenishi distributed under Apache 2 license
|
157
|
+
</div>
|
158
|
+
<!--
|
159
|
+
<h2>Riak object putter</h2>
|
160
|
+
<script lang="JavaScript">
|
161
|
+
function put(){
|
162
|
+
var key = document.getElementById("key2").value;
|
163
|
+
var value = document.getElementById("value").value;
|
164
|
+
var xmlhttp = new XMLHttpRequest();
|
165
|
+
xmlhttp.open("PUT", "/buckets/fluentlog/keys/"+key, false);
|
166
|
+
xmlhttp.setRequestHeader("Content-Type", "application/json");
|
167
|
+
xmlhttp.send(value);
|
168
|
+
var result = xmlhttp.responseText;
|
169
|
+
console.log(result);
|
170
|
+
document.getElementById("object2").innerText = result;
|
171
|
+
}
|
172
|
+
</script>
|
173
|
+
<a onClick="put();" >put</a> <input type="text" id="key2" /> <input type="text" id="value" />
|
174
|
+
<div id="object2"></div>
|
175
|
+
-->
|
176
|
+
|
177
|
+
</body>
|
178
|
+
</html>
|
@@ -8,6 +8,7 @@ class RiakOutput < BufferedOutput
|
|
8
8
|
include SetTagKeyMixin
|
9
9
|
config_set_default :include_time_key, true
|
10
10
|
|
11
|
+
config_param :bucket_name, :string, :default => "fluentlog"
|
11
12
|
config_param :nodes, :string, :default => "localhost:8087"
|
12
13
|
|
13
14
|
def initialize
|
@@ -30,7 +31,7 @@ class RiakOutput < BufferedOutput
|
|
30
31
|
def start
|
31
32
|
$log.debug " => #{@buffer.chunk_limit} #{@buffer.queue_limit} "
|
32
33
|
@conn = Riak::Client.new(:nodes => @nodes, :protocol => "pbc")
|
33
|
-
@bucket = @conn.bucket(
|
34
|
+
@bucket = @conn.bucket(@bucket_name)
|
34
35
|
@buf = {}
|
35
36
|
|
36
37
|
super
|
@@ -58,18 +59,20 @@ class RiakOutput < BufferedOutput
|
|
58
59
|
|
59
60
|
# TODO: add index for some analysis
|
60
61
|
def put_now(records, tags)
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
62
|
+
if not records.empty? then
|
63
|
+
today = Date.today
|
64
|
+
key = "#{today.to_s}-#{UUIDTools::UUID.random_create.to_s}"
|
65
|
+
robj = Riak::RObject.new(@bucket, key)
|
66
|
+
robj.raw_data = records.to_json
|
67
|
+
robj.indexes['year_int'] << today.year
|
68
|
+
robj.indexes['month_bin'] << "#{today.year}-#{"%02d" % today.month}"
|
69
|
+
tags.each do |tag|
|
70
|
+
robj.indexes['tag_bin'] << tag
|
71
|
+
end
|
72
|
+
robj.content_type = 'application/json'
|
73
|
+
robj.store
|
74
|
+
robj
|
69
75
|
end
|
70
|
-
robj.content_type = 'application/json'
|
71
|
-
robj.store
|
72
|
-
robj
|
73
76
|
end
|
74
77
|
|
75
78
|
end
|
data/setup_index.sh
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
#!/bin/sh
|
2
|
+
|
3
|
+
HOST=$1
|
4
|
+
|
5
|
+
echo setting up fluentlog_index at $HOST
|
6
|
+
curl -X PUT http://${HOST}/yz/index/fluentlog_index
|
7
|
+
echo "setting fluentlog_index to fluentlog bucket"
|
8
|
+
curl -X PUT http://${HOST}/buckets/fluentlog/props -H 'Content-type:application/json' -d '{"props":{"yz_index":"fluentlog_index"}}'
|
9
|
+
|
10
|
+
curl http://${HOST}/yz/index?index=true
|
11
|
+
curl http://${HOST}/buckets/fluentlog/props
|
metadata
CHANGED
@@ -1,110 +1,97 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-riak
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.0.3
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Kota UENISHI
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date:
|
11
|
+
date: 2014-04-25 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: fluentd
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
|
-
- - ~>
|
17
|
+
- - "~>"
|
20
18
|
- !ruby/object:Gem::Version
|
21
19
|
version: 0.10.7
|
22
20
|
type: :runtime
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
|
-
- - ~>
|
24
|
+
- - "~>"
|
28
25
|
- !ruby/object:Gem::Version
|
29
26
|
version: 0.10.7
|
30
27
|
- !ruby/object:Gem::Dependency
|
31
28
|
name: riak-client
|
32
29
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
30
|
requirements:
|
35
|
-
- - ~>
|
31
|
+
- - "~>"
|
36
32
|
- !ruby/object:Gem::Version
|
37
33
|
version: 1.0.0
|
38
34
|
type: :runtime
|
39
35
|
prerelease: false
|
40
36
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
37
|
requirements:
|
43
|
-
- - ~>
|
38
|
+
- - "~>"
|
44
39
|
- !ruby/object:Gem::Version
|
45
40
|
version: 1.0.0
|
46
41
|
- !ruby/object:Gem::Dependency
|
47
42
|
name: uuidtools
|
48
43
|
requirement: !ruby/object:Gem::Requirement
|
49
|
-
none: false
|
50
44
|
requirements:
|
51
|
-
- -
|
45
|
+
- - ">="
|
52
46
|
- !ruby/object:Gem::Version
|
53
47
|
version: 2.1.3
|
54
48
|
type: :runtime
|
55
49
|
prerelease: false
|
56
50
|
version_requirements: !ruby/object:Gem::Requirement
|
57
|
-
none: false
|
58
51
|
requirements:
|
59
|
-
- -
|
52
|
+
- - ">="
|
60
53
|
- !ruby/object:Gem::Version
|
61
54
|
version: 2.1.3
|
62
55
|
- !ruby/object:Gem::Dependency
|
63
56
|
name: rake
|
64
57
|
requirement: !ruby/object:Gem::Requirement
|
65
|
-
none: false
|
66
58
|
requirements:
|
67
|
-
- -
|
59
|
+
- - ">="
|
68
60
|
- !ruby/object:Gem::Version
|
69
61
|
version: 0.9.2
|
70
62
|
type: :development
|
71
63
|
prerelease: false
|
72
64
|
version_requirements: !ruby/object:Gem::Requirement
|
73
|
-
none: false
|
74
65
|
requirements:
|
75
|
-
- -
|
66
|
+
- - ">="
|
76
67
|
- !ruby/object:Gem::Version
|
77
68
|
version: 0.9.2
|
78
69
|
- !ruby/object:Gem::Dependency
|
79
70
|
name: simplecov
|
80
71
|
requirement: !ruby/object:Gem::Requirement
|
81
|
-
none: false
|
82
72
|
requirements:
|
83
|
-
- -
|
73
|
+
- - ">="
|
84
74
|
- !ruby/object:Gem::Version
|
85
75
|
version: 0.5.4
|
86
76
|
type: :development
|
87
77
|
prerelease: false
|
88
78
|
version_requirements: !ruby/object:Gem::Requirement
|
89
|
-
none: false
|
90
79
|
requirements:
|
91
|
-
- -
|
80
|
+
- - ">="
|
92
81
|
- !ruby/object:Gem::Version
|
93
82
|
version: 0.5.4
|
94
83
|
- !ruby/object:Gem::Dependency
|
95
84
|
name: rr
|
96
85
|
requirement: !ruby/object:Gem::Requirement
|
97
|
-
none: false
|
98
86
|
requirements:
|
99
|
-
- -
|
87
|
+
- - ">="
|
100
88
|
- !ruby/object:Gem::Version
|
101
89
|
version: 1.0.0
|
102
90
|
type: :development
|
103
91
|
prerelease: false
|
104
92
|
version_requirements: !ruby/object:Gem::Requirement
|
105
|
-
none: false
|
106
93
|
requirements:
|
107
|
-
- -
|
94
|
+
- - ">="
|
108
95
|
- !ruby/object:Gem::Version
|
109
96
|
version: 1.0.0
|
110
97
|
description: Riak plugin for Fluent event collector
|
@@ -113,43 +100,37 @@ executables: []
|
|
113
100
|
extensions: []
|
114
101
|
extra_rdoc_files: []
|
115
102
|
files:
|
116
|
-
- .gitignore
|
103
|
+
- ".gitignore"
|
117
104
|
- Gemfile
|
118
105
|
- README.md
|
119
106
|
- Rakefile
|
120
107
|
- VERSION
|
108
|
+
- browser.html
|
121
109
|
- fluent-plugin-riak.gemspec
|
122
110
|
- lib/fluent/plugin/out_riak.rb
|
123
111
|
- listkeys.rb
|
124
|
-
-
|
112
|
+
- setup_index.sh
|
125
113
|
homepage: https://github.com/kuenishi/fluent-plugin-riak
|
126
114
|
licenses: []
|
115
|
+
metadata: {}
|
127
116
|
post_install_message:
|
128
117
|
rdoc_options: []
|
129
118
|
require_paths:
|
130
119
|
- lib
|
131
120
|
required_ruby_version: !ruby/object:Gem::Requirement
|
132
|
-
none: false
|
133
121
|
requirements:
|
134
|
-
- -
|
122
|
+
- - ">="
|
135
123
|
- !ruby/object:Gem::Version
|
136
124
|
version: '0'
|
137
|
-
segments:
|
138
|
-
- 0
|
139
|
-
hash: 2812051443307104688
|
140
125
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
141
|
-
none: false
|
142
126
|
requirements:
|
143
|
-
- -
|
127
|
+
- - ">="
|
144
128
|
- !ruby/object:Gem::Version
|
145
129
|
version: '0'
|
146
|
-
segments:
|
147
|
-
- 0
|
148
|
-
hash: 2812051443307104688
|
149
130
|
requirements: []
|
150
131
|
rubyforge_project:
|
151
|
-
rubygems_version:
|
132
|
+
rubygems_version: 2.2.0
|
152
133
|
signing_key:
|
153
|
-
specification_version:
|
134
|
+
specification_version: 4
|
154
135
|
summary: Riak plugin for Fluent event collector
|
155
136
|
test_files: []
|
data/loader.rb
DELETED
@@ -1,21 +0,0 @@
|
|
1
|
-
require 'net/http'
|
2
|
-
|
3
|
-
# uri = URI("http://192.168.62.128:8888/riak.test")
|
4
|
-
|
5
|
-
def run(n, uri)
|
6
|
-
(0..n).each do |i|
|
7
|
-
json = "{\"hoge\":\"valueoooooooo ???? special? \",\"id\":#{i}}"
|
8
|
-
res = Net::HTTP.post_form(uri, 'json' => json)
|
9
|
-
if (i % 1000) == 0 then
|
10
|
-
print "#{i}: #{res.code}\n"
|
11
|
-
end
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
uri = URI("http://192.168.100.128:8888/riak.test")
|
16
|
-
n = 1024*1024
|
17
|
-
start = Time.now
|
18
|
-
run(n, uri)
|
19
|
-
duration = (Time.now - start).to_i
|
20
|
-
mps = n * 1.0 / duration
|
21
|
-
print "#{n} messages in #{duration} secs: #{mps} mps.\n"
|