fluent-plugin-riak 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
data/.gitignore ADDED
@@ -0,0 +1,19 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ coverage
6
+ InstalledFiles
7
+ lib/bundler/man
8
+ pkg
9
+ rdoc
10
+ spec/reports
11
+ test/tmp
12
+ test/version_tmp
13
+ tmp
14
+ *~
15
+ Gemfile.lock
16
+ # YARD artifacts
17
+ .yardoc
18
+ _yardoc
19
+ doc/
data/Gemfile ADDED
@@ -0,0 +1,5 @@
1
+ source :rubygems
2
+
3
+ gemspec
4
+
5
+ gem "simplecov", :require => false
data/README.md ADDED
@@ -0,0 +1,29 @@
1
+ fluent-plugin-riak
2
+ ==================
3
+
4
+ fluent-plugin-riak is a alternative for people who are not sufficient with mongo or webhdfs. Riak ( http://github.com/basho/riak ) is an open-source distributed KVS focused on availability. It also has a strong query system with secondary index (2i): see docs ( http://docs.basho.com/riak/latest/tutorials/querying/ ) for details.
5
+
6
+ Current status is still proof-of-concept: index setting and its configuration are to be decided. Also performance optimization is required. Another idea is in_tail_riak by using riak post-commit.
7
+
8
+ Pros
9
+ ----
10
+
11
+ - easy operations
12
+ - high availability
13
+ - horizontal scalability (esp. write performance)
14
+ - good night sleep
15
+
16
+ Cons
17
+ ----
18
+
19
+ - no capped table, TTL objects
20
+
21
+
22
+ License
23
+ =======
24
+
25
+ Apache 2.0
26
+
27
+ Copyright Kota UENISHI
28
+
29
+ Many Thanks to fluent-plugin-mongo
data/Rakefile ADDED
@@ -0,0 +1,17 @@
1
+ require 'bundler'
2
+ Bundler::GemHelper.install_tasks
3
+
4
+ require 'rake/testtask'
5
+
6
+ Rake::TestTask.new(:test) do |test|
7
+ test.libs << 'lib' << 'test'
8
+ test.test_files = FileList['test/plugin/*.rb']
9
+ test.verbose = true
10
+ end
11
+
12
+ task :coverage do |t|
13
+ ENV['SIMPLE_COV'] = '1'
14
+ Rake::Task["test"].invoke
15
+ end
16
+
17
+ task :default => [:build]
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.0.1
@@ -0,0 +1,24 @@
1
+ # encoding: utf-8
2
+ $:.push File.expand_path('../lib', __FILE__)
3
+
4
+ Gem::Specification.new do |gem|
5
+ gem.name = "fluent-plugin-riak"
6
+ gem.description = "Riak plugin for Fluent event collector"
7
+ gem.homepage = "https://github.com/kuenishi/fluent-plugin-riak"
8
+ gem.summary = gem.description
9
+ gem.version = File.read("VERSION").strip
10
+ gem.authors = ["Kota UENISHI"]
11
+ gem.email = "kuenishi@gmail.com"
12
+ gem.has_rdoc = false
13
+ #gem.platform = Gem::Platform::RUBY
14
+ gem.files = `git ls-files`.split("\n")
15
+ gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
16
+ gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
17
+ gem.require_paths = ['lib']
18
+
19
+ gem.add_dependency "fluentd", "~> 0.10.7"
20
+ gem.add_dependency "riak-client", "~> 1.0.0"
21
+ gem.add_development_dependency "rake", ">= 0.9.2"
22
+ gem.add_development_dependency "simplecov", ">= 0.5.4"
23
+ gem.add_development_dependency "rr", ">= 1.0.0"
24
+ end
@@ -0,0 +1,68 @@
1
+ module Fluent
2
+
3
+ class RiakOutput < BufferedOutput
4
+
5
+ Fluent::Plugin.register_output('riak', self)
6
+ include SetTimeKeyMixin
7
+ config_set_default :include_tag_key, false
8
+ include SetTagKeyMixin
9
+ config_set_default :include_time_key, true
10
+
11
+ config_param :nodes, :string, :default => "localhost:8081"
12
+
13
+ def initialize
14
+ super
15
+ require 'riak'
16
+ require 'msgpack'
17
+ end
18
+
19
+ def configure(conf)
20
+ super
21
+
22
+ @nodes = @nodes.split(',').map{ |s|
23
+ ip,port = s.split(':')
24
+ {:host => ip, :pb_port => port.to_i}
25
+ }
26
+ $log.info "riak nodes=#{@nodes}"
27
+ end
28
+
29
+ def start
30
+ $log.debug " => #{@buffer.chunk_limit} #{@buffer.queue_limit} "
31
+ @conn = Riak::Client.new(:nodes => @nodes, :protocol => "pbc")
32
+ @bucket = @conn.bucket("fluentlog")
33
+ @buf = {}
34
+
35
+ super
36
+ end
37
+
38
+ def format(tag, time, record)
39
+ [time, record].to_msgpack
40
+ end
41
+
42
+ def write(chunk)
43
+ $log.warn " <<<<<===========\n"
44
+
45
+ records = []
46
+ chunk.msgpack_each { |time, record|
47
+ # TODO: time processing and tag processing
48
+ records << record
49
+ $log.debug record
50
+ }
51
+ put_now(records)
52
+ end
53
+
54
+ private
55
+
56
+ # TODO: add index for some analysis
57
+ def put_now(obj)
58
+ key = Time.now.to_i.to_s
59
+ robj = Riak::RObject.new(@bucket, key)
60
+ robj.raw_data = obj.to_json
61
+ robj.content_type = 'text/json'
62
+ robj.store
63
+ key
64
+ end
65
+
66
+ end
67
+
68
+ end
data/listkeys.rb ADDED
@@ -0,0 +1,36 @@
1
+ require 'riak'
2
+
3
+ nodes = [{ :host => "192.168.62.129",
4
+ :pb_port => 8081 }]
5
+
6
+
7
+ c = Riak::Client.new( :nodes => nodes, :protocol => "pbc" )
8
+ #c.list_buckets
9
+ b = c.bucket("fluentlog")
10
+
11
+ def put(bucket)
12
+ o = Riak::RObject.new(bucket, "hoge")
13
+ o.content_type = "text/plain"
14
+ o.raw_data = "hogehoge"
15
+ o.store
16
+ end
17
+
18
+ def get(bucket)
19
+ o = bucket.get("hoge")
20
+ p o.content_type
21
+ p o.raw_data
22
+ end
23
+
24
+ def keys(bucket)
25
+ bucket.keys { |ks|
26
+ ks.each { |k|
27
+ o = bucket.get(k)
28
+ print "#{o.bucket.name}/#{o.key} (#{o.content_type})\n"
29
+ print " => #{o.raw_data}\n"
30
+ }
31
+ }
32
+ end
33
+
34
+ #put b
35
+ #get b
36
+ keys b
data/loader.rb ADDED
@@ -0,0 +1,21 @@
1
+ require 'net/http'
2
+
3
+ # uri = URI("http://192.168.62.128:8888/riak.test")
4
+
5
+ def run(n, uri)
6
+ (0..n).each do |i|
7
+ json = "{\"hoge\":\"valueoooooooo ???? special? \",\"id\":#{i}}"
8
+ res = Net::HTTP.post_form(uri, 'json' => json)
9
+ if (i % 1000) == 0 then
10
+ print "#{i}: #{res.code}\n"
11
+ end
12
+ end
13
+ end
14
+
15
+ uri = URI("http://192.168.100.128:8888/riak.test")
16
+ n = 1024*1024
17
+ start = Time.now
18
+ run(n, uri)
19
+ duration = (Time.now - start).to_i
20
+ mps = n * 1.0 / duration
21
+ print "#{n} messages in #{duration} secs: #{mps} mps.\n"
metadata ADDED
@@ -0,0 +1,139 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-riak
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Kota UENISHI
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2013-02-12 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: fluentd
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ~>
20
+ - !ruby/object:Gem::Version
21
+ version: 0.10.7
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ~>
28
+ - !ruby/object:Gem::Version
29
+ version: 0.10.7
30
+ - !ruby/object:Gem::Dependency
31
+ name: riak-client
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ~>
36
+ - !ruby/object:Gem::Version
37
+ version: 1.0.0
38
+ type: :runtime
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ~>
44
+ - !ruby/object:Gem::Version
45
+ version: 1.0.0
46
+ - !ruby/object:Gem::Dependency
47
+ name: rake
48
+ requirement: !ruby/object:Gem::Requirement
49
+ none: false
50
+ requirements:
51
+ - - ! '>='
52
+ - !ruby/object:Gem::Version
53
+ version: 0.9.2
54
+ type: :development
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
58
+ requirements:
59
+ - - ! '>='
60
+ - !ruby/object:Gem::Version
61
+ version: 0.9.2
62
+ - !ruby/object:Gem::Dependency
63
+ name: simplecov
64
+ requirement: !ruby/object:Gem::Requirement
65
+ none: false
66
+ requirements:
67
+ - - ! '>='
68
+ - !ruby/object:Gem::Version
69
+ version: 0.5.4
70
+ type: :development
71
+ prerelease: false
72
+ version_requirements: !ruby/object:Gem::Requirement
73
+ none: false
74
+ requirements:
75
+ - - ! '>='
76
+ - !ruby/object:Gem::Version
77
+ version: 0.5.4
78
+ - !ruby/object:Gem::Dependency
79
+ name: rr
80
+ requirement: !ruby/object:Gem::Requirement
81
+ none: false
82
+ requirements:
83
+ - - ! '>='
84
+ - !ruby/object:Gem::Version
85
+ version: 1.0.0
86
+ type: :development
87
+ prerelease: false
88
+ version_requirements: !ruby/object:Gem::Requirement
89
+ none: false
90
+ requirements:
91
+ - - ! '>='
92
+ - !ruby/object:Gem::Version
93
+ version: 1.0.0
94
+ description: Riak plugin for Fluent event collector
95
+ email: kuenishi@gmail.com
96
+ executables: []
97
+ extensions: []
98
+ extra_rdoc_files: []
99
+ files:
100
+ - .gitignore
101
+ - Gemfile
102
+ - README.md
103
+ - Rakefile
104
+ - VERSION
105
+ - fluent-plugin-riak.gemspec
106
+ - lib/fluent/plugin/out_riak.rb
107
+ - listkeys.rb
108
+ - loader.rb
109
+ homepage: https://github.com/kuenishi/fluent-plugin-riak
110
+ licenses: []
111
+ post_install_message:
112
+ rdoc_options: []
113
+ require_paths:
114
+ - lib
115
+ required_ruby_version: !ruby/object:Gem::Requirement
116
+ none: false
117
+ requirements:
118
+ - - ! '>='
119
+ - !ruby/object:Gem::Version
120
+ version: '0'
121
+ segments:
122
+ - 0
123
+ hash: -2864304810169974857
124
+ required_rubygems_version: !ruby/object:Gem::Requirement
125
+ none: false
126
+ requirements:
127
+ - - ! '>='
128
+ - !ruby/object:Gem::Version
129
+ version: '0'
130
+ segments:
131
+ - 0
132
+ hash: -2864304810169974857
133
+ requirements: []
134
+ rubyforge_project:
135
+ rubygems_version: 1.8.23
136
+ signing_key:
137
+ specification_version: 3
138
+ summary: Riak plugin for Fluent event collector
139
+ test_files: []