fluent-plugin-riak 0.0.1

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*~
+Gemfile.lock
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/Gemfile

@@ -0,0 +1,5 @@
+source :rubygems
+
+gemspec
+
+gem "simplecov", :require => false

data/README.md

@@ -0,0 +1,29 @@
+fluent-plugin-riak
+==================
+
+fluent-plugin-riak is a alternative for people who are not sufficient with mongo or webhdfs. Riak ( http://github.com/basho/riak ) is an open-source distributed KVS focused on availability. It also has a strong query system with secondary index (2i): see docs ( http://docs.basho.com/riak/latest/tutorials/querying/ ) for details.
+
+Current status is still proof-of-concept: index setting and its configuration are to be decided. Also performance optimization is required. Another idea is in_tail_riak by using riak post-commit.
+
+Pros
+----
+
+- easy operations
+- high availability
+- horizontal scalability (esp. write performance)
+- good night sleep
+
+Cons
+----
+
+- no capped table, TTL objects
+
+
+License
+=======
+
+Apache 2.0
+
+Copyright Kota UENISHI
+
+Many Thanks to fluent-plugin-mongo

data/Rakefile

@@ -0,0 +1,17 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/plugin/*.rb']
+  test.verbose = true
+end
+
+task :coverage do |t|
+  ENV['SIMPLE_COV'] = '1'
+  Rake::Task["test"].invoke
+end
+
+task :default => [:build]

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/fluent-plugin-riak.gemspec

@@ -0,0 +1,24 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-riak"
+  gem.description = "Riak plugin for Fluent event collector"
+  gem.homepage    = "https://github.com/kuenishi/fluent-plugin-riak"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Kota UENISHI"]
+  gem.email       = "kuenishi@gmail.com"
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", "~> 0.10.7"
+  gem.add_dependency "riak-client", "~> 1.0.0"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "simplecov", ">= 0.5.4"
+  gem.add_development_dependency "rr", ">= 1.0.0"
+end

data/lib/fluent/plugin/out_riak.rb

@@ -0,0 +1,68 @@
+module Fluent
+
+class RiakOutput < BufferedOutput
+
+  Fluent::Plugin.register_output('riak', self)
+  include SetTimeKeyMixin
+  config_set_default :include_tag_key, false
+  include SetTagKeyMixin
+  config_set_default :include_time_key, true
+
+  config_param :nodes, :string, :default => "localhost:8081"
+
+  def initialize
+    super
+    require 'riak'
+    require 'msgpack'
+  end
+
+  def configure(conf)
+    super
+
+    @nodes = @nodes.split(',').map{ |s|
+      ip,port = s.split(':')
+      {:host => ip, :pb_port => port.to_i}
+    }
+    $log.info "riak nodes=#{@nodes}"
+  end
+
+  def start
+    $log.debug " => #{@buffer.chunk_limit} #{@buffer.queue_limit} "
+    @conn = Riak::Client.new(:nodes => @nodes, :protocol => "pbc")
+    @bucket = @conn.bucket("fluentlog")
+    @buf = {}
+
+    super
+  end
+
+  def format(tag, time, record)
+    [time, record].to_msgpack
+  end
+
+  def write(chunk)
+    $log.warn " <<<<<===========\n"
+
+    records  = []
+    chunk.msgpack_each { |time, record|
+      # TODO: time processing and tag processing
+      records << record
+      $log.debug record
+    }
+    put_now(records)
+  end
+
+  private
+
+  # TODO: add index for some analysis
+  def put_now(obj)
+    key = Time.now.to_i.to_s
+    robj = Riak::RObject.new(@bucket, key)
+    robj.raw_data = obj.to_json
+    robj.content_type = 'text/json'
+    robj.store
+    key
+  end
+
+end
+
+end

data/listkeys.rb

@@ -0,0 +1,36 @@
+require 'riak'
+
+nodes = [{ :host => "192.168.62.129",
+           :pb_port => 8081 }]
+
+
+c = Riak::Client.new( :nodes => nodes, :protocol => "pbc" )
+#c.list_buckets
+b = c.bucket("fluentlog")
+
+def put(bucket)
+  o = Riak::RObject.new(bucket, "hoge")
+  o.content_type = "text/plain"
+  o.raw_data = "hogehoge"
+  o.store
+end
+
+def get(bucket)
+  o = bucket.get("hoge")
+  p o.content_type
+  p o.raw_data
+end
+
+def keys(bucket)
+  bucket.keys { |ks|
+    ks.each { |k|
+      o = bucket.get(k)
+      print "#{o.bucket.name}/#{o.key} (#{o.content_type})\n"
+      print " =>  #{o.raw_data}\n"
+    }
+  }
+end
+
+#put b
+#get b
+keys b

data/loader.rb

@@ -0,0 +1,21 @@
+require 'net/http'
+
+# uri = URI("http://192.168.62.128:8888/riak.test")
+
+def run(n, uri)
+  (0..n).each do |i|
+    json = "{\"hoge\":\"valueoooooooo ???? special? \",\"id\":#{i}}"
+    res = Net::HTTP.post_form(uri, 'json' => json)
+    if (i % 1000) == 0 then
+      print "#{i}: #{res.code}\n"
+    end
+  end
+end
+
+uri = URI("http://192.168.100.128:8888/riak.test")
+n = 1024*1024
+start = Time.now
+run(n, uri)
+duration = (Time.now - start).to_i
+mps = n * 1.0 / duration
+print "#{n} messages in #{duration} secs: #{mps} mps.\n"

metadata

@@ -0,0 +1,139 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-riak
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Kota UENISHI
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-02-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.7
+- !ruby/object:Gem::Dependency
+  name: riak-client
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+- !ruby/object:Gem::Dependency
+  name: rr
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+description: Riak plugin for Fluent event collector
+email: kuenishi@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-riak.gemspec
+- lib/fluent/plugin/out_riak.rb
+- listkeys.rb
+- loader.rb
+homepage: https://github.com/kuenishi/fluent-plugin-riak
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -2864304810169974857
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -2864304810169974857
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Riak plugin for Fluent event collector
+test_files: []