fluent-plugin-riak 0.0.1 → 0.0.2

data/Gemfile

@@ -3,3 +3,4 @@ source :rubygems
 gemspec
 
 gem "simplecov", :require => false
+gem "uuidtools", :require => ">= 2.1.3"

data/README.md

@@ -5,6 +5,36 @@ fluent-plugin-riak is a alternative for people who are not sufficient with mongo
 
 Current status is still proof-of-concept: index setting and its configuration are to be decided. Also performance optimization is required. Another idea is in_tail_riak by using riak post-commit.
 
+
+fluent.conf example
+-------------------
+
+```
+<match riak.**>
+  type riak
+
+  buffer_type memory
+  flush_interval 10s
+  retry_limit 5
+  retry_wait 1s
+  buffer_chunk_limit 256m
+  buffer_queue_limit 8096
+
+  # pb port
+  nodes 127.0.0.1:8087
+  #for cluster, define multiple machines
+  #nodes 192.168.100.128:10018 129.168.100.128:10028 
+</match>
+
+```
+
+key format -> 2013-02-<uuid>
+value format -> [records] in JSON
+index:
+  year_int -> year
+  month_bin -> <year>-<month>
+  tag_bin -> tags
+
 Pros
 ----
 

data/VERSION

@@ -1 +1 @@
-0.0.1
+0.0.2

data/fluent-plugin-riak.gemspec

@@ -18,6 +18,7 @@ Gem::Specification.new do |gem|
 
   gem.add_dependency "fluentd", "~> 0.10.7"
   gem.add_dependency "riak-client", "~> 1.0.0"
+  gem.add_dependency "uuidtools", ">= 2.1.3"
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "simplecov", ">= 0.5.4"
   gem.add_development_dependency "rr", ">= 1.0.0"

data/lib/fluent/plugin/out_riak.rb

@@ -4,16 +4,17 @@ class RiakOutput < BufferedOutput
 
   Fluent::Plugin.register_output('riak', self)
   include SetTimeKeyMixin
-  config_set_default :include_tag_key, false
+  config_set_default :include_tag_key, true
   include SetTagKeyMixin
   config_set_default :include_time_key, true
 
-  config_param :nodes, :string, :default => "localhost:8081"
+  config_param :nodes, :string, :default => "localhost:8087"
 
   def initialize
     super
     require 'riak'
     require 'msgpack'
+    require 'uuidtools'
   end
 
   def configure(conf)
@@ -36,31 +37,39 @@ class RiakOutput < BufferedOutput
   end
 
   def format(tag, time, record)
-    [time, record].to_msgpack
+    [time, tag, record].to_msgpack
   end
 
   def write(chunk)
-    $log.warn " <<<<<===========\n"
+    $log.debug " <<<<<===========\n"
 
     records  = []
-    chunk.msgpack_each { |time, record|
-      # TODO: time processing and tag processing
+    tags = []
+    chunk.msgpack_each { |time, tag, record|
+      record[@tag_key] = tag
+      tags << tag
       records << record
       $log.debug record
     }
-    put_now(records)
+    put_now(records, tags)
   end
 
   private
 
   # TODO: add index for some analysis
-  def put_now(obj)
-    key = Time.now.to_i.to_s
+  def put_now(records, tags)
+    today = Date.today
+    key = "#{today.to_s}-#{UUIDTools::UUID.random_create.to_s}"
     robj = Riak::RObject.new(@bucket, key)
-    robj.raw_data = obj.to_json
-    robj.content_type = 'text/json'
+    robj.raw_data = records.to_json
+    robj.indexes['year_int'] << today.year
+    robj.indexes['month_bin'] << "#{today.year}-#{today.month}"
+    tags.each do |tag|
+      robj.indexes['tag_bin'] << tag
+    end
+    robj.content_type = 'application/json'
     robj.store
-    key
+    robj
   end
 
 end

data/listkeys.rb

@@ -1,7 +1,7 @@
 require 'riak'
 
-nodes = [{ :host => "192.168.62.129",
-           :pb_port => 8081 }]
+nodes = [{ :host => "127.0.0.1",
+           :pb_port => 8087 }]
 
 
 c = Riak::Client.new( :nodes => nodes, :protocol => "pbc" )

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-riak
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -43,6 +43,22 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: uuidtools
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.1.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.1.3
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -120,7 +136,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2864304810169974857
+      hash: 2812051443307104688
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -129,7 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2864304810169974857
+      hash: 2812051443307104688
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23