fluent-plugin-riak2 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 03e66f7515f40f6b25b945a90dd5e8fd0e3e2bf6
-  data.tar.gz: 3c32370f5c55cd950c8c439e8919f8fffed7e98d
+  metadata.gz: ad3f880d225f6869f8b4086695341e479f7535ed
+  data.tar.gz: a3b7636d8e86ce1afffb5aa91947c5e242b8ed3f
 SHA512:
-  metadata.gz: d050d1bf3fe03dfc50ba78cffce9df5862d5190de55e6f5727274a485d6d880ab948b791e1363101d8b828322ed07bd3707ef52aa9296890c00fc2bd4f3c43b8
-  data.tar.gz: e33348d42363c08598e4450c32201284351e526458cd817a0b0828ff33c73777e385c1d918c6961a0cddeade29c05a2c5be5f8a100d29e8fc2f04ace964e1f35
+  metadata.gz: fed5a2cbbd620bc2b682ac5de6b91267c8ea09a9829c0ed08c3af7d34a42c4a8147afc74e77533acfe7c6601fc74b87612cf4170a945806c9ff592da7bd16604
+  data.tar.gz: 1af9e4bbc5f851f53730106c962e52d676cc821ef975e02f87dc57ad20e77b2817dcb15652e35a70fc2e142b450dde7b9db380080fe3e12b320861dcdedd8654

data/CHANGELOG

@@ -0,0 +1,13 @@
+0.0.5
+=====
+
+- update to support riak 2.0 bucket types
+- update record to support riak search 2.0 (yokozuna) style json extraction
+- write to a metadata crdt map bucket names
+
+0.0.4
+=====
+
+- hard fork from https://github.com/kuenishi/fluent-plugin-riak , see README for more info
+- bare minimum to support riak-ruby-client v2.1
+- change UUID to v1 (timestamp based)

data/VERSION

@@ -1 +1 @@
-0.0.4
+0.0.5

data/fluent-plugin-riak2.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |gem|
 
   gem.add_dependency "fluentd", "~> 0.10"
   gem.add_dependency "riak-client", "~> 2.1.0"
-  gem.add_dependency "uuidtools", ">= 2.1.3"
+  gem.add_dependency "uuidtools", "~> 2.1.3"
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "simplecov", ">= 0.5.4"
   gem.add_development_dependency "rr", ">= 1.0.0"

data/lib/fluent/plugin/out_riak2.rb

@@ -8,7 +8,9 @@ class Riak2Output < BufferedOutput
   include SetTagKeyMixin
   config_set_default :include_time_key, true
 
+  config_param :bucket_type, :string, :default => "fluentlog" # we assume that search is on and this bucket type has an search index attached to it.
   config_param :bucket_name, :string, :default => "fluentlog"
+  config_param :riak2_metadata_bucket_type, :string, :default => ""
   config_param :nodes, :string, :default => "localhost:8087"
 
   def initialize
@@ -30,10 +32,31 @@ class Riak2Output < BufferedOutput
 
   def start
     $log.debug " => #{@buffer.chunk_limit} #{@buffer.queue_limit} "
-    @conn = Riak::Client.new(:nodes => @nodes)
-    @bucket = @conn.bucket(@bucket_name)
+    @client = Riak::Client.new(:nodes => @nodes)
+    @bucket = @client.bucket(@bucket_name)
     @buf = {}
 
+    # $log.debug "riak2_metadata_bucket_type => #{@riak2_metadata_bucket_type}"
+    # $log.debug "bucket_type => #{@bucket_type}"
+
+    if not @riak2_metadata_bucket_type.empty? then
+      # Here we are storing our bucket type and bucket name in a metadata map.  This allows clients to query that map to see a list of all fluentd buckets.
+
+      # bucket_type/name/key is returns a metadata map
+      metadata_bucket_type = @riak2_metadata_bucket_type        # config defined bucket type
+      metadata_bucket_name = "fluent-plugin-riak2-metadata"     # bucket name
+      metadata_key         = "fluent-plugin-riak2-metadata-key" # root level key for our metadata map
+
+      # our metadata map has a kv where:
+      # 1. key is set_of_logfile_buckets_key
+      # 2. value is a set of strings.  each string represents the bucket type and name for a single logfile
+      set_of_logfile_buckets_key = "all_buckets"                # inner key for our set of all logfile bucket type/name
+
+      mdbucket = @client.bucket(metadata_bucket_name)
+      Riak::Crdt::DEFAULT_BUCKET_TYPES[:map] = metadata_bucket_type
+      map = Riak::Crdt::Map.new(mdbucket, metadata_key)
+      map.sets[set_of_logfile_buckets_key].add "#{@bucket_type} #{@bucket_name}"
+    end
     super
   end
 
@@ -43,38 +66,38 @@ class Riak2Output < BufferedOutput
 
   def write(chunk)
     $log.debug " <<<<<===========\n"
-
     records  = []
-    tags = []
     chunk.msgpack_each { |time, tag, record|
       record[@tag_key] = tag
-      tags << tag
       records << record
       $log.debug record
     }
-    put_now(records, tags)
+    put_now(records)
   end
 
   private
 
-  # TODO: add index for some analysis
-  def put_now(records, tags)
+  def put_now(records)
     if not records.empty? then
-      today = Date.today
-      key = "#{today.to_s}-#{UUIDTools::UUID.timestamp_create.to_s}"
-      robj = Riak::RObject.new(@bucket, key)
-      robj.raw_data = records.to_json
-      robj.indexes['year_int'] << today.year
-      robj.indexes['month_bin'] << "#{today.year}-#{"%02d" % today.month}"
-      tags.each do |tag|
-        robj.indexes['tag_bin'] << tag
-      end
-      robj.content_type = 'application/json'
-      robj.store
-      robj
-    end
-  end
+      threads = []
+      records.each { |record|
+        #if you put log statements here, you must take care to NOT forward fluentd's logs to riak.  you will trigger a recursive avalance of riak storage activity.
+        now = DateTime.now.iso8601(9)
+        key = "#{now}-#{UUIDTools::UUID.timestamp_create.to_s}"
+        # $log.debug "#{@bucket_name} #{key} \n"
 
-end
+        # we are doing the somewhat granular storage, instead of chunked, at the record level.  This probably should be a config option.
+        threads << Thread.new {
+          robj = Riak::RObject.new(@bucket, key)
+          robj.content_type = "application/json"
+          raw_data = "{\"msg_s\":\"#{record["msg"]}\",\"time_dt\":\"#{record["time"]}\",\"src_s\":\"fluentd\"}"
+          # $log.debug "raw_data #{raw_data}\n"
+          robj.raw_data = raw_data
+          robj.store(type: @bucket_type)
+        }
+      }
+    end # if
+  end # put_now
+end # class Riak2Output
 
-end
+end # Module

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-riak2
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Kota UENISHI
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-01 00:00:00.000000000 Z
+date: 2014-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -43,14 +43,14 @@ dependencies:
   name: uuidtools
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
         version: 2.1.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
         version: 2.1.3
 - !ruby/object:Gem::Dependency
@@ -102,6 +102,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- CHANGELOG
 - Gemfile
 - LICENSE
 - README.md