fluent-plugin-riak2f 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8b7beab77e858e192eaada824cace1820196a979
+  data.tar.gz: 0136c5c1862058872fd54e3ba59e82b9326fd7e5
+SHA512:
+  metadata.gz: d409ac9126547b409b619075bf83e6f96e37bd4b93aacd7d4b873bb8dc30b2283aadd8cd3aa603268b986682441c71e2f0ee7466bcab2f63da0f09bb963f213d
+  data.tar.gz: d5d8f0da7053d1b89c5f6c902c457587ed4b3b231d7f014bb8dba7bd5334f821510cb4522118ec88b01bd1c249c145ed25476c727d63ed73c3666d9f880a2b2c

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*~
+Gemfile.lock
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/CHANGELOG

@@ -0,0 +1,13 @@
+0.0.5
+=====
+
+- update to support riak 2.0 bucket types
+- update record to support riak search 2.0 (yokozuna) style json extraction
+- write to a metadata crdt map bucket names
+
+0.0.4
+=====
+
+- hard fork from https://github.com/kuenishi/fluent-plugin-riak , see README for more info
+- bare minimum to support riak-ruby-client v2.1
+- change UUID to v1 (timestamp based)

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem "simplecov", :require => false
+gem "uuidtools", :require => ">= 2.1.3"

data/LICENSE.md

@@ -0,0 +1,13 @@
+Copyright 2014 Kota UENISHI, Collective Health, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,70 @@
+[fluent-plugin-riak2f](https://github.com/kyanagimoto/fluent-plugin-riak2), a plugin for [Fluentd](http://fluentd.org)
+==================
+
+
+`fluent-plugin-riak2f` is a fluentd output plugin designed to stuff log messages into a riak cluster.
+
+This version is based on the work of [fluent-plugin-riak2](https://github.com/collectivehealth/fluent-plugin-riak2).  We are very thankful for his effort and his decision to release the work under the Apache 2 license.
+
+`fluent-plugin-riak2` is designed to be used with Riak 2.x clusters and it's yokozuna/solr based search engine.  Support for secondary indicies is limited and should be considered deprecated.
+
+Riak ( http://github.com/basho/riak ) is an open-source distributed KVS focused on availability.
+
+Current status is still proof-of-concept: index setting and its configuration are to be decided. Also performance optimization is required. Another idea is in_tail_riak by using riak post-commit.
+
+installation
+------------
+
+```bash
+$ sudo gem install fluent-plugin-riak2f
+```
+
+Notice: you need Riak configured using eleveldb as backend.
+
+
+fluent.conf example
+-------------------
+
+```
+<match riak2.**>
+  type riak2
+
+  buffer_type memory
+  flush_interval 10s
+  retry_limit 5
+  retry_wait 1s
+  buffer_chunk_limit 256m
+  buffer_queue_limit 8096
+  bucket_type defalut # if not set, will use 'default'
+  bucket_name fluentdlog # if not set, will use 'fluentdlog'
+
+  # pb port
+  nodes 127.0.0.1:8087
+  #for cluster, define multiple machines
+  #nodes 192.168.100.128:10018 129.168.100.128:10028
+</match>
+
+```
+
+- key format -> %Y%m%d%H%M%S-<uuid>
+- value format -> [records] in JSON
+- index:
+
+ - year_int -> year
+ - month_bin -> <year>-<month>
+ - tag_bin -> tags
+
+easy querying log
+-----------------
+
+```bash
+$ curl -X PUT http://localhost:8098/buckets/static/keys/browser.html -H 'Content-type: text/html' -d @browser.html
+$ open http://localhost:8098/buckets/static/keys/browser.html
+```
+
+License
+=======
+
+Apache 2.0
+
+Copyright Kota UENISHI, Collective Health, Inc.

data/Rakefile

@@ -0,0 +1,17 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/plugin/*.rb']
+  test.verbose = true
+end
+
+task :coverage do |t|
+  ENV['SIMPLE_COV'] = '1'
+  Rake::Task["test"].invoke
+end
+
+task :default => [:build]

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/browser.html

@@ -0,0 +1,178 @@
+<html>
+<head>
+  <title>Riak mapreducer</title>
+  <style type="text/css">
+body {
+  background: #678; text-align: center; width: 100%; font-family: 'Titillium', Monospace;
+}
+body div {
+  width: 80%; background: #ddd;
+  padding: 20px; text-align: left; margin: 0px auto;
+}
+h2 {
+  padding: 0px;
+  margin: 0px auto;
+}
+a {
+  text-shadow: 1px 1px #888888;
+  text-decoration:underline;
+}
+  </style>
+  <script lang="JavaScript">
+    function get_map(){    return document.getElementById("map").value; }
+    function get_reduce(){ return document.getElementById("reduce").value; }
+    function get_bucketname(){ return document.getElementById("bucket").value; }
+    function exec_query(m, r){
+      var q = { inputs : get_bucketname(),
+      query : [
+        { map:    { language: "javascript", source: m}},
+        { reduce: { language: "javascript", source: r}}
+      ]};
+      var xmlhttp = new XMLHttpRequest();
+      xmlhttp.open("POST", "/mapred", false);
+      xmlhttp.setRequestHeader("Content-Type", "application/json");
+      xmlhttp.send(JSON.stringify(q, null, ""));
+      var result = xmlhttp.responseText;
+
+      var resultBox = document.getElementById("result");
+      resultBox.innerText = result;
+    }
+  </script>
+</head>
+
+<body>
+  <div id="content">
+    <div style="text-align:right">
+    <script lang="JavaScript">
+    </script>
+    bucket name:<input type="text" id="bucket" value="fluentlog"/>
+    </div>
+
+    <h2>Run mapreduce on fluentd log</h2>
+    <table style="border: 1px solid black;">
+      <tr>
+        <th>map</th> <th>reduce</th>
+      </tr>
+      <tr>
+        <td>
+<textarea name="map" id="map" rows="16" cols="50">
+function(v){
+  var data = v.values[0].data;
+  var c = JSON.parse(data);
+  return [c.count];
+}
+</textarea>
+          </td><td>
+<textarea name="reduce" id="reduce" rows="16" cols="50" >
+function(v){
+  var sum = 0;
+  for( var i=0; i < v.length; ++i){
+    sum += v[i];
+  }
+  return [sum];
+}
+</textarea>
+        </td>
+      </tr>
+      <tr>
+        <th colspan="2"> <a onClick="exec_query(get_map(), get_reduce());">query</a> </th>
+      </tr>
+      <tr>
+        <td width="200px" colspan="2">
+    <div id="result" style="font: bold;"></div>
+    </td>
+    </tr>
+      <tr><td colspan="3">
+          <script lang="JavaScript">
+            function grep(){
+            
+            }
+          </script>
+          <script lang="JavaScript" type="text/javascript">
+            function count_rank(){
+            var m = "function(v){
+      var data = v.values[0].data;
+      var c = JSON.parse(data);
+      var tags = {};
+      for(var i=0; i<c.length; ++i){
+        count = tags[c[i].tag];
+        if(count == null){ tags[c[i].tag] = 1; }
+        else{              tags[c[i].tag] = count+1; }
+      }
+      return [tags];
+    }";
+      var r = "function(v){
+      var tags = {};
+      for(var i=0; i<v.length; ++i){
+        for (var prop in v[i]) {
+          count = tags[prop];
+          if(count == null){ tags[prop] = v[i][prop]; }
+          else{              tags[prop] = count+v[i][prop]; }
+        }
+      }
+      return [tags];
+    }";
+            exec_query(m, r);
+          }
+          </script>
+
+          <!-- <a onClick="grep();">grep</a> <input type="text" id="grep" value="debug"/> -->
+          <a onClick="count_rank();">count&amp;rank</a> <input type="text" id="count_rank" value="tag"/>
+      </td></tr>
+    </table>
+
+    <hr/>
+    <h2>Riak object viewer</h2>
+    <script lang="JavaScript">
+      function get(){
+      var key = document.getElementById("key").value;
+      var xmlhttp = new XMLHttpRequest();
+      xmlhttp.open("GET", "/buckets/"+ get_bucketname() +"/keys/"+key, false);
+      xmlhttp.send();
+      var result = xmlhttp.responseText;
+      console.log(result);
+      document.getElementById("object").innerText = result;
+      }
+    </script>
+    <a onClick="get();" >get</a> <input type="text" id="key" />
+    <div id="object"></div>
+    
+    
+    <h2>Riak keylister</h2>
+    <script lang="JavaScript">
+      function listkeys(){
+      var xmlhttp = new XMLHttpRequest();
+      xmlhttp.open("GET", "/buckets/"+ get_bucketname() +"/keys?keys=true", false);
+      xmlhttp.send();
+      var result = JSON.parse(xmlhttp.responseText);
+      console.log(result);
+      
+      document.getElementById("keylist").innerText = result.keys.sort().join("\n");
+      }
+    </script>
+    <a onClick="listkeys();" >listkeys</a>
+    <div id="keylist"></div>
+    <hr/>
+    (C)@kuenishi distributed under Apache 2 license
+  </div>
+<!--
+    <h2>Riak object putter</h2>
+    <script lang="JavaScript">
+      function put(){
+      var key = document.getElementById("key2").value;
+      var value = document.getElementById("value").value;
+      var xmlhttp = new XMLHttpRequest();
+      xmlhttp.open("PUT", "/buckets/fluentlog/keys/"+key, false);
+      xmlhttp.setRequestHeader("Content-Type", "application/json");
+      xmlhttp.send(value);
+      var result = xmlhttp.responseText;
+      console.log(result);
+      document.getElementById("object2").innerText = result;
+      }
+    </script>
+     <a onClick="put();" >put</a> <input type="text" id="key2" /> <input type="text" id="value" />
+    <div id="object2"></div>
+-->
+
+</body>
+</html>

data/fluent-plugin-riak2f.gemspec

@@ -0,0 +1,26 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-riak2f"
+  gem.description = "Riak 2.x plugin for Fluent event collector"
+  gem.homepage    = "https://github.com/kyanagimoto/fluent-plugin-riak2f"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.license     = 'Apache-2.0'
+  gem.authors     = ["koichi yanagimoto"]
+  gem.email       = "kyanagi1107@gmail.com"
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", "~> 0.10"
+  gem.add_dependency "riak-client", "~> 2.1.0"
+  gem.add_dependency "uuidtools", "~> 2.1.3"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "simplecov", ">= 0.5.4"
+  gem.add_development_dependency "rr", ">= 1.0.0"
+end

data/lib/fluent/plugin/out_riak2.rb

@@ -0,0 +1,107 @@
+module Fluent
+  class Riak2Output < BufferedOutput
+    Fluent::Plugin.register_output('riak2', self)
+    include SetTimeKeyMixin
+    config_set_default :include_tag_key, true
+    include SetTagKeyMixin
+    config_set_default :include_time_key, true
+
+    config_param :bucket_type, :string, :default => "default"
+    config_param :bucket_name, :string, :default => "fluentlog"
+    config_param :riak2_metadata_bucket_type, :string, :default => ""
+    config_param :nodes, :string, :default => "localhost:8087"
+
+    def initialize
+      super
+      require 'riak'
+      require 'msgpack'
+      require 'uuidtools'
+    end
+
+    def configure(conf)
+      super
+      @nodes = @nodes.split(',').map do |s|
+        ip,port = s.split(':')
+        { :host => ip, :pb_port => port.to_i }
+      end
+      $log.info "riak nodes=#{@nodes}"
+    end
+
+    def start
+      $log.debug " => #{@buffer.chunk_limit} #{@buffer.queue_limit} "
+      @client = Riak::Client.new(:nodes => @nodes)
+      @bucket = @client.bucket(@bucket_name)
+      @buf = {}
+
+      # $log.debug "riak2_metadata_bucket_type => #{@riak2_metadata_bucket_type}"
+      # $log.debug "bucket_type => #{@bucket_type}"
+
+      unless @riak2_metadata_bucket_type.empty?
+        # Here we are storing our bucket type and bucket name in a metadata map.  This allows clients to query that map to see a list of all fluentd buckets.
+
+        # bucket_type/name/key is returns a metadata map
+        # config defined bucket type
+        metadata_bucket_type = @riak2_metadata_bucket_type
+        # bucket name
+        metadata_bucket_name = "fluent-plugin-riak2-metadata"
+        # root level key for our metadata map
+        metadata_key = "fluent-plugin-riak2-metadata-key"
+
+        # our metadata map has a kv where:
+        # 1. key is set_of_logfile_buckets_key
+        # 2. value is a set of strings.
+        #each string represents the bucket type and name for a single logfile
+        set_of_logfile_buckets_key = "all_buckets"
+        # inner key for our set of all logfile bucket type/name
+
+        mdbucket = @client.bucket(metadata_bucket_name)
+        Riak::Crdt::DEFAULT_BUCKET_TYPES[:map] = metadata_bucket_type
+        map = Riak::Crdt::Map.new(mdbucket, metadata_key)
+        map.sets[set_of_logfile_buckets_key].add "#{@bucket_type} #{@bucket_name}"
+      end
+      super
+    end
+
+    def format(tag, time, record)
+      [time, tag, record].to_msgpack
+    end
+
+    def write(chunk)
+      $log.debug " <<<<<===========\n"
+      records  = []
+      chunk.msgpack_each do |time, tag, record|
+        record[@tag_key] = tag
+        records << record
+        $log.debug record
+      end
+      put_now(records)
+    end
+
+    private
+
+    def put_now(records)
+      unless records.empty?
+        threads = []
+        records.each do |record|
+          # if you put log statements here,
+          # you must take care to NOT forward fluentd's logs to riak.
+          # you will trigger a recursive avalance of riak storage activity.
+          now = DateTime.now.strftime("%Y%m%d%H%M%S")
+          key = "#{now}-#{UUIDTools::UUID.timestamp_create.to_s}"
+          # $log.debug "#{@bucket_name} #{key} \n"
+
+          threads << Thread.new {
+            begin
+              robj = Riak::RObject.new(@bucket, key)
+              robj.content_type = "application/json"
+              robj.raw_data = record
+              robj.store(type: @bucket_type)
+            rescue => e
+              $log.error "ERROR #{e}"
+            end
+          }
+        end
+      end
+    end
+  end
+end

data/listkeys.rb

@@ -0,0 +1,36 @@
+require 'riak'
+
+nodes = [{ :host => "127.0.0.1",
+           :pb_port => 8087 }]
+
+
+c = Riak::Client.new( :nodes => nodes, :protocol => "pbc" )
+#c.list_buckets
+b = c.bucket("fluentlog")
+
+def put(bucket)
+  o = Riak::RObject.new(bucket, "hoge")
+  o.content_type = "text/plain"
+  o.raw_data = "hogehoge"
+  o.store
+end
+
+def get(bucket)
+  o = bucket.get("hoge")
+  p o.content_type
+  p o.raw_data
+end
+
+def keys(bucket)
+  bucket.keys { |ks|
+    ks.each { |k|
+      o = bucket.get(k)
+      print "#{o.bucket.name}/#{o.key} (#{o.content_type})\n"
+      print " =>  #{o.raw_data}\n"
+    }
+  }
+end
+
+#put b
+#get b
+keys b

data/setup_index.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+HOST=$1
+
+echo setting up fluentlog_index at $HOST
+curl -X PUT http://${HOST}/yz/index/fluentlog_index
+echo "setting fluentlog_index to fluentlog bucket"
+curl -X PUT http://${HOST}/buckets/fluentlog/props -H 'Content-type:application/json' -d '{"props":{"yz_index":"fluentlog_index"}}'
+
+curl http://${HOST}/yz/index?index=true
+curl http://${HOST}/buckets/fluentlog/props

metadata

@@ -0,0 +1,139 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-riak2f
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- koichi yanagimoto
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-06-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: riak-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: uuidtools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.3
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+- !ruby/object:Gem::Dependency
+  name: rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+description: Riak 2.x plugin for Fluent event collector
+email: kyanagi1107@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- CHANGELOG
+- Gemfile
+- LICENSE.md
+- README.md
+- Rakefile
+- VERSION
+- browser.html
+- fluent-plugin-riak2f.gemspec
+- lib/fluent/plugin/out_riak2.rb
+- listkeys.rb
+- setup_index.sh
+homepage: https://github.com/kyanagimoto/fluent-plugin-riak2f
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Riak 2.x plugin for Fluent event collector
+test_files: []