fluent-plugin-riak2 0.0.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 03e66f7515f40f6b25b945a90dd5e8fd0e3e2bf6
+  data.tar.gz: 3c32370f5c55cd950c8c439e8919f8fffed7e98d
+SHA512:
+  metadata.gz: d050d1bf3fe03dfc50ba78cffce9df5862d5190de55e6f5727274a485d6d880ab948b791e1363101d8b828322ed07bd3707ef52aa9296890c00fc2bd4f3c43b8
+  data.tar.gz: e33348d42363c08598e4450c32201284351e526458cd817a0b0828ff33c73777e385c1d918c6961a0cddeade29c05a2c5be5f8a100d29e8fc2f04ace964e1f35

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*~
+Gemfile.lock
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem "simplecov", :require => false
+gem "uuidtools", :require => ">= 2.1.3"

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2014 Kota UENISHI, Collective Health, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,74 @@
+[fluent-plugin-riak2](https://github.com/collectivehealth/fluent-plugin-riak2g), a plugin for [Fluentd](http://fluentd.org)
+==================
+
+
+`fluent-plugin-riak2` is a fluentd output plugin designed to stuff log messages into a riak cluster.
+
+This version is based on the work of [Kota UENISHI's fluent-plugin-riak](https://github.com/kuenishi/fluent-plugin-riak).  We are very thankful for his effort and his decision to release the work under the Apache 2 license.
+
+`fluent-plugin-riak2` is designed to be used with Riak 2.x clusters and it's yokozuna/solr based search engine.  Support for secondary indicies is limited and should be considered deprecated.
+
+Riak ( http://github.com/basho/riak ) is an open-source distributed KVS focused on availability.
+
+Current status is still proof-of-concept: index setting and its configuration are to be decided. Also performance optimization is required. Another idea is in_tail_riak by using riak post-commit.
+
+installation
+------------
+
+```bash
+$ sudo gem install fluent-plugin-riak2
+```
+
+Notice: you need Riak configured using eleveldb as backend.
+
+
+fluent.conf example
+-------------------
+
+```
+<match riak2.**>
+  type riak2
+
+  buffer_type memory
+  flush_interval 10s
+  retry_limit 5
+  retry_wait 1s
+  buffer_chunk_limit 256m
+  buffer_queue_limit 8096
+
+  # pb port
+  nodes 127.0.0.1:8087
+  #for cluster, define multiple machines
+  #nodes 192.168.100.128:10018 129.168.100.128:10028 
+</match>
+
+```
+
+- key format -> 2013-02-<uuid>
+- value format -> [records] in JSON
+- index:
+
+ - year_int -> year
+ - month_bin -> <year>-<month>
+ - tag_bin -> tags
+
+easy querying log
+-----------------
+
+```bash
+$ curl -X PUT http://localhost:8098/buckets/static/keys/browser.html -H 'Content-type: text/html' -d @browser.html
+$ open http://localhost:8098/buckets/static/keys/browser.html
+```
+
+License
+=======
+
+Apache 2.0
+
+Copyright Kota UENISHI, Collective Health, Inc.
+
+Kota UENISHI's original message:
+
+Many Thanks to fluent-plugin-mongo
+
+Collective Health, Inc. would like to thank Kota UENISHI for his effort and his decision to release the work under the Apache 2 license.

data/Rakefile

@@ -0,0 +1,17 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/plugin/*.rb']
+  test.verbose = true
+end
+
+task :coverage do |t|
+  ENV['SIMPLE_COV'] = '1'
+  Rake::Task["test"].invoke
+end
+
+task :default => [:build]

data/VERSION

@@ -0,0 +1 @@
+0.0.4

data/browser.html

@@ -0,0 +1,178 @@
+<html>
+<head>
+  <title>Riak mapreducer</title>
+  <style type="text/css">
+body {
+  background: #678; text-align: center; width: 100%; font-family: 'Titillium', Monospace;
+}
+body div {
+  width: 80%; background: #ddd;
+  padding: 20px; text-align: left; margin: 0px auto;
+}
+h2 {
+  padding: 0px;
+  margin: 0px auto;
+}
+a {
+  text-shadow: 1px 1px #888888;
+  text-decoration:underline;
+}
+  </style>
+  <script lang="JavaScript">
+    function get_map(){    return document.getElementById("map").value; }
+    function get_reduce(){ return document.getElementById("reduce").value; }
+    function get_bucketname(){ return document.getElementById("bucket").value; }
+    function exec_query(m, r){
+      var q = { inputs : get_bucketname(),
+      query : [
+        { map:    { language: "javascript", source: m}},
+        { reduce: { language: "javascript", source: r}}
+      ]};
+      var xmlhttp = new XMLHttpRequest();
+      xmlhttp.open("POST", "/mapred", false);
+      xmlhttp.setRequestHeader("Content-Type", "application/json");
+      xmlhttp.send(JSON.stringify(q, null, ""));
+      var result = xmlhttp.responseText;
+
+      var resultBox = document.getElementById("result");
+      resultBox.innerText = result;
+    }
+  </script>
+</head>
+
+<body>
+  <div id="content">
+    <div style="text-align:right">
+    <script lang="JavaScript">
+    </script>
+    bucket name:<input type="text" id="bucket" value="fluentlog"/>
+    </div>
+
+    <h2>Run mapreduce on fluentd log</h2>
+    <table style="border: 1px solid black;">
+      <tr>
+        <th>map</th> <th>reduce</th>
+      </tr>
+      <tr>
+        <td>
+<textarea name="map" id="map" rows="16" cols="50">
+function(v){
+  var data = v.values[0].data;
+  var c = JSON.parse(data);
+  return [c.count];
+}
+</textarea>
+          </td><td>
+<textarea name="reduce" id="reduce" rows="16" cols="50" >
+function(v){
+  var sum = 0;
+  for( var i=0; i < v.length; ++i){
+    sum += v[i];
+  }
+  return [sum];
+}
+</textarea>
+        </td>
+      </tr>
+      <tr>
+        <th colspan="2"> <a onClick="exec_query(get_map(), get_reduce());">query</a> </th>
+      </tr>
+      <tr>
+        <td width="200px" colspan="2">
+    <div id="result" style="font: bold;"></div>
+    </td>
+    </tr>
+      <tr><td colspan="3">
+          <script lang="JavaScript">
+            function grep(){
+            
+            }
+          </script>
+          <script lang="JavaScript" type="text/javascript">
+            function count_rank(){
+            var m = "function(v){
+      var data = v.values[0].data;
+      var c = JSON.parse(data);
+      var tags = {};
+      for(var i=0; i<c.length; ++i){
+        count = tags[c[i].tag];
+        if(count == null){ tags[c[i].tag] = 1; }
+        else{              tags[c[i].tag] = count+1; }
+      }
+      return [tags];
+    }";
+      var r = "function(v){
+      var tags = {};
+      for(var i=0; i<v.length; ++i){
+        for (var prop in v[i]) {
+          count = tags[prop];
+          if(count == null){ tags[prop] = v[i][prop]; }
+          else{              tags[prop] = count+v[i][prop]; }
+        }
+      }
+      return [tags];
+    }";
+            exec_query(m, r);
+          }
+          </script>
+
+          <!-- <a onClick="grep();">grep</a> <input type="text" id="grep" value="debug"/> -->
+          <a onClick="count_rank();">count&amp;rank</a> <input type="text" id="count_rank" value="tag"/>
+      </td></tr>
+    </table>
+
+    <hr/>
+    <h2>Riak object viewer</h2>
+    <script lang="JavaScript">
+      function get(){
+      var key = document.getElementById("key").value;
+      var xmlhttp = new XMLHttpRequest();
+      xmlhttp.open("GET", "/buckets/"+ get_bucketname() +"/keys/"+key, false);
+      xmlhttp.send();
+      var result = xmlhttp.responseText;
+      console.log(result);
+      document.getElementById("object").innerText = result;
+      }
+    </script>
+    <a onClick="get();" >get</a> <input type="text" id="key" />
+    <div id="object"></div>
+    
+    
+    <h2>Riak keylister</h2>
+    <script lang="JavaScript">
+      function listkeys(){
+      var xmlhttp = new XMLHttpRequest();
+      xmlhttp.open("GET", "/buckets/"+ get_bucketname() +"/keys?keys=true", false);
+      xmlhttp.send();
+      var result = JSON.parse(xmlhttp.responseText);
+      console.log(result);
+      
+      document.getElementById("keylist").innerText = result.keys.sort().join("\n");
+      }
+    </script>
+    <a onClick="listkeys();" >listkeys</a>
+    <div id="keylist"></div>
+    <hr/>
+    (C)@kuenishi distributed under Apache 2 license
+  </div>
+<!--
+    <h2>Riak object putter</h2>
+    <script lang="JavaScript">
+      function put(){
+      var key = document.getElementById("key2").value;
+      var value = document.getElementById("value").value;
+      var xmlhttp = new XMLHttpRequest();
+      xmlhttp.open("PUT", "/buckets/fluentlog/keys/"+key, false);
+      xmlhttp.setRequestHeader("Content-Type", "application/json");
+      xmlhttp.send(value);
+      var result = xmlhttp.responseText;
+      console.log(result);
+      document.getElementById("object2").innerText = result;
+      }
+    </script>
+     <a onClick="put();" >put</a> <input type="text" id="key2" /> <input type="text" id="value" />
+    <div id="object2"></div>
+-->
+
+</body>
+</html>

data/fluent-plugin-riak2.gemspec

@@ -0,0 +1,26 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-riak2"
+  gem.description = "Riak 2.x plugin for Fluent event collector"
+  gem.homepage    = "https://github.com/collectivehealth/fluent-plugin-riak2"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.license     = 'Apache-2.0'
+  gem.authors     = ["Kota UENISHI", "Matt Nunogawa"]
+  gem.email       = "matt@collectivehealth.com"
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", "~> 0.10"
+  gem.add_dependency "riak-client", "~> 2.1.0"
+  gem.add_dependency "uuidtools", ">= 2.1.3"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "simplecov", ">= 0.5.4"
+  gem.add_development_dependency "rr", ">= 1.0.0"
+end

data/lib/fluent/plugin/out_riak2.rb

@@ -0,0 +1,80 @@
+module Fluent
+
+class Riak2Output < BufferedOutput
+
+  Fluent::Plugin.register_output('riak2', self)
+  include SetTimeKeyMixin
+  config_set_default :include_tag_key, true
+  include SetTagKeyMixin
+  config_set_default :include_time_key, true
+
+  config_param :bucket_name, :string, :default => "fluentlog"
+  config_param :nodes, :string, :default => "localhost:8087"
+
+  def initialize
+    super
+    require 'riak'
+    require 'msgpack'
+    require 'uuidtools'
+  end
+
+  def configure(conf)
+    super
+
+    @nodes = @nodes.split(',').map{ |s|
+      ip,port = s.split(':')
+      {:host => ip, :pb_port => port.to_i}
+    }
+    $log.info "riak nodes=#{@nodes}"
+  end
+
+  def start
+    $log.debug " => #{@buffer.chunk_limit} #{@buffer.queue_limit} "
+    @conn = Riak::Client.new(:nodes => @nodes)
+    @bucket = @conn.bucket(@bucket_name)
+    @buf = {}
+
+    super
+  end
+
+  def format(tag, time, record)
+    [time, tag, record].to_msgpack
+  end
+
+  def write(chunk)
+    $log.debug " <<<<<===========\n"
+
+    records  = []
+    tags = []
+    chunk.msgpack_each { |time, tag, record|
+      record[@tag_key] = tag
+      tags << tag
+      records << record
+      $log.debug record
+    }
+    put_now(records, tags)
+  end
+
+  private
+
+  # TODO: add index for some analysis
+  def put_now(records, tags)
+    if not records.empty? then
+      today = Date.today
+      key = "#{today.to_s}-#{UUIDTools::UUID.timestamp_create.to_s}"
+      robj = Riak::RObject.new(@bucket, key)
+      robj.raw_data = records.to_json
+      robj.indexes['year_int'] << today.year
+      robj.indexes['month_bin'] << "#{today.year}-#{"%02d" % today.month}"
+      tags.each do |tag|
+        robj.indexes['tag_bin'] << tag
+      end
+      robj.content_type = 'application/json'
+      robj.store
+      robj
+    end
+  end
+
+end
+
+end

data/listkeys.rb

@@ -0,0 +1,36 @@
+require 'riak'
+
+nodes = [{ :host => "127.0.0.1",
+           :pb_port => 8087 }]
+
+
+c = Riak::Client.new( :nodes => nodes, :protocol => "pbc" )
+#c.list_buckets
+b = c.bucket("fluentlog")
+
+def put(bucket)
+  o = Riak::RObject.new(bucket, "hoge")
+  o.content_type = "text/plain"
+  o.raw_data = "hogehoge"
+  o.store
+end
+
+def get(bucket)
+  o = bucket.get("hoge")
+  p o.content_type
+  p o.raw_data
+end
+
+def keys(bucket)
+  bucket.keys { |ks|
+    ks.each { |k|
+      o = bucket.get(k)
+      print "#{o.bucket.name}/#{o.key} (#{o.content_type})\n"
+      print " =>  #{o.raw_data}\n"
+    }
+  }
+end
+
+#put b
+#get b
+keys b

data/setup_index.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+HOST=$1
+
+echo setting up fluentlog_index at $HOST
+curl -X PUT http://${HOST}/yz/index/fluentlog_index
+echo "setting fluentlog_index to fluentlog bucket"
+curl -X PUT http://${HOST}/buckets/fluentlog/props -H 'Content-type:application/json' -d '{"props":{"yz_index":"fluentlog_index"}}'
+
+curl http://${HOST}/yz/index?index=true
+curl http://${HOST}/buckets/fluentlog/props

metadata

@@ -0,0 +1,139 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-riak2
+version: !ruby/object:Gem::Version
+  version: 0.0.4
+platform: ruby
+authors:
+- Kota UENISHI
+- Matt Nunogawa
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-12-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: riak-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: uuidtools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.1.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.1.3
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+- !ruby/object:Gem::Dependency
+  name: rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+description: Riak 2.x plugin for Fluent event collector
+email: matt@collectivehealth.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- VERSION
+- browser.html
+- fluent-plugin-riak2.gemspec
+- lib/fluent/plugin/out_riak2.rb
+- listkeys.rb
+- setup_index.sh
+homepage: https://github.com/collectivehealth/fluent-plugin-riak2
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Riak 2.x plugin for Fluent event collector
+test_files: []