RubyGems - fluent-plugin-chriak2 - Versions diffs - 0.0.7 - Mend

fluent-plugin-chriak2 0.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (12) hide show

checksums.yaml +7 -0
data/.gitignore +19 -0
data/Gemfile +6 -0
data/README.md +83 -0
data/Rakefile +17 -0
data/VERSION +1 -0
data/browser.html +178 -0
data/fluent-plugin-riak.gemspec +25 -0
data/lib/fluent/plugin/out_chriak2.rb +80 -0
data/listkeys.rb +36 -0
data/setup_index.sh +11 -0
metadata +137 -0

checksums.yaml ADDED

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a9c2af38a4e19d22efa8d9d22c0cf7e272613b16
+  data.tar.gz: 765e4bdda704d21403f018dd3542a8f2b417f8b2
+SHA512:
+  metadata.gz: 0ee5c145ab9e91fd52f1e3b09e2a1842abe33764f3c9a4da1a401b75ba9c6761349e5cbd32a19c133c28685ed76951a7686f59d879f9ded05cc66d2f67d1d9be
+  data.tar.gz: 5a84647c9c6ae53bd2b95b03363f12aaeacf73a11088ec352552ba5b92e415cca354c95f1c0140b481c415e7d08a7b15d044ad45c5a220b7cc5718ac73a18cb7

data/.gitignore ADDED

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*~
+Gemfile.lock
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/Gemfile ADDED

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+gemspec
+gem "simplecov", :require => false
+gem "uuidtools", :require => ">= 2.1.3"

data/README.md ADDED

@@ -0,0 +1,83 @@
+fluent-plugin-riak, a plugin for [Fluentd](http://fluentd.org)
+==================
+fluent-plugin-riak is a alternative for people who are not sufficient with mongo or webhdfs. Riak ( http://github.com/basho/riak ) is an open-source distributed KVS focused on availability. It also has a strong query system with secondary index (2i): see docs ( http://docs.basho.com/riak/latest/tutorials/querying/ ) for details.
+Current status is still proof-of-concept: index setting and its configuration are to be decided. Also performance optimization is required. Another idea is in_tail_riak by using riak post-commit.
+installation
+------------
+```bash
+$ sudo gem install fluent-plugin-riak
+```
+Notice: you need Riak configured using eleveldb as backend.
+fluent.conf example
+-------------------
+```
+<match riak.**>
+  type riak
+  buffer_type memory
+  flush_interval 10s
+  retry_limit 5
+  retry_wait 1s
+  buffer_chunk_limit 256m
+  buffer_queue_limit 8096
+  # pb port
+  nodes 127.0.0.1:8087
+  #for cluster, define multiple machines
+  #nodes 192.168.100.128:10018 129.168.100.128:10028
+</match>
+```
+- key format -> 2013-02-<uuid>
+- value format -> [records] in JSON
+- index:
+ - year_int -> year
+ - month_bin -> <year>-<month>
+ - tag_bin -> tags
+easy querying log
+-----------------
+```bash
+$ curl -X PUT http://localhost:8098/buckets/static/keys/browser.html -H 'Content-type: text/html' -d @browser.html
+$ open http://localhost:8098/buckets/static/keys/browser.html
+```
+Pros
+----
+- easy operations
+- high availability
+- horizontal scalability (esp. write performance)
+- good night sleep
+Cons
+----
+- no capped table, TTL objects
+TODOs
+-----
+- refine browser.html query interface with cool features
+- rething index structures
+License
+=======
+Apache 2.0
+Copyright Kota UENISHI
+Many Thanks to fluent-plugin-mongo

data/Rakefile ADDED

@@ -0,0 +1,17 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/plugin/*.rb']
+  test.verbose = true
+end
+task :coverage do |t|
+  ENV['SIMPLE_COV'] = '1'
+  Rake::Task["test"].invoke
+end
+task :default => [:build]

data/VERSION ADDED

	@@ -0,0 +1 @@
1	+ 0.0.7

data/browser.html ADDED

@@ -0,0 +1,178 @@
+<html>
+<head>
+  <title>Riak mapreducer</title>
+  <style type="text/css">
+body {
+  background: #678; text-align: center; width: 100%; font-family: 'Titillium', Monospace;
+}
+body div {
+  width: 80%; background: #ddd;
+  padding: 20px; text-align: left; margin: 0px auto;
+}
+h2 {
+  padding: 0px;
+  margin: 0px auto;
+}
+a {
+  text-shadow: 1px 1px #888888;
+  text-decoration:underline;
+}
+  </style>
+  <script lang="JavaScript">
+    function get_map(){    return document.getElementById("map").value; }
+    function get_reduce(){ return document.getElementById("reduce").value; }
+    function get_bucketname(){ return document.getElementById("bucket").value; }
+    function exec_query(m, r){
+      var q = { inputs : get_bucketname(),
+      query : [
+        { map:    { language: "javascript", source: m}},
+        { reduce: { language: "javascript", source: r}}
+      ]};
+      var xmlhttp = new XMLHttpRequest();
+      xmlhttp.open("POST", "/mapred", false);
+      xmlhttp.setRequestHeader("Content-Type", "application/json");
+      xmlhttp.send(JSON.stringify(q, null, ""));
+      var result = xmlhttp.responseText;
+      var resultBox = document.getElementById("result");
+      resultBox.innerText = result;
+    }
+  </script>
+</head>
+<body>
+  <div id="content">
+    <div style="text-align:right">
+    <script lang="JavaScript">
+    </script>
+    bucket name:<input type="text" id="bucket" value="fluentlog"/>
+    </div>
+    <h2>Run mapreduce on fluentd log</h2>
+    <table style="border: 1px solid black;">
+      <tr>
+        <th>map</th> <th>reduce</th>
+      </tr>
+      <tr>
+        <td>
+<textarea name="map" id="map" rows="16" cols="50">
+function(v){
+  var data = v.values[0].data;
+  var c = JSON.parse(data);
+  return [c.count];
+}
+</textarea>
+          </td><td>
+<textarea name="reduce" id="reduce" rows="16" cols="50" >
+function(v){
+  var sum = 0;
+  for( var i=0; i < v.length; ++i){
+    sum += v[i];
+  }
+  return [sum];
+}
+</textarea>
+        </td>
+      </tr>
+      <tr>
+        <th colspan="2"> <a onClick="exec_query(get_map(), get_reduce());">query</a> </th>
+      </tr>
+      <tr>
+        <td width="200px" colspan="2">
+    <div id="result" style="font: bold;"></div>
+    </td>
+    </tr>
+      <tr><td colspan="3">
+          <script lang="JavaScript">
+            function grep(){
+            }
+          </script>
+          <script lang="JavaScript" type="text/javascript">
+            function count_rank(){
+            var m = "function(v){
+      var data = v.values[0].data;
+      var c = JSON.parse(data);
+      var tags = {};
+      for(var i=0; i<c.length; ++i){
+        count = tags[c[i].tag];
+        if(count == null){ tags[c[i].tag] = 1; }
+        else{              tags[c[i].tag] = count+1; }
+      }
+      return [tags];
+    }";
+      var r = "function(v){
+      var tags = {};
+      for(var i=0; i<v.length; ++i){
+        for (var prop in v[i]) {
+          count = tags[prop];
+          if(count == null){ tags[prop] = v[i][prop]; }
+          else{              tags[prop] = count+v[i][prop]; }
+        }
+      }
+      return [tags];
+    }";
+            exec_query(m, r);
+          }
+          </script>
+          <!-- <a onClick="grep();">grep</a> <input type="text" id="grep" value="debug"/> -->
+          <a onClick="count_rank();">count&amp;rank</a> <input type="text" id="count_rank" value="tag"/>
+      </td></tr>
+    </table>
+    <hr/>
+    <h2>Riak object viewer</h2>
+    <script lang="JavaScript">
+      function get(){
+      var key = document.getElementById("key").value;
+      var xmlhttp = new XMLHttpRequest();
+      xmlhttp.open("GET", "/buckets/"+ get_bucketname() +"/keys/"+key, false);
+      xmlhttp.send();
+      var result = xmlhttp.responseText;
+      console.log(result);
+      document.getElementById("object").innerText = result;
+      }
+    </script>
+    <a onClick="get();" >get</a> <input type="text" id="key" />
+    <div id="object"></div>
+    <h2>Riak keylister</h2>
+    <script lang="JavaScript">
+      function listkeys(){
+      var xmlhttp = new XMLHttpRequest();
+      xmlhttp.open("GET", "/buckets/"+ get_bucketname() +"/keys?keys=true", false);
+      xmlhttp.send();
+      var result = JSON.parse(xmlhttp.responseText);
+      console.log(result);
+      document.getElementById("keylist").innerText = result.keys.sort().join("\n");
+      }
+    </script>
+    <a onClick="listkeys();" >listkeys</a>
+    <div id="keylist"></div>
+    <hr/>
+    (C)@kuenishi distributed under Apache 2 license
+  </div>
+<!--
+    <h2>Riak object putter</h2>
+    <script lang="JavaScript">
+      function put(){
+      var key = document.getElementById("key2").value;
+      var value = document.getElementById("value").value;
+      var xmlhttp = new XMLHttpRequest();
+      xmlhttp.open("PUT", "/buckets/fluentlog/keys/"+key, false);
+      xmlhttp.setRequestHeader("Content-Type", "application/json");
+      xmlhttp.send(value);
+      var result = xmlhttp.responseText;
+      console.log(result);
+      document.getElementById("object2").innerText = result;
+      }
+    </script>
+     <a onClick="put();" >put</a> <input type="text" id="key2" /> <input type="text" id="value" />
+    <div id="object2"></div>
+-->
+</body>
+</html>

data/fluent-plugin-riak.gemspec ADDED

@@ -0,0 +1,25 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-chriak2"
+  gem.description = "Riak 2 plugin for Fluent event collector"
+  gem.homepage    = "https://github.com/collectivehealth/fluent-plugin-riak"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Kota UENISHI", "Matt Nunogawa"]
+  gem.email       = "matt@collectivehealth.com"
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+  gem.add_dependency "fluentd", "~> 0.10"
+  gem.add_dependency "riak-client", "~> 2.1.0"
+  gem.add_dependency "uuidtools", ">= 2.1.3"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "simplecov", ">= 0.5.4"
+  gem.add_development_dependency "rr", ">= 1.0.0"
+end

data/lib/fluent/plugin/out_chriak2.rb ADDED

@@ -0,0 +1,80 @@
+module Fluent
+class Chriak2Output < BufferedOutput
+  Fluent::Plugin.register_output('chriak2', self)
+  include SetTimeKeyMixin
+  config_set_default :include_tag_key, true
+  include SetTagKeyMixin
+  config_set_default :include_time_key, true
+  config_param :bucket_name, :string, :default => "fluentlog"
+  config_param :nodes, :string, :default => "localhost:8087"
+  def initialize
+    super
+    require 'riak'
+    require 'msgpack'
+    require 'uuidtools'
+  end
+  def configure(conf)
+    super
+    @nodes = @nodes.split(',').map{ |s|
+      ip,port = s.split(':')
+      {:host => ip, :pb_port => port.to_i}
+    }
+    $log.info "riak nodes=#{@nodes}"
+  end
+  def start
+    $log.debug " => #{@buffer.chunk_limit} #{@buffer.queue_limit} "
+    @conn = Riak::Client.new(:nodes => @nodes, :protocol => "pbc")
+    @bucket = @conn.bucket(@bucket_name)
+    @buf = {}
+    super
+  end
+  def format(tag, time, record)
+    [time, tag, record].to_msgpack
+  end
+  def write(chunk)
+    $log.debug " <<<<<===========\n"
+    records  = []
+    tags = []
+    chunk.msgpack_each { |time, tag, record|
+      record[@tag_key] = tag
+      tags << tag
+      records << record
+      $log.debug record
+    }
+    put_now(records, tags)
+  end
+  private
+  # TODO: add index for some analysis
+  def put_now(records, tags)
+    if not records.empty? then
+      today = Date.today
+      key = "#{today.to_s}-#{UUIDTools::UUID.timestamp_create.to_s}"
+      robj = Riak::RObject.new(@bucket, key)
+      robj.raw_data = records.to_json
+      robj.indexes['year_int'] << today.year
+      robj.indexes['month_bin'] << "#{today.year}-#{"%02d" % today.month}"
+      tags.each do |tag|
+        robj.indexes['tag_bin'] << tag
+      end
+      robj.content_type = 'application/json'
+      robj.store
+      robj
+    end
+  end
+end
+end

data/listkeys.rb ADDED

@@ -0,0 +1,36 @@
+require 'riak'
+nodes = [{ :host => "127.0.0.1",
+           :pb_port => 8087 }]
+c = Riak::Client.new( :nodes => nodes, :protocol => "pbc" )
+#c.list_buckets
+b = c.bucket("fluentlog")
+def put(bucket)
+  o = Riak::RObject.new(bucket, "hoge")
+  o.content_type = "text/plain"
+  o.raw_data = "hogehoge"
+  o.store
+end
+def get(bucket)
+  o = bucket.get("hoge")
+  p o.content_type
+  p o.raw_data
+end
+def keys(bucket)
+  bucket.keys { |ks|
+    ks.each { |k|
+      o = bucket.get(k)
+      print "#{o.bucket.name}/#{o.key} (#{o.content_type})\n"
+      print " =>  #{o.raw_data}\n"
+    }
+  }
+end
+#put b
+#get b
+keys b

data/setup_index.sh ADDED

@@ -0,0 +1,11 @@
+#!/bin/sh
+HOST=$1
+echo setting up fluentlog_index at $HOST
+curl -X PUT http://${HOST}/yz/index/fluentlog_index
+echo "setting fluentlog_index to fluentlog bucket"
+curl -X PUT http://${HOST}/buckets/fluentlog/props -H 'Content-type:application/json' -d '{"props":{"yz_index":"fluentlog_index"}}'
+curl http://${HOST}/yz/index?index=true
+curl http://${HOST}/buckets/fluentlog/props

metadata ADDED

@@ -0,0 +1,137 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-chriak2
+version: !ruby/object:Gem::Version
+  version: 0.0.7
+platform: ruby
+authors:
+- Kota UENISHI
+- Matt Nunogawa
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2014-11-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: riak-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: uuidtools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.1.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.1.3
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+- !ruby/object:Gem::Dependency
+  name: rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+description: Riak 2 plugin for Fluent event collector
+email: matt@collectivehealth.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- VERSION
+- browser.html
+- fluent-plugin-riak.gemspec
+- lib/fluent/plugin/out_chriak2.rb
+- listkeys.rb
+- setup_index.sh
+homepage: https://github.com/collectivehealth/fluent-plugin-riak
+licenses: []
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.0.14
+signing_key:
+specification_version: 4
+summary: Riak 2 plugin for Fluent event collector
+test_files: []