fluent-plugin-hbase 0.1.0

data/.gitignore

@@ -0,0 +1,5 @@
+.idea
+Gemfile.lock
+*~
+*.iml
+*.gem

data/AUTHORS

@@ -0,0 +1 @@
+KUOKA Yusuke <kuoka _at_ furyu.jp>

data/ChangeLog

@@ -0,0 +1,6 @@
+
+
+Release 0.1.0 - 2012/11/19
+
+  * First release
+

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/README.rdoc

@@ -0,0 +1,97 @@
+= HBase output plugin for Fluent event collector
+
+== Overview
+
+*HBase* output plugin buffers event logs in local file and puts it to HBase periodically.
+
+== Installation
+
+Simply use RubyGems:
+
+    gem install fluent-plugin-hbase
+
+== Configuration
+
+    <match pattern>
+      type hbase
+
+      tag_column_name HBASE_COLUMN
+      time_column_name HBASE_COLUMN
+      fields_to_columns_mapping MAPPING_FROM_JSON_FIELDS_TO_HBASE_COLUMNS
+      hbase_host YOUR_HBASE_HOST
+      hbase_port YOUR_HBASE_PORT
+      hbase_table YOUR_HBASE_TABLE_NAME
+    </match>
+
+[tag_column_name (required)] The HBase column to save the tag attached to each Fleuntd event log,
+in the format "[Column family name]:[Column name]".
+For example, to save tags to the column "c" in the column family "cf", use "cf:c".
+
+[time_column_name (required)] The HBase column to save the time each Fluentd event log was sent,
+in the format "[Column family name]:[Colum name]".
+For example, to save the time to the column "t" in the column family "cf", use "cf:t".
+
+[fields_to_columns_mapping (required)] The mapping from JSON fields to HBase columns,
+in the format "[JSON_FIELD1]=>[HBASE_COLUMN1],[JSON_FIELD2]=>[HBASE_COLUMN2],...".
+
+Each JSON_FIELD is formatted as field names separated by dot(.)s, e.g. "a.b.c".
+Each HBASE_COLUMN is formatted as a tripled of a column family name, a colon(:), a column name, e.g. "cf:c".
+
+[hbase_host (required)] HBase host
+
+[hbase_port (required)] HBase port
+
+[hbase_table (required)] HBase table name
+
+See example/fluent.conf for the configuration should work.
+
+You can also test the configuration running a fluentd instance:
+
+ fluentd -c example/fluent.conf --plugin lib/fluent/plugin
+
+== Prerequiresites
+
+You must setup your own Hadoop and HBase clusters and open appropriate ports to enable
+the plugin to access HBase via HBase Thrift Server.
+
+The plugin is tested solely on the system with:
+
+- Hadoop 1.0.4
+- HBase 0.94.0
+- Java 1.6.0_37
+- Mac OS X 10.8 Mountain Lion
+
+for now.
+
+Please let me know if you find the plugin to work in any other environments.
+
+== Running
+
+To make the plugin work, you need running instances of:
+
+- Hadoop
+- HBase
+- HBase Thrift Server w/ the compact (buffered) protocol (not the framed protocol)
+
+The procedure may be:
+
+1. Start Hadoop
+
+ $ start-all.sh
+
+2. Start HBase
+
+ $ start-hbase.sh
+
+3. Start HBase Thrift Server with the compact protocol
+
+Use the thread pool server as it is the only server supports the compact protocol:
+
+ $ hbase thrift start -threadpool
+
+4. Run Fluentd
+
+== Copyright
+
+Copyright:: Copyright (c) 2012 FURYU CORPORATION
+License::   Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,14 @@
+
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]
+

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/example/fluent.conf

@@ -0,0 +1,97 @@
+
+## built-in TCP input
+## $ echo <json> | fluent-cat <tag>
+<source>
+  type forward
+</source>
+
+## built-in UNIX socket input
+#<source>
+#  type unix
+#</source>
+
+# HTTP input
+# http://localhost:8888/<tag>?json=<json>
+<source>
+  type http
+  port 8888
+</source>
+
+## File input
+## read apache logs with tag=apache.access
+#<source>
+#  type tail
+#  format apache
+#  path /var/log/httpd-access.log
+#  tag apache.access
+#</source>
+
+# Listen DRb for debug
+<source>
+  type debug_agent
+  port 24230
+</source>
+
+
+## match tag=apache.access and write to file
+#<match apache.access>
+#  type file
+#  path /var/log/fluent/access
+#</match>
+
+## match tag=debug.** and dump to console
+<match debug.**>
+  type stdout
+</match>
+
+## match tag=system.** and forward to another fluent server
+#<match system.**>
+#  type forward
+#  host 192.168.0.11
+#  <secondary>
+#    host 192.168.0.12
+#  </secondary>
+#</match>
+
+## match tag=myapp.** and forward and write to file
+#<match myapp.**>
+#  type copy
+#  <store>
+#    type forward
+#    host 192.168.0.13
+#    buffer_type file
+#    buffer_path /var/log/fluent/myapp-forward
+#    retry_limit 50
+#    flush_interval 10s
+#  </store>
+#  <store>
+#    type file
+#    path /var/log/fluent/myapp
+#  </store>
+#</match>
+
+## match fluent's internal events
+#<match fluent.**>
+#  type null
+#</match>
+
+## match not matched logs and write to file
+#<match **>
+#  type file
+#  path /var/log/fluent/else
+#  compress gz
+#</match>
+
+<match **>
+  type hbase
+
+  # Just to write to HBase rapidly
+  flush_interval 1
+
+  tag_column_name event:tag
+  time_column_name event:time
+  fields_to_columns_mapping foo=>event:foo,iam.nested=>event:nested
+  hbase_host localhost
+  hbase_port 9090
+  hbase_table events
+</match>

data/example/run-fluentd.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+fluentd -c example/fluent.conf --plugin lib/fluent/plugin

data/fluent-plugin-hbase.gemspec

@@ -0,0 +1,22 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-hbase"
+  gem.description = "HBase output plugin for Fluent event collector"
+  gem.homepage    = "https://github.com/Furyu/fluent-plugin-hbase"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["KUOKA Yusuke"]
+  gem.email       = "kuoka@furyu.jp"
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", "~> 0.10.0"
+  gem.add_dependency "massive_record", "~> 0.2.2"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+end

data/lib/fluent/plugin/out_hbase.rb

@@ -0,0 +1,99 @@
+module Fluent
+
+  class HBaseOutput < Fluent::BufferedOutput
+    Fluent::Plugin.register_output('hbase', self)
+
+    def initialize
+      super
+      require 'massive_record'
+    end
+
+    config_param :tag_column_name, :string, :default => nil
+    config_param :time_column_name, :string, :default => nil
+    config_param :fields_to_columns_mapping, :string
+    config_param :hbase_host, :string, :default => 'localhost'
+    config_param :hbase_port, :integer, :default => 9090
+    config_param :hbase_table, :string
+
+    def configure(conf)
+      super
+
+      @fields_to_columns = @fields_to_columns_mapping.split(",").map { |src_to_dst|
+        src_to_dst.split("=>")
+      }
+      @mapping = Hash[*@fields_to_columns.flatten]
+    end
+
+    def start
+      super
+
+      @conn = MassiveRecord::Wrapper::Connection.new(:host => @hbase_host, :port => @hbase_port)
+      @table = MassiveRecord::Wrapper::Table.new(@conn, @hbase_table.intern)
+
+      unless @table.exists?
+        columns = [@tag_column_name, @time_column_name] + @mapping.values
+        column_families = columns.map {|column_family_with_column|
+          column_family, column = column_family_with_column.split(":")
+
+          if column.nil?
+            raise <<MESSAGE
+Unexpected format for column name: #{column_family_with_column}
+Each destination column in the 'record_to_columns_mapping' option
+must be specified in the format of \"column_family:column\".
+Are you sure you included ':' in column names?
+MESSAGE
+          end
+
+          column_family.intern
+        }
+
+        @table.create_column_families(column_families)
+        @table.save
+      end
+    end
+
+    def format(tag, time, record)
+      row_values = {}
+
+      row_values[@tag_column_name] = tag unless @tag_column_name.nil?
+      row_values[@time_column_name] = time unless @time_column_name.nil?
+
+      @fields_to_columns.each {|field,column|
+
+        next if field.nil? or column.nil?
+
+        components = field.split(".")
+        value = record
+        for c in components
+          value = value[c]
+
+          break if value.nil?
+        end
+
+        row_values[column] = value
+      }
+
+      row_values.to_msgpack
+    end
+
+    def write(chunk)
+      chunk.msgpack_each {|row_values|
+        event = {}
+
+        row_values.each {|column_family_and_column, value|
+          column_family, column = column_family_and_column.split(":")
+
+          (event[column_family.intern] ||= {}).update({column => value})
+        }
+
+        row = MassiveRecord::Wrapper::Row.new
+        row.id = SecureRandom.uuid
+        row.values = event
+        row.table = @table
+        row.save
+      }
+    end
+
+  end
+
+end

data/test/out_hbase.rb

@@ -0,0 +1,134 @@
+require 'fluent/test'
+require 'fluent/plugin/out_hbase'
+
+class HBaseOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    tag_column_name event:tag
+    time_column_name event:time
+    fields_to_columns_mapping foo=>event:foo,iam.nested=>event:nested
+    hbase_host localhost
+    hbase_port 9090
+    hbase_table events
+    buffer_type memory
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::HBaseOutput) do
+      # We don't want to connect the HBase instance while testing
+      def start
+        super
+      end
+
+      # prevents writes to the HBase instance while testing
+      def write(chunk)
+        chunk.read
+      end
+    end.configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal 'event:tag', d.instance.tag_column_name
+    assert_equal 'event:time', d.instance.time_column_name
+    assert_equal 'foo=>event:foo,iam.nested=>event:nested', d.instance.fields_to_columns_mapping
+    assert_equal 'localhost', d.instance.hbase_host
+    assert_equal 9090, d.instance.hbase_port
+    assert_equal 'events', d.instance.hbase_table
+  end
+
+  def test_format
+    d = create_driver
+
+    time_in_int = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.emit(
+        {
+            "foo" => "foo1",
+            "iam" => {
+                "nested" => "nested1"
+            }
+        },
+        time_in_int
+    )
+
+    d.emit(
+        {
+            "foo" => "foo2",
+            "iam" => {
+                "nested" => "nested2"
+            }
+        },
+        time_in_int
+    )
+
+    expected1 = {
+        "event:tag" => "test",
+        "event:time" => time_in_int,
+        "event:foo" => "foo1",
+        "event:nested" => "nested1"
+    }.to_msgpack
+
+    expected2 = {
+        "event:tag" => "test",
+        "event:time" => time_in_int,
+        "event:foo" => "foo2",
+        "event:nested" => "nested2"
+    }.to_msgpack
+
+    d.expect_format expected1
+    d.expect_format expected2
+
+    d.run
+  end
+
+  def test_write
+    d = create_driver
+
+    time_in_int = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.emit(
+        {
+            "foo" => "foo1",
+            "iam" => {
+                "nested" => "nested1"
+            }
+        },
+        time_in_int
+    )
+
+    d.emit(
+        {
+            "foo" => "foo2",
+            "iam" => {
+                "nested" => "nested2"
+            }
+        },
+        time_in_int
+    )
+
+    expected1 = {
+        "event:tag" => "test",
+        "event:time" => time_in_int,
+        "event:foo" => "foo1",
+        "event:nested" => "nested1"
+    }.to_msgpack
+
+    expected2 = {
+        "event:tag" => "test",
+        "event:time" => time_in_int,
+        "event:foo" => "foo2",
+        "event:nested" => "nested2"
+    }.to_msgpack
+
+    # HBaseOutputTest#write returns chunk.read
+    data = d.run
+
+    assert_equal expected1 + expected2, data
+  end
+
+end
+

metadata

@@ -0,0 +1,111 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-hbase
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- KUOKA Yusuke
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-11-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+- !ruby/object:Gem::Dependency
+  name: massive_record
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: HBase output plugin for Fluent event collector
+email: kuoka@furyu.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- AUTHORS
+- ChangeLog
+- Gemfile
+- README.rdoc
+- Rakefile
+- VERSION
+- example/fluent.conf
+- example/run-fluentd.sh
+- fluent-plugin-hbase.gemspec
+- lib/fluent/plugin/out_hbase.rb
+- test/out_hbase.rb
+homepage: https://github.com/Furyu/fluent-plugin-hbase
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 1566921651398845132
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 1566921651398845132
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: HBase output plugin for Fluent event collector
+test_files:
+- test/out_hbase.rb