fluent-plugin-ilm_hbase 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d56ffa36dcbb7c9502a25a7ecc176c6c57ae29f4
+  data.tar.gz: f4db2bbfbe8506080b5c0589795172f7ce09a18f
+SHA512:
+  metadata.gz: 50f3c3b36fafdf998b552f586edb167c16e59bd1dd85a40fcd89f15f1e34283d31ec3b41f06aeac64e1188653aeffa34c720060624f96eb7771dad5831454132
+  data.tar.gz: 81d10b73c64ea7c249e097ea28659adb9d1e8d8dc028e44aaf829d72c3bb003ef274b4b83c0f7c01e35a8214b0a7e03cd3253acd7d22ad2daaca8c3599f5f492

data/lib/fluent/plugin/out-ilm_hbase.rb

@@ -0,0 +1,127 @@
+require "fluent/plugin/ilm_hbase/version"
+
+module Fluent
+  module Plugin
+    class IlmHbaseOutput < Fluent::BufferedOutput
+    Fluent::Plugin.register_output('ilm_hbase', self)
+
+    def initialize
+      super
+      require 'massive_record'
+    end
+
+    # Format dates with ISO 8606 by default
+    #     # http://www.w3.org/TR/NOTE-datetime
+    config_param :time_format, :string, :default => '%Y-%m-%dT%H:%M:%S.%L%:z'
+
+    include SetTagKeyMixin
+    config_set_default :include_tag_key, false
+    config_set_default :tag_key, nil
+
+    include SetTimeKeyMixin
+    config_set_default :include_time_key, false
+    config_set_default :time_key, nil
+
+
+    config_param :include_custom_timestamp, :bool, :default=>false
+    config_param :custom_time_stamp_key, :string, :default => nil
+    config_param :tag_column_name, :string, :default => nil
+    config_param :time_column_name, :string, :default => nil
+    config_param :fields_to_columns_mapping, :string
+    config_param :hbase_host, :string, :default => 'localhost'
+    config_param :hbase_port, :integer, :default => 9090
+    config_param :hbase_table, :string
+
+    def configure(conf)
+      super
+
+      @fields_to_columns = @fields_to_columns_mapping.split(",").map { |src_to_dst|
+        src_to_dst.split("=>")
+      }
+      @mapping = Hash[*@fields_to_columns.flatten]
+    end
+
+    def start
+      super
+
+      @conn = MassiveRecord::Wrapper::Connection.new(:host => @hbase_host, :port => @hbase_port)
+      @table = MassiveRecord::Wrapper::Table.new(@conn, @hbase_table.intern)
+
+      unless @table.exists?
+        columns = ([@tag_column_name, @time_column_name] + @mapping.values).reject(&:nil?)
+        column_families = columns.map {|column_family_with_column|
+          column_family, column = column_family_with_column.split(":")
+
+          if column.nil? or column_family.nil?
+            raise <<MESSAGE
+Unexpected format for column name: #{column_family_with_column}
+Each destination column in the 'record_to_columns_mapping' option
+must be specified in the format of \"column_family:column\".
+Are you sure you included ':' in column names?
+MESSAGE
+          end
+
+          column_family.intern
+        }
+        column_families.uniq!
+
+        @table.create_column_families(column_families)
+        @table.save
+      end
+    end
+
+    def format(tag, time, record)
+      row_values = {}
+
+      row_values[@tag_column_name] = tag unless @tag_column_name.nil?
+      row_values[@time_column_name] = time unless @time_column_name.nil?
+
+      @fields_to_columns.each {|field,column|
+
+        next if field.nil? or column.nil?
+
+        components = field.split(".")
+        value = record
+        for c in components
+          value = value[c]
+
+          break if value.nil?
+        end
+
+        row_values[column] = value
+      }
+
+      row_values.to_msgpack
+    end
+
+    def write(chunk)
+        customtime = 0
+        chunk.msgpack_each{|row_values|
+        row_values.each {|column_family_and_column, value|
+          if column_family_and_column == @custom_time_stamp_key
+                customtime = value.to_i
+          end
+
+        }
+
+        }
+
+      chunk.msgpack_each {|row_values|
+        event = {}
+
+        row_values.each {|column_family_and_column, value|
+          column_family, column = column_family_and_column.split(":")
+
+          (event[column_family.intern] ||= {}).update({column => value})
+        }
+
+        row = MassiveRecord::Wrapper::Row.new
+        row.id = customtime
+        row.values = event
+        row.table = @table
+        row.save
+      }
+    end
+    end
+  end
+end

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-ilm_hbase
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- testman
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-06-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: '"test gem"'
+email:
+- test@rubygems.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/fluent/plugin/out-ilm_hbase.rb
+homepage: http://rubygems.org/gems/fluent-plugin-ilm_hbase
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: '"modified hbase output plugin"'
+test_files: []