fluent-plugin-bigobject_avro 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 326049b14af5b95c7d140d443e3d1f3a965612d3
+  data.tar.gz: be90f9fe1309aba2ea90ccd10071cb265081ac7d
+SHA512:
+  metadata.gz: 5be4be8a7093e6d91fd17e2e95d1551530c2a801c2d9e7231a660136b399f3d1a4c8d694511500b8da2e4cb5a6414af3eae7b4ce990ec89d15c4ea317e621533
+  data.tar.gz: 19145ab4285e976a6abbf12a9a7c2abc588bd1d61bb2ea91f52590135c22de451caa71590cef08bc48df9933ce3ca62ecb40f82cb1811ebc1312da4567da85ed

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-bigobject.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 BigObject 
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,54 @@
+# fluent-plugin-bigobject
+
+Fluentd output plugin for inserting data to BigObject using AVRO
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fluent-plugin-bigobject_avro'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-bigobject_avro
+
+## Usage
+
+Configure BigObject URL and the table/column to be mapped in BigObject
+
+```apache
+
+# send data to BigObject using avro by providing schema_file in each table
+# BigObject will create the table if it does not already exist.
+
+<match bo.insert_avro.*>
+  type bigobject
+
+  log_level info
+
+  # specify the bigobject host/port to connect to
+  bigobject_hostname 192.168.59.103
+  bigobject_port 9091
+
+  remove_tag_prefix bo.insert_avro.
+  flush_interval 60s
+
+  <table>
+      pattern customer
+      schema_file /fluentd/input/avsc/Customer_binary.avsc
+  </table>
+</match>
+
+```
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/fluent-plugin-bigobject_avro.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-bigobject_avro"
+  gem.version       = "0.0.1"
+  gem.authors       = ["Andrea Sung"]
+  gem.email         = ["andrea@bigobject.io"]
+  gem.description   = %q{Fluentd output plugin to insert BIGOBJECT }
+  gem.summary       = %q{Fluentd output plugin to insert BIGOBJECT}
+  gem.homepage      = "https://github.com/bigobject-inc/fluent-plugin-bigobject_avro"
+  gem.license       = "MIT"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_runtime_dependency "fluentd"
+  gem.add_runtime_dependency "json"
+  gem.add_runtime_dependency "avro"
+  gem.add_development_dependency "rake"
+end

data/lib/fluent/plugin/out_bigobject_avro.rb

@@ -0,0 +1,176 @@
+class Fluent::BigObjectOutput_AVRO < Fluent::BufferedOutput
+
+  Fluent::Plugin.register_output('bigobject_avro', self)
+  
+  include Fluent::SetTimeKeyMixin
+  include Fluent::SetTagKeyMixin
+
+  config_param :bigobject_hostname, :string
+  config_param :bigobject_port, :integer
+  config_param :remove_tag_prefix, :string, :default => nil
+  config_param :send_unknown_chunks, :string,  :default=>true
+
+  attr_accessor :tables
+  
+  unless method_defined?(:log)
+    define_method(:log) { $log }
+  end
+  
+  class TableElement
+    include Fluent::Configurable
+
+    config_param :column_mapping, :string, :default=>nil
+    config_param :pattern, :string, :default=>nil
+    config_param :schema_file, :string
+
+    attr_reader :mpattern
+
+    def initialize(log, bo_hostname, bo_port)
+      super()
+      @log = log
+      @bo_hostname = bo_hostname
+      @bo_port = bo_port
+      @bo_url="http://#{@bo_hostname}:#{@bo_port}/cmd"
+    end
+
+    def configure(conf)
+      super
+
+      @avro_schema = Avro::Schema.parse(File.open(@schema_file, "rb").read)
+      @avro_writer = Avro::IO::DatumWriter.new(@avro_schema)
+
+      @mpattern = Fluent::MatchPattern.create(pattern)
+      @mapping = (@column_mapping==nil)? nil:parse_column_mapping(@column_mapping)
+      @log.info("column mapping for #{pattern} - #{@mapping}")
+      @format_proc = Proc.new { |record|
+        if (@mapping==nil)
+          record
+        else
+          new_record = {}
+          @mapping.each { |k, c|
+            new_record[c] = record[k]
+            }
+          new_record
+        end
+      }
+    end
+    
+    #Send data to Bigobject using binary AVRO
+    def send_binary(chunk)
+      
+      buffer = StringIO.new()      
+      dw = Avro::DataFile::Writer.new(buffer, @avro_writer, @avro_schema)
+      i=0
+      chunk.msgpack_each { |tag, time, data|
+         data = @format_proc.call(data)
+         dw<<data
+         i+=1
+      }
+      dw.flush
+
+      begin
+        socket = TCPSocket.open(@bo_hostname, @bo_port)
+        begin
+          #timeout=60
+          opt = [1, 60].pack('I!I!')  # { int l_onoff; int l_linger; }
+          socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
+  
+          opt = [60, 0].pack('L!L!')  # struct timeval
+          socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
+          socket.write(buffer.string)
+        ensure
+          socket.close
+        end
+        
+      rescue Exception => e 
+          @log.error(e.message)  
+          raise "Failed to send_binary: #{e.message}"
+      end
+      @log.debug("sending #{i} rows to bigobject via avro")
+    end
+    
+    def to_s
+      "pattern:#{pattern}, column_mapping:#{column_mapping}"
+    end
+
+    private
+    def parse_column_mapping(column_mapping_conf)
+      mapping = {}
+      column_mapping_conf.split(',').each { |column_map|
+        key, column = column_map.strip.split(':', 2)
+        column = key if column.nil?
+        mapping[key] = column
+      }
+      mapping
+    end
+    
+  end #end class
+  
+  def initialize
+    super
+    require 'avro'
+    log.info("bigobject_avro initialize")
+  end  
+  
+  def configure(conf)
+    super
+    
+    if remove_tag_prefix = conf['remove_tag_prefix']
+      @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
+    end
+
+    @tables = []
+    @default_table = nil
+ 
+    conf.elements.select { |e|
+      e.name == 'table'
+    }.each { |e|
+      te = TableElement.new(log, @bigobject_hostname, @bigobject_port)
+      te.configure(e)
+      @tables << te
+    }
+    
+#    @tables.each {|t| puts t.to_s}
+  end
+  
+  def start
+    super
+    log.info("bigobject_avro start")
+  end
+  
+  def shutdown
+    super
+  end 
+
+  # This method is called when an event reaches to Fluentd.
+  def format(tag, time, record)
+    [tag, time, record].to_msgpack
+  end
+  
+  # This method is called every flush interval. Write the buffer chunk
+  # to files or databases here. 
+  # 'chunk' is a buffer chunk that includes multiple formatted events. 
+  def write(chunk)
+    unknownChunks = []
+    @tables.each { |table|
+      if table.mpattern.match(chunk.key)
+        return table.send_binary(chunk)
+      end
+    }
+    
+    log.warn("unknown chunk #{chunk.key}")
+      
+  end
+  
+  def format_tag(tag)
+    if @remove_tag_prefix
+      tag.gsub(@remove_tag_prefix, '')
+    else
+      tag
+    end
+  end
+  
+  def emit(tag, es, chain)
+    super(tag, es, chain, format_tag(tag))
+  end
+end

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-bigobject_avro
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Andrea Sung
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-11-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: avro
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 'Fluentd output plugin to insert BIGOBJECT '
+email:
+- andrea@bigobject.io
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-bigobject_avro.gemspec
+- lib/fluent/plugin/out_bigobject_avro.rb
+homepage: https://github.com/bigobject-inc/fluent-plugin-bigobject_avro
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Fluentd output plugin to insert BIGOBJECT
+test_files: []