fluent-plugin-bigobject 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0f32d55fb8208a02537e14888097a8ee2e6db740
+  data.tar.gz: d4b41b61f8886371fb10b5fb12b32a2df0ed9508
+SHA512:
+  metadata.gz: 8562552cdf32bf6b3bfd166c1812bab657b2915763560bb7d054b7651d7dd437496f802480e2b6391956ef998d9d4a4fbefcdc50260d47ec0975eeee66e424d7
+  data.tar.gz: 5c5d5413c9847d2c5016b7b00cf417e106e5a10cc61a6726ebaf0172ba1dc9e3504e20fb23c3a7345202e1a69a0d238ca10d94e1fa2e12a5dd8facc73b133087

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-bigobject.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 MacroData 
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,59 @@
+# fluent-plugin-bigobject
+
+Fluentd output plugin for inserting data to BigObject
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fluent-plugin-bigobject'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-bigobject
+
+## Usage
+
+Configure BigObject URL and the table/column to be mapped in BigObject
+
+```apache
+<match bo.insert.*>
+  type bigobject
+
+  log_level info
+
+  # specify the bigobject_url to connect to
+  bigobject_url http://192.168.59.103:9090/cmd
+
+  remove_tag_prefix bo.insert. 
+  flush_interval 5s
+
+  <table>
+      table Customer
+      column_mapping id,name,language,state,company,gender,age
+      pattern customer
+      #bo_workspace
+      #bo_opts
+  </table>
+
+  <table>
+    table search_test3
+    # map to different column name in BigObject
+    column_mapping 'ID:id3, TEXT:text3'
+    pattern test3
+  </table>
+
+</match>
+```
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/fluent-plugin-bigobject.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-bigobject"
+  gem.version       = "0.0.3"
+  gem.authors       = ["Andrea Sung"]
+  gem.email         = ["andrea@bigobject.io"]
+  gem.description   = %q{Fluentd output plugin to insert BIGOBJECT }
+  gem.summary       = %q{Fluentd output plugin to insert BIGOBJECT}
+  gem.homepage      = "https://github.com/andreasung/fluent-plugin-bigobject"
+  gem.license       = "MIT"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_runtime_dependency "fluentd"
+  gem.add_runtime_dependency "rest-client"
+  gem.add_runtime_dependency "json"
+  gem.add_development_dependency "rake"
+end

data/lib/fluent/plugin/out_bigobject.rb

@@ -0,0 +1,199 @@
+class Fluent::BigObjectOutput < Fluent::BufferedOutput
+  Fluent::Plugin.register_output('bigobject', self)
+  
+  include Fluent::SetTimeKeyMixin
+  include Fluent::SetTagKeyMixin
+
+  config_param :bigobject_url, :string
+  config_param :remove_tag_prefix, :string, :default => nil
+  config_param :send_unknown_chunks, :string,  :default=>true
+
+  attr_accessor :tables
+  
+  unless method_defined?(:log)
+    define_method(:log) { $log }
+  end
+  
+  class TableElement
+    include Fluent::Configurable
+
+    config_param :table, :string
+    config_param :column_mapping, :string
+    config_param :pattern, :string, :default=>nil
+    config_param :bo_workspace, :string, :default=>nil
+    config_param :bo_opts, :string, :default=>nil
+
+    attr_reader :mpattern
+
+    def initialize(log)
+      super()
+      @log = log
+    end
+
+    def configure(conf)
+      super
+      @mpattern = Fluent::MatchPattern.create(pattern)
+      @mapping = parse_column_mapping(@column_mapping)
+      @log.info("column mapping for #{table} - #{@mapping}")
+      @format_proc = Proc.new { |record|
+        new_record = {}
+        @mapping.each { |k, c|
+          new_record[c] = record[k]
+        }
+        new_record
+      }
+    end
+
+    def send(bourl, chunk)
+      stmts = Array.new
+      i=0
+      columns = nil
+      chunk.msgpack_each { |tag, time, data|
+         keys = Array.new
+         values = Array.new
+         data = @format_proc.call(data)
+         data.keys.sort.each do |key|
+           keys << key
+           values << data[key]
+         end
+         if columns.to_s.empty?
+           columns = "(#{keys.join(",")})"
+         end
+         #single quote each column data
+         stmts.push("('#{values.join("','")}')")
+         i+=1
+      }
+      
+      sendStmt = "INSERT INTO #{@table}  #{columns} VALUES" + stmts.join(",")
+#      @log.debug("sendStmt=", sendStmt)
+      @log.info("bigobject start insert #{i} rows")
+      resp = sendBO(bourl, sendStmt)
+      parsed = JSON.parse(resp)
+      err = parsed['Err']
+#      puts "Content=#{parsed['Content']}, Err=#{err}"
+      if (err.to_s!='')
+        @log.error("[BigObject] #{err}")
+      end
+      @log.info("bigobject end insert #{i} rows")
+      
+    end
+    
+    def to_s
+      "table:#{table}, column_mapping:#{column_mapping}, pattern:#{pattern}"
+    end
+
+    private
+    def parse_column_mapping(column_mapping_conf)
+      mapping = {}
+      column_mapping_conf.split(',').each { |column_map|
+        key, column = column_map.strip.split(':', 2)
+        column = key if column.nil?
+        mapping[key] = column
+      }
+      mapping
+    end
+    
+    def formatRequest(stmt)
+      params = Hash.new
+      params['Stmt'] = stmt
+
+      if @bo_workspace.to_s!=''
+        params['Workspace'] = @bo_workspace
+      end
+      if @bo_opts.to_s!=''
+        params['Opts'] = @bo_opts
+      end
+      return params
+    end
+    
+    def sendBO(bourl, sendStmt)
+      params = formatRequest(sendStmt)
+      @log.debug("\nbourl=#{bourl} params=#{params.to_json}")
+    
+      begin
+        resp = RestClient.post bourl, params.to_json, :content_type =>:json, :accept =>:json
+        @log.debug("resp= #{resp.body}")  
+      rescue Exception => e 
+        @log.error(e.message)  
+        raise "Failed to sendBO: #{e.message}"
+      end
+      return resp
+    end
+   
+  end #end class
+  
+  def initialize
+    super
+    require 'rest-client'
+    require 'json'
+    log.info("bigobject initialize")
+  end  
+  
+  def configure(conf)
+    super
+    
+    if remove_tag_prefix = conf['remove_tag_prefix']
+      @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
+    end
+
+    @tables = []
+    @default_table = nil
+    conf.elements.select { |e|
+      e.name == 'table'
+    }.each { |e|
+      te = TableElement.new(log)
+      te.configure(e)
+#      puts "conf.elements #{e}"
+      @tables << te
+    }
+    
+#    @tables.each {|t| puts t.to_s}
+  end
+  
+  def start
+    super
+    log.info("bigobject start")
+  end
+  
+  def shutdown
+    super
+    log.info("bigobject shutdown")
+  end 
+  
+  
+  # This method is called when an event reaches to Fluentd.
+  def format(tag, time, record)
+#    puts "tag=#{tag}, time=#{time}, record=#{record}"
+    [tag, time, record].to_msgpack
+  end
+  
+  # This method is called every flush interval. Write the buffer chunk
+  # to files or databases here. 
+  # 'chunk' is a buffer chunk that includes multiple formatted events. 
+  def write(chunk)
+    unknownChunks = []
+    @tables.each { |table|
+#      puts "write table #{table}"
+#      puts "chunk.key= #{chunk.key}"
+      if table.mpattern.match(chunk.key)
+        log.info("add known chunk #{chunk.key}")
+        return table.send(@bigobject_url, chunk)
+      end
+    }
+    
+    log.warn("unknown chunk #{chunk.key}")
+      
+  end
+  
+  def format_tag(tag)
+    if @remove_tag_prefix
+      tag.gsub(@remove_tag_prefix, '')
+    else
+      tag
+    end
+  end
+  
+  def emit(tag, es, chain)
+    super(tag, es, chain, format_tag(tag))
+  end
+end

metadata

@@ -0,0 +1,107 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-bigobject
+version: !ruby/object:Gem::Version
+  version: 0.0.3
+platform: ruby
+authors:
+- Andrea Sung
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-09-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 'Fluentd output plugin to insert BIGOBJECT '
+email:
+- andrea@bigobject.io
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-bigobject.gemspec
+- lib/fluent/plugin/out_bigobject.rb
+homepage: https://github.com/andreasung/fluent-plugin-bigobject
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Fluentd output plugin to insert BIGOBJECT
+test_files: []