fluent-plugin-bigobject 0.0.4 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b532c44be9dd5707702865658e23cf5caa576a8c
-  data.tar.gz: 178e9d20b25b1e6b0b92fe7cf8bbb65a44105c37
+  metadata.gz: a7bfafe5bf6153cafec9e23ce80ef2ab726dfe71
+  data.tar.gz: 4e45524019de554685fc87474f1cadc0f3bf7003
 SHA512:
-  metadata.gz: 74fe232ff45bd2faf2332d941e67d5da9dc2ec0466a4344a235272ea14eac4ffe5d2ad9ad44d420759e32e5366418dc9c67d5b5dc65f1aeb6560bba391fc405c
-  data.tar.gz: 59babcb0dc08e943635bc74c9ba8943ed43a256b72598bd95c15e0bdaa5aa75c5ba325cc3d2c1feb29fe3e0d193e92909582b84e42b3ec2bbc878dde60aa5f0f
+  metadata.gz: dccf9efcdd694916ba407cc7f84889297d43a5e17ed630a65a2cc33d1ec5bb78106a354e4c8b54f02da0f23b871af0b5ffe713d3d42d1b9084cfc3ea6e71e758
+  data.tar.gz: ce965346785948872b30ca6bf0b1e4a3deb08a91309428615c1b45cdb0c847f81c9285e73ddc6db0700b881a01ef31b997ae6400bd7a0552b8b3344c64ffe396

data/README.md

@@ -28,27 +28,48 @@ Configure BigObject URL and the table/column to be mapped in BigObject
 
   log_level info
 
-  # specify the bigobject_url to connect to
-  bigobject_url http://192.168.59.103:9090/cmd
+  # specify the bigobject to connect to
+  bigobject_hostname 192.168.59.103
+  bigobject_port 9091
 
   remove_tag_prefix bo.insert. 
   flush_interval 5s
 
   <table>
-      table Customer
-      column_mapping id,name,language,state,company,gender,age
+      #example of sending data to BigObject using binary avro
+      #table name is specified in avsc file
       pattern customer
-      #bo_workspace
-      #bo_opts
+      schema_file /fluentd/input/avsc/Customer.avsc
+
+      #optional - 
+      #column_mapping id1:id,name,language,state,company,gender,age
   </table>
 
+</match>
+
+<match bo.insert_rest.*>
+  type bigobject
+
+  log_level info 
+
+  # specify the bigobject_url to connect to
+  bigobject_hostname 192.168.59.103
+  bigobject_port 9090
+
+  remove_tag_prefix bo.insert_rest. 
+  flush_interval 5s
+
   <table>
-    table search_test3
-    # map to different column name in BigObject
-    column_mapping 'ID:id3, TEXT:text3'
-    pattern test3
-  </table>
+      #example of sending data to BigObject using restful API 
+      table Customer2
+      pattern customer2
 
+      #optional -- 
+      #column_mapping id1:id,name,language,state,company,gender,age
+      #bo_workspace
+      #bo_opts
+
+  </table>
 </match>
 ```
 

data/fluent-plugin-bigobject.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-bigobject"
-  gem.version       = "0.0.4"
+  gem.version       = "0.0.7"
   gem.authors       = ["Andrea Sung"]
   gem.email         = ["andrea@bigobject.io"]
   gem.description   = %q{Fluentd output plugin to insert BIGOBJECT }
@@ -17,5 +17,6 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "fluentd"
   gem.add_runtime_dependency "rest-client"
   gem.add_runtime_dependency "json"
+  gem.add_development_dependency "avro"
   gem.add_development_dependency "rake"
 end

data/lib/fluent/plugin/out_bigobject.rb

@@ -1,10 +1,12 @@
 class Fluent::BigObjectOutput < Fluent::BufferedOutput
+
   Fluent::Plugin.register_output('bigobject', self)
   
   include Fluent::SetTimeKeyMixin
   include Fluent::SetTagKeyMixin
 
-  config_param :bigobject_url, :string
+  config_param :bigobject_hostname, :string
+  config_param :bigobject_port, :integer
   config_param :remove_tag_prefix, :string, :default => nil
   config_param :send_unknown_chunks, :string,  :default=>true
 
@@ -17,34 +19,58 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
   class TableElement
     include Fluent::Configurable
 
-    config_param :table, :string
-    config_param :column_mapping, :string
+    config_param :table, :string, :default=>nil
+    config_param :column_mapping, :string, :default=>nil
     config_param :pattern, :string, :default=>nil
     config_param :bo_workspace, :string, :default=>nil
     config_param :bo_opts, :string, :default=>nil
+    config_param :schema_file, :string, :default => nil
 
     attr_reader :mpattern
 
-    def initialize(log)
+    def initialize(log, bo_hostname, bo_port)
       super()
       @log = log
+      @bo_hostname = bo_hostname
+      @bo_port = bo_port
+      @bo_url="http://#{@bo_hostname}:#{@bo_port}/cmd"
     end
 
     def configure(conf)
       super
+      if (@table==nil)&&(@schema_file==nil)
+        raise "Table name and schema_file cannot be both nil. Please specify <schema_file> if using avro input or <table> is using restful api." 
+      end 
+      if (isBinary) 
+        @avro_schema = Avro::Schema.parse(File.open(@schema_file, "rb").read)
+        @avro_writer = Avro::IO::DatumWriter.new(@avro_schema)
+      else 
+        @avro_schema = nil
+        @avro_writer = nil
+      end
+
       @mpattern = Fluent::MatchPattern.create(pattern)
-      @mapping = parse_column_mapping(@column_mapping)
+      @mapping = (@column_mapping==nil)? nil:parse_column_mapping(@column_mapping)
       @log.info("column mapping for #{table} - #{@mapping}")
       @format_proc = Proc.new { |record|
-        new_record = {}
-        @mapping.each { |k, c|
-          new_record[c] = record[k]
-        }
-        new_record
+        if (@mapping==nil)
+          record
+        else
+          new_record = {}
+          @mapping.each { |k, c|
+            new_record[c] = record[k]
+            }
+          new_record
+        end
       }
     end
+    
+    def isBinary()
+      return !(@schema_file.to_s.empty?)
+    end
 
-    def send(bourl, chunk)
+    #Send Data to Bigobject using Restful API
+    def send_rest(chunk)
       stmts = Array.new
       i=0
       columns = nil
@@ -59,25 +85,64 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
          if columns.to_s.empty?
            columns = "(#{keys.join(",")})"
          end
-         #single quote each column data
          stmts.push("('#{values.join("','")}')")
          i+=1
       }
       
       sendStmt = "INSERT INTO #{@table}  #{columns} VALUES" + stmts.join(",")
-#      @log.debug("sendStmt=", sendStmt)
-      @log.info("bigobject start insert #{i} rows")
-      resp = sendBO(bourl, sendStmt)
+      
+      resp = sendBO(@bo_url, sendStmt)
       parsed = JSON.parse(resp)
       err = parsed['Err']
-#      puts "Content=#{parsed['Content']}, Err=#{err}"
       if (err.to_s!='')
         @log.error("[BigObject] #{err}")
       end
-      @log.info("bigobject end insert #{i} rows")
+      @log.debug("bigobject insert #{i} rows")
       
     end
     
+    #Send data to Bigobject using binary AVRO
+    def send_binary(chunk)
+      
+      buffer = StringIO.new()      
+      dw = Avro::DataFile::Writer.new(buffer, @avro_writer, @avro_schema)
+      i=0
+      chunk.msgpack_each { |tag, time, data|
+         data = @format_proc.call(data)
+         dw<<data
+         i+=1
+      }
+      dw.flush
+
+      begin
+        socket = TCPSocket.open(@bo_hostname, @bo_port)
+        begin
+          #timeout=60
+          opt = [1, 60].pack('I!I!')  # { int l_onoff; int l_linger; }
+          socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
+  
+          opt = [60, 0].pack('L!L!')  # struct timeval
+          socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
+          socket.write(buffer.string)
+        ensure
+          socket.close
+        end
+        
+      rescue Exception => e 
+          @log.error(e.message)  
+          raise "Failed to send_binary: #{e.message}"
+      end
+      @log.debug("bigobject send #{i} rows")
+    end
+    
+    def send(chunk)
+        if (isBinary) 
+          send_binary(chunk)
+        else 
+          send_rest(chunk)
+        end
+    end
+    
     def to_s
       "table:#{table}, column_mapping:#{column_mapping}, pattern:#{pattern}"
     end
@@ -108,8 +173,6 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
     
     def sendBO(bourl, sendStmt)
       params = formatRequest(sendStmt)
-      @log.debug("\nbourl=#{bourl} params=#{params.to_json}")
-    
       begin
         resp = RestClient.post bourl, params.to_json, :content_type =>:json, :accept =>:json
         @log.debug("resp= #{resp.body}")  
@@ -117,6 +180,7 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
         @log.error(e.message)  
         raise "Failed to sendBO: #{e.message}"
       end
+      
       return resp
     end
    
@@ -126,6 +190,7 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
     super
     require 'rest-client'
     require 'json'
+    require 'avro'
     log.info("bigobject initialize")
   end  
   
@@ -138,12 +203,12 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
 
     @tables = []
     @default_table = nil
+ 
     conf.elements.select { |e|
       e.name == 'table'
     }.each { |e|
-      te = TableElement.new(log)
+      te = TableElement.new(log, @bigobject_hostname, @bigobject_port)
       te.configure(e)
-#      puts "conf.elements #{e}"
       @tables << te
     }
     
@@ -157,13 +222,10 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
   
   def shutdown
     super
-    log.info("bigobject shutdown")
   end 
-  
-  
+
   # This method is called when an event reaches to Fluentd.
   def format(tag, time, record)
-#    puts "tag=#{tag}, time=#{time}, record=#{record}"
     [tag, time, record].to_msgpack
   end
   
@@ -173,11 +235,8 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
   def write(chunk)
     unknownChunks = []
     @tables.each { |table|
-#      puts "write table #{table}"
-#      puts "chunk.key= #{chunk.key}"
       if table.mpattern.match(chunk.key)
-        log.info("add known chunk #{chunk.key}")
-        return table.send(@bigobject_url, chunk)
+        return table.send(chunk)
       end
     }
     
@@ -196,4 +255,4 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
   def emit(tag, es, chain)
     super(tag, es, chain, format_tag(tag))
   end
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigobject
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.7
 platform: ruby
 authors:
 - Andrea Sung
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-10 00:00:00.000000000 Z
+date: 2015-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: avro
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement