fluent-plugin-bigobject 0.0.10 → 0.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a45409dcfa7c2d66087c327e66fb4d5aa6855434
-  data.tar.gz: aa29ec1f4e23c6f2e37526d6cb39013a0fefd536
+  metadata.gz: 476a311048599f15ef608f470019aee3755f480a
+  data.tar.gz: b5ed61454e7d2aed2bff4d7aef43707b7b3b7d1a
 SHA512:
-  metadata.gz: 63d9945a6f7db0e106eef8d2bbdebdb4316e8b47b833f952c81f5ac1533c0b050adc65a9ee065a1fcab21dbd3c3376cf2c8e585ce4b662401f4f21075375c773
-  data.tar.gz: 5d89f98e21cd45b2b5cd74add1aac255ffca1bbef80dd0df42d78a5db81eb4dd219795003047cd799381f3cc5a08508b1184edbc9e94c49ab57e5ae68c947e93
+  metadata.gz: c0f3e2399d792c2fc2b92ba74c15a8cfa2dca4e67734553006c3c9730f3da84f53b46f0954d4f51fc40587f72c24b51f7c6e731451e703a6465a5b316938847d
+  data.tar.gz: a5ff36959b8f31f02e28c2be9b464e18eee2d35557599c9da6a13d53ad35f1b04552d6d910c37fc6708c2b7a17662091817a2c6ff09f2d1ddb2a7eecd12c2d1c

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015 MacroData 
+Copyright (c) 2015 BigObject 
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -24,27 +24,46 @@ Configure BigObject URL and the table/column to be mapped in BigObject
 
 ```apache
 
-# send data to BigObject using avro by providing schema_file in each table
-<match bo.insert_avro.*>
-  type bigobject
-
-  log_level info
-
-  # specify the bigobject host/port to connect to
-  bigobject_hostname 192.168.59.103
-  bigobject_port 9091
-
-  remove_tag_prefix bo.insert_avro.
-  flush_interval 60s
-
-  <table>
-      pattern customer
-      schema_file /fluentd/input/avsc/Customer_binary.avsc
-  </table>
-</match>
-
-# send data to BigObject using Restful API. Tables need to be created in advance in BigObject.
-<match bo.insert_rest.*>
+#sample source to read csv file
+<source>
+  type tail
+
+  #path- where you placed your input data
+  path ./input/Customer.csv
+
+  # pos_file where you record file position
+  pos_file ./log/customer.log.pos
+
+  # for bigobject output plugin, use tag bigobject.${table_pattern}.${event}.${primary_key}
+  # ${primary_key} is not needed for insert
+  tag bigobject.cust.insert
+
+  #input file format
+  format csv
+
+  # keys - columns in csv file
+  keys id,name,language,state,company,gender,age
+
+  #types - string/bool/integer/float/time/array
+  types age1:integer
+
+</source>
+
+# Send data to BigObject using Restful API. Tables need to be created in advance in BigObject.
+# depending on the event in tag received, will send data to BigObject for insert/update/delete. 
+#
+# Tag for each event - bigobject.${table_pattern}.${event}.${primary_key}.
+# ${table_pattern} : will match to the <pattern> in <table> section of bigobject output plugin
+# ${event} : valid event type by insert/update/delete.
+# ${primary_key} : the primary key for table, optional for insert event.
+# if primary_key is integer type in BigObject, set bo_primary_key_is_int to true 
+#
+# Eg:
+# tag bigobject.cust.insert ==> INSERT INTO <table> VALUES ...
+# tag bigobject.cust.delete.id ==> DELETE FROM <table> WHERE id=...
+# tag bigobject.cust.update.id ==> UPDATE <table> SET ... WHERE id=...
+
+<match bigobject.**>
   type bigobject
 
   log_level info
@@ -53,14 +72,15 @@ Configure BigObject URL and the table/column to be mapped in BigObject
   bigobject_hostname 192.168.59.103
   bigobject_port 9090
 
-  remove_tag_prefix bo.insert_rest.
+  remove_tag_prefix bigobject.
   flush_interval 60s
 
   <table>
       table Customer
-      pattern customer
+      pattern cust
 
       #optional-
+      #bo_primary_key_is_int true #defualts to false
       #column_mapping id,name,language,state,company,gender,age
       #bo_workspace
       #bo_opts

data/fluent-plugin-bigobject.gemspec

@@ -1,12 +1,12 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-bigobject"
-  gem.version       = "0.0.10"
+  gem.version       = "0.0.12"
   gem.authors       = ["Andrea Sung"]
   gem.email         = ["andrea@bigobject.io"]
-  gem.description   = %q{Fluentd output plugin to insert BIGOBJECT }
-  gem.summary       = %q{Fluentd output plugin to insert BIGOBJECT}
-  gem.homepage      = "https://github.com/macrodatalab/fluent-plugin-bigobject"
+  gem.description   = %q{Fluentd output plugin to insert/update/delete data in BIGOBJECT }
+  gem.summary       = %q{Fluentd output plugin to insert/update/delete data in BIGOBJECT}
+  gem.homepage      = "https://github.com/bigobject-inc/fluent-plugin-bigobject"
   gem.license       = "MIT"
 
   gem.files         = `git ls-files`.split($\)
@@ -17,6 +17,5 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "fluentd"
   gem.add_runtime_dependency "rest-client"
   gem.add_runtime_dependency "json"
-  gem.add_runtime_dependency "avro"
   gem.add_development_dependency "rake"
 end

data/lib/fluent/plugin/out_bigobject.rb

@@ -8,7 +8,10 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
   config_param :bigobject_hostname, :string
   config_param :bigobject_port, :integer
   config_param :remove_tag_prefix, :string, :default => nil
-  config_param :send_unknown_chunks, :string,  :default=>true
+  config_param :tag_format, :string, :default => nil
+  
+#  DEFAULT_TAG_FORMAT = /(?<table_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$/
+  DEFAULT_TAG_FORMAT = /^(?<table_name>[^\.]+)\.(?<event>[^\.]+)(\.(?<primary_key>[^\.]+))*/
 
   attr_accessor :tables
   
@@ -19,35 +22,26 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
   class TableElement
     include Fluent::Configurable
 
-    config_param :table, :string, :default=>nil
+    config_param :table, :string
     config_param :column_mapping, :string, :default=>nil
     config_param :pattern, :string, :default=>nil
     config_param :bo_workspace, :string, :default=>nil
     config_param :bo_opts, :string, :default=>nil
-    config_param :schema_file, :string, :default => nil
+    config_param :bo_primary_key_is_int, :bool, :default=>false
 
     attr_reader :mpattern
 
-    def initialize(log, bo_hostname, bo_port)
+    def initialize(log, bo_hostname, bo_port, tag_format)
       super()
       @log = log
       @bo_hostname = bo_hostname
       @bo_port = bo_port
       @bo_url="http://#{@bo_hostname}:#{@bo_port}/cmd"
+      @tag_format = tag_format
     end
 
     def configure(conf)
       super
-      if (@table==nil)&&(@schema_file==nil)
-        raise "Table name and schema_file cannot be both nil. Please specify <schema_file> if using avro input or <table> is using restful api." 
-      end 
-      if (isBinary) 
-        @avro_schema = Avro::Schema.parse(File.open(@schema_file, "rb").read)
-        @avro_writer = Avro::IO::DatumWriter.new(@avro_schema)
-      else 
-        @avro_schema = nil
-        @avro_writer = nil
-      end
 
       @mpattern = Fluent::MatchPattern.create(pattern)
       @mapping = (@column_mapping==nil)? nil:parse_column_mapping(@column_mapping)
@@ -65,82 +59,71 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
       }
     end
     
-    def isBinary()
-      return !(@schema_file.to_s.empty?)
+    def getPkeyValue(value)
+      if (@bo_primary_key_is_int)
+           return value
+      else
+           return"\"#{value}\""
+      end
     end
-
+    
     #Send Data to Bigobject using Restful API
-    def send_rest(chunk)
-      stmts = Array.new
-      i=0
+    def send(chunk)
+      insertStmts = Array.new
+      deleteStmts = Array.new
+      
       columns = nil
       chunk.msgpack_each { |tag, time, data|
+         tag_parts = tag.match(@tag_format)
+         target_event = tag_parts['event']
+         id_key = tag_parts['primary_key']
+         
          keys = Array.new
          values = Array.new
          data = @format_proc.call(data)
          data.keys.sort.each do |key|
-           keys << key
-           values << data[key].to_json
+            keys << key
+            values << data[key].to_json
          end
-         if columns.to_s.empty?
-           columns = "(#{keys.join(",")})"
+          
+         if (target_event=='insert')
+            if columns.to_s.empty?
+              columns = "(#{keys.join(",")})"
+            end
+            insertStmts.push("(#{values.join(",")})")
+         elsif (target_event=='update')
+           pkey=""
+           updates = Array.new
+           keys.zip(values) { |key, value|
+               if (key==id_key)
+                 pkey = getPkeyValue(value)
+               else
+                 updates.push("#{key}=#{value}")
+               end 
+           }
+           sendStmt = "UPDATE #{table} SET #{updates.join(",")} WHERE #{id_key}=#{pkey}"
+           sendBO(@bo_url, sendStmt)   
+         elsif (target_event=='delete')
+           keys.zip(values) { |key, value|
+                if (key==id_key)
+                  pkey = getPkeyValue(value)
+                end
+                deleteStmts.push("#{id_key}=#{pkey}")
+            }
          end
-         stmts.push("(#{values.join(",")})")
-         #stmts.push("(\"#{values.join("\",\"")}\")")
-         i+=1
       }
       
-      sendStmt = "INSERT INTO #{@table}  #{columns} VALUES" + stmts.join(",")
-      resp = sendBO(@bo_url, sendStmt)
-      parsed = JSON.parse(resp)
-      err = parsed['Err']
-      if (err.to_s!='')
-        @log.error("[BigObject] #{err}")
-      end
-      @log.debug("bigobject insert #{i} rows")
-      
-    end
-    
-    #Send data to Bigobject using binary AVRO
-    def send_binary(chunk)
+      if insertStmts.length>0
+        sendStmt = "INSERT INTO #{@table}  #{columns} VALUES " + insertStmts.join(",")
+        sendBO(@bo_url, sendStmt)
+        @log.debug("sending #{insertStmts.length} rows to bigobject for insert via Restful API")
+      end 
       
-      buffer = StringIO.new()      
-      dw = Avro::DataFile::Writer.new(buffer, @avro_writer, @avro_schema)
-      i=0
-      chunk.msgpack_each { |tag, time, data|
-         data = @format_proc.call(data)
-         dw<<data
-         i+=1
-      }
-      dw.flush
-
-      begin
-        socket = TCPSocket.open(@bo_hostname, @bo_port)
-        begin
-          #timeout=60
-          opt = [1, 60].pack('I!I!')  # { int l_onoff; int l_linger; }
-          socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
-  
-          opt = [60, 0].pack('L!L!')  # struct timeval
-          socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
-          socket.write(buffer.string)
-        ensure
-          socket.close
-        end
-        
-      rescue Exception => e 
-          @log.error(e.message)  
-          raise "Failed to send_binary: #{e.message}"
+      if deleteStmts.length>0
+        sendStmt = "DELETE FROM #{@table} WHERE " + deleteStmts.join(" or ")
+        sendBO(@bo_url, sendStmt)
+        @log.debug("sending #{deleteStmts.length} rows to bigobject for delete via Restful API")
       end
-      @log.debug("bigobject send #{i} rows")
-    end
-    
-    def send(chunk)
-        if (isBinary) 
-          send_binary(chunk)
-        else 
-          send_rest(chunk)
-        end
     end
     
     def to_s
@@ -181,7 +164,12 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
         raise "Failed to sendBO: #{e.message}"
       end
       
-      return resp
+      parsed = JSON.parse(resp)
+      err = parsed['Err']
+      if (err.to_s!='')
+        @log.error("[BigObject] #{err}")
+      end
+      
     end
    
   end #end class
@@ -190,7 +178,6 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
     super
     require 'rest-client'
     require 'json'
-    require 'avro'
     log.info("bigobject initialize")
   end  
   
@@ -201,13 +188,19 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
       @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
     end
 
+    if @tag_format.nil? || @tag_format == DEFAULT_TAG_FORMAT
+      @tag_format = DEFAULT_TAG_FORMAT
+    else
+      @tag_format = Regexp.new(conf['tag_format'])
+    end
+    
     @tables = []
     @default_table = nil
  
     conf.elements.select { |e|
       e.name == 'table'
     }.each { |e|
-      te = TableElement.new(log, @bigobject_hostname, @bigobject_port)
+      te = TableElement.new(log, @bigobject_hostname, @bigobject_port, @tag_format)
       te.configure(e)
       @tables << te
     }
@@ -234,8 +227,12 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
   # 'chunk' is a buffer chunk that includes multiple formatted events. 
   def write(chunk)
     unknownChunks = []
+    tag = chunk.key
+    tag_parts = tag.match(@tag_format)
+    target_table = tag_parts['table_name']
+      
     @tables.each { |table|
-      if table.mpattern.match(chunk.key)
+      if table.mpattern.match(target_table)
         return table.send(chunk)
       end
     }
@@ -253,6 +250,7 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
   end
   
   def emit(tag, es, chain)
-    super(tag, es, chain, format_tag(tag))
+    nt = format_tag(tag)
+    super(nt, es, chain, nt)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigobject
 version: !ruby/object:Gem::Version
-  version: 0.0.10
+  version: 0.0.12
 platform: ruby
 authors:
 - Andrea Sung
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-07 00:00:00.000000000 Z
+date: 2015-11-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -52,20 +52,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: avro
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -80,7 +66,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 'Fluentd output plugin to insert BIGOBJECT '
+description: 'Fluentd output plugin to insert/update/delete data in BIGOBJECT '
 email:
 - andrea@bigobject.io
 executables: []
@@ -94,7 +80,7 @@ files:
 - Rakefile
 - fluent-plugin-bigobject.gemspec
 - lib/fluent/plugin/out_bigobject.rb
-homepage: https://github.com/macrodatalab/fluent-plugin-bigobject
+homepage: https://github.com/bigobject-inc/fluent-plugin-bigobject
 licenses:
 - MIT
 metadata: {}
@@ -117,5 +103,5 @@ rubyforge_project:
 rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
-summary: Fluentd output plugin to insert BIGOBJECT
+summary: Fluentd output plugin to insert/update/delete data in BIGOBJECT
 test_files: []