fluent-plugin-aliyun-odps 0.0.9 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3db6116eb96dd0bcba4322be5691dda1db3b8c84
-  data.tar.gz: eccbe6c58ce579d2b88de9a19742520bf749fdcf
+  metadata.gz: 24f7ee66fa1af9233ffca9aa110d10c6e74cf4d7
+  data.tar.gz: 7dd141195033863d501c3e2be9ecc0e0e2075c94
 SHA512:
-  metadata.gz: 7475f393381c276de37b1ca0df214db31fdb72eb60857cf29bc5038118dab497c1e28d1ad52ad303bb4756fd0fea3ce4d6b7fe98d288e8a8504b9fb04c74d52b
-  data.tar.gz: 7f0a56ac253cb3c88b4328c4d7a83c000a912034ca4290b8d4c0de8de972a16336d4df13a3a5e8fdfe697472f945ea3c19917a3f0d283ac3508f6692aeda85a2
+  metadata.gz: 749bc9bfc987581ed7bbbff279ba61a695a3138ddb11a1f8c3d8d3588eee60d1c3c67a0aebb4fffa144215869827dcc9abb029551f6a9b87d75ebc55eaf4ea9e
+  data.tar.gz: 87ab9ea47266abb85bf222195cd64210a6fd9284864cec71964bd405da8fcd53bcfa68cc1802615024642e97ea71281b205a0069c6a22f890eec972cd750c546

data/CHANGELOG.md

@@ -9,4 +9,6 @@ Add error msg when add partition fail, support fast crc
 0.0.8
 Add abandon mode, fix fluent retry bug, fix partition mixed mode bug.
 0.0.9
-Hotfix retry log error bug.
+Hotfix retry log error bug.
+0.1.0
+Add partition when catch NoSuchPartition.

data/VERSION

@@ -1 +1 @@
-0.0.9
+0.1.0

data/lib/fluent/plugin/out_aliyun_odps.rb

@@ -97,15 +97,6 @@ module Fluent
           @client = OdpsDatahub::StreamClient.new(odpsConfig, config[:project], @table)
           @client.loadShard(@shard_number)
           @client.waitForShardLoad
-          partitionMaps=@client.getPartitionList
-          @partitionList=[]
-          for map in partitionMaps do
-            partitionName=''
-            map.each { |k, v|
-              partitionName+=k+"="+v+","
-            }
-            @partitionList<<partitionName.chomp(",")
-          end
         rescue => e
           raise "loadShard failed,"+e.message
         end
@@ -180,7 +171,7 @@ module Fluent
             end
 
           rescue => e
-            raise "Failed to format the data:"+e.backtrace.inspect.to_s
+            raise "Failed to format the data:"+ e.message + " " +e.backtrace.inspect.to_s
           end
         }
 
@@ -188,19 +179,11 @@ module Fluent
           #multi thread
           sendThread = Array.new
           unless @partition.blank? then
-            partitions.each { |k, v|
-              #if the partition is not exist, create one
-              unless @partitionList.include?(k)
-                @client.addPartition(k)
-                @partitionList << k
-                @log.info "add partition "+k
-              end
-            }
             for thread in 0..@thread_number-1
               sendThread[thread] = Thread.start(thread) do |threadId|
-                retryTime = @retry_time
-                begin
-                  partitions.each { |k, v|
+                partitions.each { |k, v|
+                  retryTime = @retry_time
+                  begin
                     sendCount = v.size/@thread_number
                     restCount = 0
                     if threadId == @thread_number-1
@@ -208,27 +191,34 @@ module Fluent
                     end
                     @client.createStreamArrayWriter().write(v[sendCount*threadId..sendCount*(threadId+1)+restCount-1], k)
                     @log.info "Successfully  import "+(sendCount+restCount).to_s+" data to partition:"+k+",table:"+@table+" at threadId:"+threadId.to_s
-                  }
-                rescue => e
-                  # reload shard
-                  if e.message.include? "ShardNotReady" or e.message.include? "InvalidShardId"
-                    @log.warn "write failed, msg:" + e.message + ", reload shard."
-                    @client.loadShard(@shard_number)
-                    @client.waitForShardLoad
-                  end
-                  if retryTime > 0
-                    @log.warn "Fail to write, retry in " + @retry_interval.to_s + "sec. Error at threadId:"+threadId.to_s+" Msg:"+e.message
-                    sleep(@retry_interval)
-                    retryTime -= 1
-                    retry
-                  else
-                    if (@abandon_mode)
-                      @log.error "Retry failed, abandon this pack. Msg:" + e.message
+                  rescue => e
+                    # reload shard
+                    if e.message.include? "ShardNotReady" or e.message.include? "InvalidShardId"
+                      @log.warn "write failed, msg:" + e.message + ", reload shard."
+                      @client.loadShard(@shard_number)
+                      @client.waitForShardLoad
+                    elsif e.message.include? "NoSuchPartition"
+                      begin
+                        @client.addPartition(k)
+                        @log.info "add partition "+ k
+                      rescue => ex
+                        @log.error "add partition failed"+ ex.message
+                      end
+                    end
+                    if retryTime > 0
+                      @log.warn "Fail to write, retry in " + @retry_interval.to_s + "sec. Error at threadId:"+threadId.to_s+" Msg:"+e.message
+                      sleep(@retry_interval)
+                      retryTime -= 1
+                      retry
                     else
-                      raise e
+                      if (@abandon_mode)
+                        @log.error "Retry failed, abandon this pack. Msg:" + e.message
+                      else
+                        raise e
+                      end
                     end
                   end
-                end
+                }
               end
             end
           else
@@ -273,7 +263,7 @@ module Fluent
           end
         rescue => e
           # ignore other exceptions to use Fluentd retry
-          raise "write records failed,"+e.backtrace.inspect.to_s
+          raise "write records failed," + e.message + " " +e.backtrace.inspect.to_s
         end
       end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-aliyun-odps
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.1.0
 platform: ruby
 authors:
 - Xiao Dong
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-05 00:00:00.000000000 Z
+date: 2016-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd