fluent-plugin-aliyun-odps 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8f2a437d1d6538a45d143aee240a08c6026450e9
-  data.tar.gz: 66c71203d5721ce578511f531e6e118755bcdf0f
+  metadata.gz: a83605973fae13e78ea73e422533ef6d613fe5fc
+  data.tar.gz: 8109f1fc93a0fce0869914893d073d25c911bf6d
 SHA512:
-  metadata.gz: 01aa158831e7788e28b58d8765ebb138f6489a23b673b75e5546881f0ca9118090e67406c51bae7592bcf1b47e3377473ac558b18d46ce1b811474cf9d72f80d
-  data.tar.gz: e2e82e1492eb4de88e093acf83196ce3ec0a159d76657d5dd210d027c1364c0ccc00ca760ceedaadf28eb999ec1fe273f4d4df46cad257f35ed709401cef19b0
+  metadata.gz: c614f06a4063a1b78bb728ec48d0530a05b1f5951bd7c49932df8fb642fe300f979ca2acba507988bb5bee4b647a91c4557e51c1d181e257600b76daef0ad205
+  data.tar.gz: 6a0c6f9027fde9a0ebd23445d89b522b1d5f4ea5744875e94d20a2bd8d1b37c210bcf1d84f139579f7b151310022258d58f52d1b26d56ae851ef860955b0fe94

data/CHANGELOG.md

@@ -1,2 +1,4 @@
 0.0.4
-Fix datetime format bug, support String, DateTime, Time type when write to a datetime field.
+Fix datetime format bug, support String, DateTime, Time type when write to a datetime field.
+0.0.5
+Add reload shard when import fails, and remove unload shard operation when shut down.

data/README.cn.md

@@ -91,7 +91,7 @@ $ cp aliyun-odps-fluentd-plugin/lib/fluent/plugin/* {YOUR_FLUENTD_DIRECTORY}/lib
         - ʱ���ʽ�ؼ���: partition ctime=${datetime.strftime('%Y%m%d')} ������datetimeΪsource��ijʱ���ʽ�ֶΣ����Ϊ%Y%m%d��ʽ��Ϊ�������ƣ�
 - time_format(Optional):
     - ���ʹ��ʱ���ʽ�ؼ���Ϊ<partition>, �����ñ�����. ����: source[datetime]="29/Aug/2015:11:10:16 +0800",������<time_format>Ϊ"%d/%b/%Y:%H:%M:%S %z"
-- shard_number(Optional):ָ��shard����������С��create tableʱָ����shardֵ����.
+- shard_number(Optional):ָ��shard���������������shard[0,shard_number-1]��Χ�ڵ�shard��д�����ݣ�����Ϊ����0��С��table��Ӧshard�������޵�����.
 
 ## �ٷ���վ
 ---

data/README.md

@@ -97,7 +97,7 @@ $ cp aliyun-odps-fluentd-plugin/lib/fluent/plugin/* {YOUR_FLUENTD_DIRECTORY}/lib
         - key words int time format: partition ctime=${datetime.strftime('%Y%m%d')}
 - time_format(Optional):
     - if you are using the key words to set your <partition> and the key word is in time format, please set the param <time_format>. example: source[datetime] = "29/Aug/2015:11:10:16 +0800", and the param <time_format> is "%d/%b/%Y:%H:%M:%S %z"
-- shard_number(Optional):less than the number you set when create the hubtable.
+- shard_number(Optional): will write data to shards between [0,shard_number-1], this config must more than 0 and less than the max shard number of your table.
 
 ## Useful Links
 ---

data/VERSION

@@ -1 +1 @@
-0.0.4
+0.0.5

data/lib/fluent/plugin/exceptions.rb

@@ -20,6 +20,7 @@ module OdpsDatahub
   $INVALID_ARGUMENT = "InvalidArgument"
   $SCHEMA_NOT_MATCH = "SchemaNotMatch"
   $PACK_SIZE_EXCEED = "PackSizeExceed"
+  $LOADSHARD_TIMEOUT = "LoadShardTimeOut"
   $ACCESS_DENIED = "AccessDenied"
 
   class OdpsDatahubException < StandardError

data/lib/fluent/plugin/odps/odps_table.rb

@@ -101,7 +101,7 @@ module OdpsDatahub
       if value.is_a?String and value.length < $STRING_MAX_LENTH
         setValue(idx, value)
       else
-        raise "value show be String and len < #$STRING_MAX_LENTH"
+        raise "value show be String and len < " + $STRING_MAX_LENTH.to_s
       end
     end
 

data/lib/fluent/plugin/out_aliyun_odps.rb

@@ -49,10 +49,9 @@ module Fluent
       config_param :fields, :string, :default => nil
       config_param :partition, :string, :default => nil
       config_param :num_retries, :integer, :default => 5
-      config_param :shard_number, :integer, :default => 1
+      config_param :shard_number, :integer, :default => 5
       config_param :thread_number, :integer, :default => 1
-      config_param :time_format, :string, :default => nil 
-      config_param :record_batch_size, :integer, :default => 10
+      config_param :time_format, :string, :default => nil
       config_param :time_out, :integer, :default => 300
       attr_accessor :partitionList
       attr_reader :client
@@ -88,34 +87,13 @@ module Fluent
                                                  config[:aliyun_odps_endpoint],
                                                  config[:aliyun_odps_hub_endpoint],
                                                  config[:project])
-        if @record_batch_size<=0
-          raise "the table "+ @table+"'s record_batch_size is must more than 0"
+        if @shard_number<=0
+          raise "shard number must more than 0"
         end
         begin
           @client = OdpsDatahub::StreamClient.new(odpsConfig, config[:project], @table)
           @client.loadShard(@shard_number)
-          allLoaded = false
-          loadtime=0
-          while !allLoaded do
-            count = 0
-            #get json like [{"ShardId": "0","State": "loaded"},{"ShardId": "1","State": "loaded"}]
-            @client.getShardStatus.each { |shard|
-              if shard["State"] != "loaded"
-                sleep(5)
-                loadtime+=5
-                break
-              else
-                count += 1
-              end
-              if count == @shard_number
-                allLoaded = true
-                @log.info "All shareds are loaded successfully"
-              end
-              if loadtime>=300
-                raise "Load shared timeout"
-              end
-            }
-          end
+          @client.waitForShardLoad
           for i in 0..@thread_number-1
             @writer[i] = @client.createStreamArrayWriter()
           end
@@ -269,13 +247,18 @@ module Fluent
             sendThread[thread].join
           end
         rescue => e
+          # reload shard
+          if e.message.include? "ShardNotReady" or e.message.include? "InvalidShardId"
+            @client.loadShard(@shard_number)
+            @client.waitForShardLoad
+          end
           # ignore other exceptions to use Fluentd retry
           raise "write records failed,"+e.message
         end
       end
 
       def close()
-        @client.loadShard(0)
+        #@client.loadShard(0)
       end
 
     end

data/lib/fluent/plugin/stream_client.rb

@@ -87,6 +87,24 @@ module OdpsDatahub
       return json_obj["ShardStatus"]
     end
 
+    def waitForShardLoad(timeOut = 120) #seconds
+      if timeOut < 0
+        raise OdpsDatahubException.new($INVALID_ARGUMENT, "waitForShardLoad param invalid")
+      end
+      allLoaded = false
+      loadtime=0
+      while loadtime < timeOut do
+        if isShardLoadCompleted
+          return
+        end
+        sleep(5)
+        loadtime+=5
+      end
+      if !isShardLoadCompleted
+        raise OdpsDatahubException.new($LOADSHARD_TIMEOUT, "waitForShardLoad timeOut")
+      end
+    end
+
     def loadShard(idx)
       if idx < 0
         raise OdpsDatahubException.new($INVALID_ARGUMENT, "loadShard num invalid")
@@ -107,5 +125,15 @@ module OdpsDatahub
     def getResource
       return "/projects/" + @mProject + "/tables/" + @mTable
     end
+
+    def isShardLoadCompleted
+      #get json like [{"ShardId": "0","State": "loaded"},{"ShardId": "1","State": "loaded"}]
+      getShardStatus.each { |shard|
+        if shard["State"] != "loaded"
+          return false
+        end
+      }
+      return true
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-aliyun-odps
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Xiao Dong
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-30 00:00:00.000000000 Z
+date: 2015-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd