fluent-plugin-aliyun-odps 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 24f7ee66fa1af9233ffca9aa110d10c6e74cf4d7
-  data.tar.gz: 7dd141195033863d501c3e2be9ecc0e0e2075c94
+  metadata.gz: 96472dd18671b2adc39de0934ea13aac995b81b9
+  data.tar.gz: b9f1ffc43b9f1b81be6081201d811074e28a3bb5
 SHA512:
-  metadata.gz: 749bc9bfc987581ed7bbbff279ba61a695a3138ddb11a1f8c3d8d3588eee60d1c3c67a0aebb4fffa144215869827dcc9abb029551f6a9b87d75ebc55eaf4ea9e
-  data.tar.gz: 87ab9ea47266abb85bf222195cd64210a6fd9284864cec71964bd405da8fcd53bcfa68cc1802615024642e97ea71281b205a0069c6a22f890eec972cd750c546
+  metadata.gz: e3d190f5bbc2f7efb676485541b593b7a5e58177aa9ac41a6c04ba4f958594e3540a57c3fdd04571e5c6ee4e765cfe8419fc8265302f2bb1e4c103ee8f0ec4e0
+  data.tar.gz: f179cef524c9925924238c6660d05df73eaaa10833ac228b59d326f07cbd3b5b3be69885f71e5b4c1d3d3436997451a48db113b824ab3c953ff77004bba4e85e

data/CHANGELOG.md

@@ -11,4 +11,6 @@ Add abandon mode, fix fluent retry bug, fix partition mixed mode bug.
 0.0.9
 Hotfix retry log error bug.
 0.1.0
-Add partition when catch NoSuchPartition.
+Add partition when catch NoSuchPartition.
+0.1.1
+Fix some log format.

data/README.cn.md

@@ -27,18 +27,6 @@
 $ gem install fluent-plugin-aliyun-odps
 ```
 
-### Դ�밲װ���
-
-* ���Ȱ�װ����������fluentd�Լ�protobuf��
-* Ȼ�󽫲��Ŀ¼������Fluentd���Ŀ¼�£����磺 ��aliyun-odps-fluentd-plugin/lib/fluent/pluginĿ¼���Ƶ�{YOUR_FLUENTD_DIRECTORY}/lib/fluent/plugin�С�
-
-```
-$ gem install protobuf
-$ gem install fluentd --no-ri --no-rdoc
-$ git clone https://github.com/aliyun/aliyun-odps-fluentd-plugin.git
-$ cp aliyun-odps-fluentd-plugin/lib/fluent/plugin/* {YOUR_FLUENTD_DIRECTORY}/lib/fluent/plugin/ -r
-```
-
 ### ���ʹ��ʾ��
 
 ```
@@ -63,12 +51,16 @@ $ cp aliyun-odps-fluentd-plugin/lib/fluent/plugin/* {YOUR_FLUENTD_DIRECTORY}/lib
   buffer_queue_limit 128
   flush_interval 5s
   project your_projectName
+  enable_fast_crc true
   <table in.log>
 	table your_tableName
 	fields remote,method,path,code,size,agent
 	partition ctime=${datetime.strftime('%Y%m%d')}
 	time_format %d/%b/%Y:%H:%M:%S %z
 	shard_number 1
+	retry_time 3
+	retry_interval 1
+	abandon_mode true
   </table>
 </match>
 ```
@@ -94,7 +86,10 @@ $ cp aliyun-odps-fluentd-plugin/lib/fluent/plugin/* {YOUR_FLUENTD_DIRECTORY}/lib
 - time_format(Optional):
     - ���ʹ��ʱ���ʽ�ؼ���Ϊ<partition>, �����ñ�����. ����: source[datetime]="29/Aug/2015:11:10:16 +0800",������<time_format>Ϊ"%d/%b/%Y:%H:%M:%S %z"
 - shard_number(Optional):ָ��shard���������������shard[0,shard_number-1]��Χ�ڵ�shard��д�����ݣ�����Ϊ����0��С��table��Ӧshard�������޵�����.
-- enable_fast_crc(Optional): ʹ�ÿ���crc���㣬�⽫�����������ܣ���������ʹ�����ⲿ���صĶ�̬���ӿ⣬Ŀǰ��֧��64λlinux��windowsϵͳ��
+- enable_fast_crc(Optional): ʹ�ÿ���crc���㣬�⽫�����������ܣ���������ʹ�����ⲿ���صĶ�̬���ӿ⣬Ŀǰ��֧��64λlinux��windowsϵͳ.
+- retry_time(Optional): ����ÿ��pack����ʱ�������Դ�����Ĭ��3��.
+- retry_interval(Optional): ���Լ����Ĭ��1s.
+- abandon_mode(Optional): Ĭ��Ϊfalse�����ó�true��������retry_time�����������ݰ�������Ὣ�쳣���͸�fluentd������fluentd�����Ի������ԣ�����������ܻᵼ�������ظ�.
 
 ## �ٷ���վ
 ---

data/README.md

@@ -20,30 +20,10 @@ To get started using this plugin, you will need these things:
 
 ### Install the Plugin
 
-install the project from gem or github:
+install the plugin from gem:
 
 ```
 $ gem install fluent-plugin-aliyun-odps
-$ git clone https://github.com/aliyun/aliyun-odps-fluentd-plugin.git
-```
-
-Use gem to install dependency:
-
-```
-$ gem install protobuf
-$ gem install fluentd --no-ri --no-rdoc
-```
-
-Your plugin is in aliyun-odps-fluentd-plugin/lib/fluent/plugin, entry file is out_odps.rb.
-
-### Use the Plugin
-
-- If you installed this plugin from gem, please ignore this step.
-- Move the plugin dir into the plugin directory of Fluentd.
-- (i.e., copy the folder aliyun-odps-fluentd-plugin/lib/fluent/plugin into {YOUR_FLUENTD_DIRECTORY}/lib/fluent/plugin).
-
-```
-$ cp aliyun-odps-fluentd-plugin/lib/fluent/plugin/* {YOUR_FLUENTD_DIRECTORY}/lib/fluent/plugin/ -r
 ```
 
 ### ODPS Fluentd plugin now is available. Following is a simple example of how to write ODPS output configuration.
@@ -70,12 +50,16 @@ $ cp aliyun-odps-fluentd-plugin/lib/fluent/plugin/* {YOUR_FLUENTD_DIRECTORY}/lib
   buffer_queue_limit 128
   flush_interval 5s
   project your_projectName
+  enable_fast_crc true
   <table in.log>
 	table your_tableName
 	fields remote,method,path,code,size,agent
 	partition ctime=${datetime.strftime('%Y%m%d')}
 	time_format %d/%b/%Y:%H:%M:%S %z
 	shard_number 1
+    retry_time 3
+    retry_interval 1
+    abandon_mode true
   </table>
 </match>
 ```
@@ -101,6 +85,9 @@ $ cp aliyun-odps-fluentd-plugin/lib/fluent/plugin/* {YOUR_FLUENTD_DIRECTORY}/lib
     - if you are using the key words to set your <partition> and the key word is in time format, please set the param <time_format>. example: source[datetime] = "29/Aug/2015:11:10:16 +0800", and the param <time_format> is "%d/%b/%Y:%H:%M:%S %z"
 - shard_number(Optional): will write data to shards between [0,shard_number-1], this config must more than 0 and less than the max shard number of your table.
 - enable_fast_crc(Optional): use fast crc.so to calculate crc, this will improve speed up a lot, but this is not supported in some os.
+- retry_time(Optional): retry times when exception happens for each pack, default 3.
+- retry_interval(Optional): interval for retry, default 1s.
+- abandon_mode(Optional): default false. Setting this to true will abandon pack data after @retry_time, otherwise will raise a exception to fluentd and use fluentd's retry, but this may cause duplicated data.
 
 ## Useful Links
 ---

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.1.1

data/lib/fluent/plugin/out_aliyun_odps.rb

@@ -134,7 +134,7 @@ module Fluent
                         partition_name+=","+partition_column+"="+partition_value
                       end
                     else
-                      raise "partition has no corresponding source key or the partition expression is wrong,"+data.data.to_s
+                      raise "partition has no corresponding source key or the partition expression is wrong,"+data.to_s
                     end
                   elsif p.include? "=${"
                     key=p[p.index("{")+1, p.index("}")-1-p.index("{")]
@@ -190,29 +190,30 @@ module Fluent
                       restCount = v.size%@thread_number
                     end
                     @client.createStreamArrayWriter().write(v[sendCount*threadId..sendCount*(threadId+1)+restCount-1], k)
-                    @log.info "Successfully  import "+(sendCount+restCount).to_s+" data to partition:"+k+",table:"+@table+" at threadId:"+threadId.to_s
+                    @log.info "Successfully import "+(sendCount+restCount).to_s+" data to partition:"+k+",table:"+@table+" at threadId:"+threadId.to_s
                   rescue => e
+                    @log.warn "Fail to write, error at threadId:"+threadId.to_s+" Msg:"+e.message + " partitions:" + k.to_s + " table:" + @table
                     # reload shard
                     if e.message.include? "ShardNotReady" or e.message.include? "InvalidShardId"
-                      @log.warn "write failed, msg:" + e.message + ", reload shard."
+                      @log.warn "Reload shard."
                       @client.loadShard(@shard_number)
                       @client.waitForShardLoad
                     elsif e.message.include? "NoSuchPartition"
                       begin
                         @client.addPartition(k)
-                        @log.info "add partition "+ k
+                        @log.info "Add partition "+ k + " table:" + @table
                       rescue => ex
-                        @log.error "add partition failed"+ ex.message
+                        @log.error "Add partition failed"+ ex.message + " partitions:" + k.to_s + " table:" + @table
                       end
                     end
                     if retryTime > 0
-                      @log.warn "Fail to write, retry in " + @retry_interval.to_s + "sec. Error at threadId:"+threadId.to_s+" Msg:"+e.message
+                      @log.info "Retry in " + @retry_interval.to_s + "sec. Partitions:" + k.to_s + " table:" + @table
                       sleep(@retry_interval)
                       retryTime -= 1
                       retry
                     else
                       if (@abandon_mode)
-                        @log.error "Retry failed, abandon this pack. Msg:" + e.message
+                        @log.error "Retry failed, abandon this pack. Msg:" + e.message + " partitions:" + k.to_s + " table:" + @table
                       else
                         raise e
                       end
@@ -236,20 +237,21 @@ module Fluent
                   @client.createStreamArrayWriter().write(records[sendCount*threadId..sendCount*(threadId+1)+restCount-1])
                   @log.info "Successfully import "+(sendCount+restCount).to_s+" data to table:"+@table+" at threadId:"+threadId.to_s
                 rescue => e
+                  @log.warn "Fail to write, error at threadId:"+threadId.to_s+" Msg:"+e.message + " table:" + @table
                   # reload shard
                   if e.message.include? "ShardNotReady" or e.message.include? "InvalidShardId"
-                    @log.warn "write failed, msg:" + e.message + ", reload shard."
+                    @log.warn "Reload shard."
                     @client.loadShard(@shard_number)
                     @client.waitForShardLoad
                   end
                   if retryTime > 0
-                    @log.warn "Fail to write, retry in " + @retry_interval.to_s + "sec. Error at threadId:"+threadId.to_s+" Msg:"+e.message
+                    @log.info "Retry in " + @retry_interval.to_s + "sec. Table:" + @table
                     sleep(@retry_interval)
                     retryTime -= 1
                     retry
                   else
                     if (@abandon_mode)
-                      @log.error "Retry failed, abandon this pack. Msg:" + e.message
+                      @log.error "Retry failed, abandon this pack. Msg:" + e.message + " Table:" + @table
                     else
                       raise e
                     end

data/lib/fluent/plugin/serialize/serializer.rb

@@ -62,7 +62,7 @@ module OdpsDatahub
 
     def encodeString(value)
       value_to_encode = value.dup
-      #value_to_encode.encode!(::Protobuf::Field::StringField::ENCODING, :invalid => :replace, :undef => :replace, :replace => "")
+      value_to_encode.encode!(::Protobuf::Field::StringField::ENCODING, :invalid => :replace, :undef => :replace, :replace => "")
       value_to_encode.force_encoding(::Protobuf::Field::BytesField::BYTES_ENCODING)
       string_bytes = ::Protobuf::Field::VarintField.encode(value_to_encode.size)
       string_bytes << value_to_encode

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-aliyun-odps
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Xiao Dong
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-06 00:00:00.000000000 Z
+date: 2016-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd