fluent-plugin-aliyun-odps 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3557dd3c4653efb184fb04200769c5a9f53a6947
-  data.tar.gz: 6e35787b0de40f61db279b5d640209e8d800d5e9
+  metadata.gz: 06cc7b0378693eab79c346e913747092b1ef84af
+  data.tar.gz: 8fe96f58093519df5ddf21e1cf4f8b1aaa71f0de
 SHA512:
-  metadata.gz: 7e37bd48d24117a0968f3fd6646b3ce8c64f133138410cbebcae287fe8ccfb6b4a8d1ebe57dd140e8c25fe1a4d9a430ef4028647508cd1514ab4283d29a6a1ad
-  data.tar.gz: 8112131d1bb7e0c09b31f98af58e2b28370107a5e715b5f1c95f5e5811cac1bafb3b692cbb9f2367ba2105db4bdfcc7be193996b2f0c93be723b1b190a2190b5
+  metadata.gz: 6b2306b8bcc789d32b3bceb6640c5ebb8dfa23524df89020026c4693415ac0d445abf0cefee8f2c28f1595853549868eb92ba85ba78988d1f0b4148b0c6336d1
+  data.tar.gz: 835ef6d097a2599dc1839d7db5957b38755b09416020370c609c99c18fa51560e407a116becfcdf287d6ee472baccdd958ee90454342832781edbadcb75d5fd2

data/CHANGELOG.md

@@ -19,4 +19,8 @@ Use XStreamPack.
 0.1.3
 Drop record with error log when parse partition failed.
 0.1.5
-Fix string encode replace unknow char
+Fix string encode replace unknow char
+0.1.6
+Fix raise exception bug
+0.1.7
+Add data_encoding config to format data.

data/README.cn.md

@@ -127,6 +127,7 @@ select * from students;
   flush_interval 5s
   project your_projectName
   enable_fast_crc true
+  data_encoding UTF-8
   <table nginx.access>
   table nginx_logs #对应日志写入的odps表
   fields remote,method,path,code,size,agent,requesttime
@@ -170,6 +171,7 @@ select * from students;
 - retry_time(Optional): 发送每个pack数据时内置重试次数，默认3次.
 - retry_interval(Optional): 重试间隔，默认1s.
 - abandon_mode(Optional): 默认为false，设置成true会在重试retry_time后抛弃该数据包，否则会将异常抛送给fluentd，利用fluentd的重试机制重试，这种情况可能会导致数据重复.
+- data_encoding(Optional): 默认使用源数据标示的encode方式，根据string.encoding获取，如果需要指定源数据编码方式，请设置该值，支持的类型："US-ASCII", "ASCII-8BIT", "UTF-8", "ISO-8859-1", "Shift_JIS", "EUC-JP", "Windows-31J", "BINARY", "CP932", "eucJP"
 
 ## 常见使用问题以及异常描述
 ---
@@ -193,7 +195,9 @@ select * from students;
  - 这个错误信息抛出代表解析partition过程出现问题，请检查partition配置，如果数据中存在脏数据也可能遇到这个问题。
 * 如何更改为淘宝源RubyGem？
  - RubyGems 镜像[https://ruby.taobao.org/]
-
+* 导入数据抛异常"\xE8" from ASCII-8BIT to UTF-8 (Encoding::UndefinedConversionError)
+ - 该错误往往由于source插件在读取数据时，数据真实编码为utf-8,但是transport给fluend的string.encoding却为ASCII-8BIT导致类似错误，这种情况需要设置data_encoding来进行转码。
+ 
  ## 官方网站
  - [Fluentd User Guide](http://docs.fluentd.org/)
 

data/VERSION

@@ -1 +1 @@
-0.1.6
+0.1.7

data/lib/fluent/plugin/conf/config.rb

@@ -18,6 +18,7 @@
 #
 module OdpsDatahub
   $USE_FAST_CRC = false
+  $DATA_ENCODE = nil
   class OdpsConfig
     attr_accessor :accessId, :accessKey, :odpsEndpoint, :datahubEndpoint, :defaultProjectName, :userAgent
 
@@ -33,5 +34,13 @@ module OdpsDatahub
     def self.setFastCrc(value)
       $USE_FAST_CRC = value
     end
+
+    def self.setEncode(value)
+      if ["US-ASCII", "ASCII-8BIT", "UTF-8", "ISO-8859-1", "Shift_JIS", "EUC-JP", "Windows-31J", "BINARY", "CP932", "eucJP"].include?(value)
+        $DATA_ENCODE = value
+      else
+        raise "Unsupported encoding type."
+      end
+    end
   end
 end

data/lib/fluent/plugin/crc/crc.rb

@@ -46,7 +46,7 @@ module OdpsDatahub
         when /solaris|bsd/
           "unix"
         else
-          raise Error::WebDriverError, "unspport os"
+          raise "unspport os"
       end
     end
   end

data/lib/fluent/plugin/out_aliyun_odps.rb

@@ -36,6 +36,7 @@ module Fluent
     config_param :project, :string, :default => nil
     config_param :format, :string, :default => 'out_file'
     config_param :enable_fast_crc, :bool, :default => false
+    config_param :data_encoding, :string, :default => nil
 
     attr_accessor :tables
 
@@ -328,6 +329,9 @@ module Fluent
           raise e.to_s
         end
       end
+      if (@data_encoding != nil)
+        OdpsDatahub::OdpsConfig::setEncode(@data_encoding)
+      end
       #初始化各个table object
       @tables.each { |te|
         te.init(config)

data/lib/fluent/plugin/serialize/serializer.rb

@@ -61,7 +61,6 @@ module OdpsDatahub
     end
 
     def encodeString(value)
-      value.encode!(::Protobuf::Field::StringField::ENCODING, :invalid => :replace, :undef => :replace, :replace => "")
       value.force_encoding(::Protobuf::Field::BytesField::BYTES_ENCODING)
       string_bytes = ::Protobuf::Field::VarintField.encode(value.size)
       string_bytes << value
@@ -116,7 +115,11 @@ module OdpsDatahub
                 writeTag(col.mIdx + 1, ::Protobuf::WireType::VARINT, upStream)
                 upStream.write(encodeDataTime(cellValue))
               when $ODPS_STRING
-                cellValue = cellValue.encode('utf-8')
+                if $DATA_ENCODE != nil
+                  cellValue.encode!(::Protobuf::Field::StringField::ENCODING, $DATA_ENCODE)
+                else
+                  cellValue.encode!(::Protobuf::Field::StringField::ENCODING)
+                end
                 encode_str = encodeString(cellValue)
                 crc32cRecord.write(cellValue)
                 writeTag(col.mIdx + 1, ::Protobuf::WireType::LENGTH_DELIMITED, upStream)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-aliyun-odps
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Xiao Dong
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-26 00:00:00.000000000 Z
+date: 2016-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd