fluent-plugin-aliyun-odps 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a83605973fae13e78ea73e422533ef6d613fe5fc
-  data.tar.gz: 8109f1fc93a0fce0869914893d073d25c911bf6d
+  metadata.gz: 33c05a1a6b13ea410537698e8bd6ee0ded655fae
+  data.tar.gz: 90f5de535dc2d9d48d65442e4f84bebc3d0870fb
 SHA512:
-  metadata.gz: c614f06a4063a1b78bb728ec48d0530a05b1f5951bd7c49932df8fb642fe300f979ca2acba507988bb5bee4b647a91c4557e51c1d181e257600b76daef0ad205
-  data.tar.gz: 6a0c6f9027fde9a0ebd23445d89b522b1d5f4ea5744875e94d20a2bd8d1b37c210bcf1d84f139579f7b151310022258d58f52d1b26d56ae851ef860955b0fe94
+  metadata.gz: cbc2cf4a5cef08a7f7235972f8bdb4e99a24ffbf4b1d78dd5912683d6eddde45c5e363186c5317085f2965a783c608dc24dba6f1f7b6ace6abf997642566f387
+  data.tar.gz: 606b07286d5b0ab22dbc01aa7619b9718257b1083e985b295f1ac1be6cc280727ab1edfb4dbae30bce86d5575fc9530f2a6c109a3f49ad821431c51cf1e08095

data/CHANGELOG.md

@@ -1,4 +1,6 @@
 0.0.4
 Fix datetime format bug, support String, DateTime, Time type when write to a datetime field.
 0.0.5
-Add reload shard when import fails, and remove unload shard operation when shut down.
+Add reload shard when import fails, and remove unload shard operation when shut down.
+0.0.6
+Add decimal support��fix string input while setting double and int.

data/VERSION

@@ -1 +1 @@
-0.0.5
+0.0.6

data/lib/fluent/plugin/odps/odps_table.rb

@@ -16,6 +16,7 @@
 #specific language governing permissions and limitations
 #under the License.
 #
+
 require 'rexml/document'
 require_relative 'xml_template'
 require_relative 'odps_table_schema'
@@ -56,22 +57,34 @@ module OdpsDatahub
     def setBigInt(idx, value)
       if value.is_a?Integer
         setValue(idx, value)
+      elsif value.is_a?String
+        setValue(idx, value.to_i)
       else
-        raise "value show be Integer"
+        raise "value show be Integer, idx:" + idx.to_s + " value:" + value.to_s
       end
     end
 
     def setDouble(idx, value)
       if value.is_a?Float
         setValue(idx, value)
+      elsif value.is_a?String
+        setValue(idx, value.to_f)
       else
-        raise "value show be Float"
+        raise "value show be Float, idx:" + idx.to_s + " value:" + value.to_s
       end
     end
 
     def setBoolean(idx, value)
-      if value != false and value != true
-        raise "value must be bool"
+      if value.is_a?String
+        if value == "true"
+          setValue(idx, true)
+        elsif value == "false"
+          setValue(idx, false)
+        else
+          raise "value must be true or false, idx:" + idx.to_s + " value:" + value.to_s
+        end
+      elsif value != false and value != true
+        raise "value must be bool or string[true,false], idx:" + idx.to_s + " value:" + value.to_s
       end
       setValue(idx, value)
     end
@@ -93,7 +106,19 @@ module OdpsDatahub
           raise "Parse string to datetime failed, string:" + value
         end
       else
-        raise "DateTime cell should be in Integer or Time or DateTime format."
+        raise "DateTime cell should be in Integer or Time or DateTime format, idx:" + idx.to_s + " value:" + value.to_s
+      end
+    end
+
+    def setDecimal(idx, value)
+      if value.is_a?String
+        setValue(idx, value)
+      elsif value.is_a?Float
+          setValue(idx, value.to_s)
+      elsif value.is_a?BigDecimal
+        setValue(idx, value.to_s)
+      else
+        raise "value can not be convert to decimal, idx:" + idx.to_s + " value:" + value.to_s
       end
     end
 
@@ -101,14 +126,14 @@ module OdpsDatahub
       if value.is_a?String and value.length < $STRING_MAX_LENTH
         setValue(idx, value)
       else
-        raise "value show be String and len < " + $STRING_MAX_LENTH.to_s
+        raise "value show be String and len < " + $STRING_MAX_LENTH.to_s + ", idx:" + idx.to_s + " value:" + value.to_s
       end
     end
 
     private
     def setValue(idx, value)
       if idx < 0 or idx >= @mSchema.getColumnCount
-        raise "idx out of range"
+        raise "idx out of range, idx:" + idx.to_s + " value:" + value.to_s
       end
       @mValues[idx] = value
     end

data/lib/fluent/plugin/odps/odps_table_schema.rb

@@ -17,11 +17,12 @@
 #under the License.
 #
 module OdpsDatahub
-  $ODPS_BIGINT = "bigint"
-  $ODPS_DOUBLE = "double"
-  $ODPS_BOOLEAN = "boolean"
-  $ODPS_DATETIME = "datetime"
-  $ODPS_STRING = "string"
+  $ODPS_BIGINT = "bigint"	      #8�ֽ��з�������
+  $ODPS_DOUBLE = "double"	      #8�ֽ�˫���ȸ�����
+  $ODPS_BOOLEAN = "boolean"     #������
+  $ODPS_DATETIME = "datetime"	  #�������ͣ�ȡֵ��Χ��0001-01-01 00:00:00 ~ 9999-12-31 23:59:59
+  $ODPS_STRING = "string"	    #�ַ�������
+  $ODPS_DECIMAL = "decimal"	    #�ַ�������
 
   class OdpsTableColumn
     attr_reader :mName, :mType, :mIdx

data/lib/fluent/plugin/out_aliyun_odps.rb

@@ -249,6 +249,7 @@ module Fluent
         rescue => e
           # reload shard
           if e.message.include? "ShardNotReady" or e.message.include? "InvalidShardId"
+            @log.warn "write failed, msg:" + e.message
             @client.loadShard(@shard_number)
             @client.waitForShardLoad
           end

data/lib/fluent/plugin/serialize/serializer.rb

@@ -120,6 +120,10 @@ module OdpsDatahub
                 crc32cRecord.update(cellValue)
                 writeTag(col.mIdx + 1, ::Protobuf::WireType::LENGTH_DELIMITED, upStream)
                 upStream.write(encodeString(cellValue))
+              when $ODPS_DECIMAL
+                crc32cRecord.update(cellValue)
+                writeTag(col.mIdx + 1, ::Protobuf::WireType::LENGTH_DELIMITED, upStream)
+                upStream.write(encodeString(cellValue))
               else
                 raise OdpsDatahubException.new($INVALID_ARGUMENT, "invalid mType")
             end

data/lib/fluent/plugin/stream_writer.rb

@@ -141,6 +141,8 @@ module OdpsDatahub
             record.setDouble(i, value[i])
           when $ODPS_STRING
             record.setString(i, value[i])
+          when $ODPS_DECIMAL
+            record.setDecimal(i, value[i])
           else
             raise OdpsDatahubException.new($INVALID_ARGUMENT, "unsupported schema type")
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-aliyun-odps
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Xiao Dong
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-22 00:00:00.000000000 Z
+date: 2015-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd