fluent-plugin-aliyun-odps 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 33c05a1a6b13ea410537698e8bd6ee0ded655fae
-  data.tar.gz: 90f5de535dc2d9d48d65442e4f84bebc3d0870fb
+  metadata.gz: 1e49e64231330fc82748d84415440558c8503b30
+  data.tar.gz: e328545513c76cf9f90cded935e7548037ff83ef
 SHA512:
-  metadata.gz: cbc2cf4a5cef08a7f7235972f8bdb4e99a24ffbf4b1d78dd5912683d6eddde45c5e363186c5317085f2965a783c608dc24dba6f1f7b6ace6abf997642566f387
-  data.tar.gz: 606b07286d5b0ab22dbc01aa7619b9718257b1083e985b295f1ac1be6cc280727ab1edfb4dbae30bce86d5575fc9530f2a6c109a3f49ad821431c51cf1e08095
+  metadata.gz: a032d3ea1acdfd27f9f002a1d9eacb5a2b516c18f042b9a8f6aa4d6235220bf7e8c2da9aa7bc321a628d96dee364fbbdbef48ec06a97c404b208ea02b66e3ab4
+  data.tar.gz: 5bdc7642e0a3d1d58dcdc0bc6afc1288d152b21703f79db3a66edb3efdd156637481340f280d58d977be596b68fefe09b4773a78355f3827c881c37f19cda484

data/CHANGELOG.md

@@ -3,4 +3,6 @@ Fix datetime format bug, support String, DateTime, Time type when write to a dat
 0.0.5
 Add reload shard when import fails, and remove unload shard operation when shut down.
 0.0.6
-Add decimal support��fix string input while setting double and int.
+Add decimal support��fix string input while setting double and int.
+0.0.7
+Add error msg when add partition fail, support fast crc�� remove pack size limit.

data/README.md

@@ -98,6 +98,7 @@ $ cp aliyun-odps-fluentd-plugin/lib/fluent/plugin/* {YOUR_FLUENTD_DIRECTORY}/lib
 - time_format(Optional):
     - if you are using the key words to set your <partition> and the key word is in time format, please set the param <time_format>. example: source[datetime] = "29/Aug/2015:11:10:16 +0800", and the param <time_format> is "%d/%b/%Y:%H:%M:%S %z"
 - shard_number(Optional): will write data to shards between [0,shard_number-1], this config must more than 0 and less than the max shard number of your table.
+- enable_fast_crc(Optional): use fast crc.so to calculate crc, this will improve speed up a lot, but this is not supported in some os.
 
 ## Useful Links
 ---

data/VERSION

@@ -1 +1 @@
-0.0.6
+0.0.7

data/lib/fluent/plugin/conf/config.rb

@@ -17,6 +17,7 @@
 #under the License.
 #
 module OdpsDatahub
+  $USE_FAST_CRC = false
   class OdpsConfig
     attr_accessor :accessId, :accessKey, :odpsEndpoint, :datahubEndpoint, :defaultProjectName, :userAgent
 
@@ -28,5 +29,9 @@ module OdpsDatahub
       @defaultProject = defaultProjectName
       @userAgent = ""
     end
+
+    def self.setFastCrc(value)
+      $USE_FAST_CRC = value
+    end
   end
 end

data/lib/fluent/plugin/crc/crc.rb

@@ -0,0 +1,53 @@
+#
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing,
+#software distributed under the License is distributed on an
+#"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#KIND, either express or implied.  See the License for the
+#specific language governing permissions and limitations
+#under the License.
+#
+require 'rbconfig'
+module OdpsDatahub
+  class CrcCalculator
+    # @param [StringIO] data
+    # @return crc32c to_i
+    def self.calculate(data)
+      if (!$USE_FAST_CRC)
+        require_relative 'origin/crc32c'
+        crc32c = Digest::CRC32c.new
+        crc32c.update(data.string)
+        return crc32c.checksum.to_i
+      elsif getOsType == "linux" || getOsType == "unix"
+        require_relative 'lib/linux/crc32c.so'
+        return Crc32c.calculate(data.string, data.length, 0).to_i
+      elsif getOsType == "windows"
+        require_relative 'lib/win/crc32c.so'
+        return Crc32c.calculate(data.string, data.length, 0).to_i
+      end
+    end
+
+    def self.getOsType
+      host_os = RbConfig::CONFIG['host_os']
+      case host_os
+        when /mswin|msys|mingw|cygwin|bccwin|wince|emc/
+          "windows"
+        when /linux/
+          "linux"
+        when /solaris|bsd/
+          "unix"
+        else
+          raise Error::WebDriverError, "unspport os"
+      end
+    end
+  end
+end

data/lib/fluent/plugin/http/http_connection.rb

@@ -16,6 +16,7 @@
 #specific language governing permissions and limitations
 #under the License.
 #
+
 require 'net/http'
 require 'base64'
 require 'openssl'

data/lib/fluent/plugin/http/http_flag.rb

@@ -18,7 +18,7 @@
 #
 module OdpsDatahub
   $SDK_UA_STR = "ODPS Ruby SDK v0.1"
-  $MAX_PACK_SIZE = 2048*1024
+  $MAX_PACK_SIZE = 2048*10*1024
   class HttpHeaders
     $AUTHORIZATION = "Authorization"
     $CACHE_CONTROL = "Cache-Control"

data/lib/fluent/plugin/odps/odps_table.rb

@@ -225,7 +225,7 @@ module OdpsDatahub
 
       res = conn.getResponse
       if res.code != '200'
-        raise "Add partition failed with error" + res.code.to_s
+        raise "Add partition failed with error " + res.body
       end
 
       if res.to_hash['Content-Length'] != "0" and not res.body.to_s.include?"Instance"

data/lib/fluent/plugin/out_aliyun_odps.rb

@@ -35,6 +35,7 @@ module Fluent
     config_param :aliyun_odps_hub_endpoint, :string, :default => nil
     config_param :project, :string, :default => nil
     config_param :format, :string, :default => 'out_file'
+    config_param :enable_fast_crc, :bool, :default => false
 
     attr_accessor :tables
 
@@ -299,6 +300,10 @@ module Fluent
           :aliyun_odps_endpoint => @aliyun_odps_endpoint,
           :aliyun_odps_hub_endpoint => @aliyun_odps_hub_endpoint,
       }
+      #init Global setting
+      if (@enable_fast_crc)
+        OdpsDatahub::OdpsConfig::setFastCrc(true)
+      end
       #初始化各个table object
       @tables.each { |te|
         te.init(config)

data/lib/fluent/plugin/serialize/deserializer.rb

@@ -18,7 +18,7 @@
 #
 require 'stringio'
 require 'protobuf'
-require_relative '../digest/crc32c'
+require_relative '../crc/origin/crc32c'
 require_relative '../odps/odps_table'
 
 module OdpsDatahub

data/lib/fluent/plugin/serialize/serializer.rb

@@ -19,7 +19,7 @@
 require 'stringio'
 require 'protobuf'
 require_relative '../exceptions'
-require_relative '../digest/crc32c'
+require_relative '../crc/crc'
 require_relative '../odps/odps_table'
 
 module OdpsDatahub
@@ -87,10 +87,10 @@ module OdpsDatahub
     end
 
     def serialize(upStream, recordList)
-      crc32cPack = ::Digest::CRC32c.new
+      crc32cPack = StringIO.new
       if recordList.is_a?Array
         recordList.each { |record|
-          crc32cRecord = ::Digest::CRC32c.new
+          crc32cRecord = StringIO.new
           schema = OdpsTableSchema.new
           schema = record.getTableSchema
           schema.mCols.each { | col |
@@ -98,45 +98,45 @@ module OdpsDatahub
             if cellValue == nil
               next
             end
-            crc32cRecord.update(encodeFixed32(col.mIdx + 1))
+            crc32cRecord.write(encodeFixed32(col.mIdx + 1))
             case col.mType
               when $ODPS_BIGINT
-                crc32cRecord.update(encodeFixed64(cellValue))
+                crc32cRecord.write(encodeFixed64(cellValue))
                 writeTag(col.mIdx + 1, ::Protobuf::WireType::VARINT, upStream)
                 upStream.write(encodeSInt64(cellValue))
               when $ODPS_DOUBLE
-                crc32cRecord.update(encodeDouble(cellValue))
+                crc32cRecord.write(encodeDouble(cellValue))
                 writeTag(col.mIdx + 1, ::Protobuf::WireType::FIXED64, upStream)
                 upStream.write(encodeDouble(cellValue))
               when $ODPS_BOOLEAN
-                crc32cRecord.update(encodeBool(cellValue))
+                crc32cRecord.write(encodeBool(cellValue))
                 writeTag(col.mIdx + 1, ::Protobuf::WireType::VARINT, upStream)
                 upStream.write(encodeBool(cellValue))
               when $ODPS_DATETIME
-                crc32cRecord.update(encodeFixed64(cellValue))
+                crc32cRecord.write(encodeFixed64(cellValue))
                 writeTag(col.mIdx + 1, ::Protobuf::WireType::VARINT, upStream)
                 upStream.write(encodeDataTime(cellValue))
               when $ODPS_STRING
-                crc32cRecord.update(cellValue)
+                crc32cRecord.write(cellValue)
                 writeTag(col.mIdx + 1, ::Protobuf::WireType::LENGTH_DELIMITED, upStream)
                 upStream.write(encodeString(cellValue))
               when $ODPS_DECIMAL
-                crc32cRecord.update(cellValue)
+                crc32cRecord.write(cellValue)
                 writeTag(col.mIdx + 1, ::Protobuf::WireType::LENGTH_DELIMITED, upStream)
                 upStream.write(encodeString(cellValue))
               else
                 raise OdpsDatahubException.new($INVALID_ARGUMENT, "invalid mType")
             end
           }
-          recordCrc = crc32cRecord.checksum.to_i
+          recordCrc = CrcCalculator::calculate(crc32cRecord)
           writeTag($TUNNEL_END_RECORD, ::Protobuf::WireType::VARINT, upStream)
           upStream.write(encodeUInt32(recordCrc))
-          crc32cPack.update(encodeFixed32(recordCrc))
+          crc32cPack.write(encodeFixed32(recordCrc))
         }
         writeTag($TUNNEL_META_COUNT, ::Protobuf::WireType::VARINT, upStream)
         upStream.write(encodeSInt64(recordList.size))
         writeTag($TUNNEL_META_CHECKSUM, ::Protobuf::WireType::VARINT, upStream)
-        upStream.write(encodeUInt32(crc32cPack.checksum))
+        upStream.write(encodeUInt32(CrcCalculator::calculate(crc32cPack)))
       else
         raise OdpsDatahubException.new($INVALID_ARGUMENT, "param must be a array")
       end

data/lib/fluent/plugin/stream_writer.rb

@@ -93,7 +93,7 @@ module OdpsDatahub
       upStream = Zlib::Deflate.deflate(@mUpStream.string)
       header[$CONTENT_MD5] = Digest::MD5.hexdigest(upStream)
       header[$CONTENT_LENGTH] = upStream.length.to_s
-      #MAX_LENGTH 2048KB
+      #MAX_LENGTH 2048*10KB
       if upStream.length > $MAX_PACK_SIZE
         raise OdpsDatahubException.new($PACK_SIZE_EXCEED, "pack size:" + upStream.length.to_s)
       end
@@ -109,6 +109,7 @@ module OdpsDatahub
       if res.code != "200"
         raise OdpsDatahubException.new(json_obj["Code"], "write failed because " + json_obj["Message"])
       end
+      return json_obj
     end
 
     private

data/odps_example.conf

@@ -21,6 +21,7 @@
   buffer_queue_limit 128
   flush_interval 5s
   project your_projectName
+  enable_fast_crc false
   <table in.log>
 	table your_tableName
 	fields r1,r2,r3,r4,r5,r6,blue

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-aliyun-odps
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Xiao Dong
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-02 00:00:00.000000000 Z
+date: 2015-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -125,15 +125,17 @@ files:
 - CHANGELOG.md
 - Gemfile
 - License
-- README.cn.md
 - README.md
 - Rakefile
 - VERSION
 - fluent-plugin-aliyun-odps.gemspec
 - lib/fluent/plugin/conf/config.rb
-- lib/fluent/plugin/digest/crc.rb
-- lib/fluent/plugin/digest/crc32.rb
-- lib/fluent/plugin/digest/crc32c.rb
+- lib/fluent/plugin/crc/crc.rb
+- lib/fluent/plugin/crc/lib/linux/crc32c.so
+- lib/fluent/plugin/crc/lib/win/crc32c.so
+- lib/fluent/plugin/crc/origin/crc.rb
+- lib/fluent/plugin/crc/origin/crc32.rb
+- lib/fluent/plugin/crc/origin/crc32c.rb
 - lib/fluent/plugin/exceptions.rb
 - lib/fluent/plugin/http/http_connection.rb
 - lib/fluent/plugin/http/http_flag.rb

data/README.cn.md

@@ -1,112 +0,0 @@
-# Aliyun ODPS Plugin for Fluentd
-
-## ��ʼʹ��
----
-
-### ����
-
-- �������ݴ�������(Open Data Processing Service�����ODPS)�ǰ���Ͱ������з��ĺ������ݴ���ƽ̨����Ҫ�����������ṹ�����ݵĴ洢�ͼ��㣬�����ṩ�������ݲֿ�Ľ�������Լ���Դ����ݵķ�����ģ����
-- ODPS DataHub Service(DHS)��һ��ODPS���ڽ��������û��ṩʵʱ���ݵķ���(Publish)�Ͷ���(Subscribe)�Ĺ��ܡ�
-
-
-### ����Ҫ��
-
-ʹ�ô˲������Ҫ�߱����»���:
-
-1. Ruby 2.1.0 �����
-2. Gem 2.4.5 �����
-3. Fluentd-0.10.49 ����� (*[Home Page](http://www.fluentd.org/)*)
-4. Protobuf-3.5.1 �����(Ruby protobuf)
-
-### GEM��װ
-
-��ruby gem��װʹ��:
-
-```
-$ gem install fluent-plugin-aliyun-odps
-```
-
-### Դ�밲װ���
-
-* ���Ȱ�װ����������fluentd�Լ�protobuf��
-* Ȼ�󽫲��Ŀ¼������Fluentd���Ŀ¼�£����磺 ��aliyun-odps-fluentd-plugin/lib/fluent/pluginĿ¼���Ƶ�{YOUR_FLUENTD_DIRECTORY}/lib/fluent/plugin�С�
-
-```
-$ gem install protobuf
-$ gem install fluentd --no-ri --no-rdoc
-$ git clone https://github.com/aliyun/aliyun-odps-fluentd-plugin.git
-$ cp aliyun-odps-fluentd-plugin/lib/fluent/plugin/* {YOUR_FLUENTD_DIRECTORY}/lib/fluent/plugin/ -r
-```
-
-### ���ʹ��ʾ��
-
-```
-<source>
-   type tail
-   path /opt/log/in/in.log
-   pos_file /opt/log/in/in.log.pos
-   refresh_interval 5s
-   tag in.log
-   format /^(?<remote>[^ ]*) - - \[(?<datetime>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*) "-" "(?<agent>[^\"]*)"$/
-   time_format %Y%b%d %H:%M:%S %z
-</source>
-```
-```
-<match in.**>
-  type aliyun_odps
-  aliyun_access_id ************
-  aliyun_access_key *********
-  aliyun_odps_endpoint http://service.odps.aliyun.com/api
-  aliyun_odps_hub_endpoint http://dh.odps.aliyun.com
-  buffer_chunk_limit 2m
-  buffer_queue_limit 128
-  flush_interval 5s
-  project your_projectName
-  <table in.log>
-	table your_tableName
-	fields remote,method,path,code,size,agent
-	partition ctime=${datetime.strftime('%Y%m%d')}
-	time_format %d/%b/%Y:%H:%M:%S %z
-	shard_number 1
-  </table>
-</match>
-```
-### ����˵��
-
-- type(Fixed): �̶�ֵ aliyun_odps.
-- aliyun_access_id(Required):������access_id.
-- aliyun_access_key(Required):������access key.
-- aliyun_odps_hub_endpoint(Required):�����ķ�������ESC�ϣ���ѱ�ֵ�趨Ϊ http://dh-ext.odps.aliyun-inc.com, ��������Ϊ http://dh.odps.aliyun.com.
-- aliyunodps_endpoint(Required):�����ķ�������ESC�ϣ���ѱ�ֵ�趨Ϊ http://odps-ext.aiyun-inc.com/api, ��������Ϊ http://service.odps.aliyun.com/api .
-- buffer_chunk_limit(Optional): ���С��֧�֡�k��(KB),��m��(MB),��g��(GB)��λ��Ĭ�� 8MB������ֵ2MB.
-- buffer_queue_limit(Optional): ����д�С����ֵ��buffer_chunk_limit��ͬ����������������С��
-- flush_interval(Optional): ǿ�Ʒ��ͼ�����ﵽʱ��������δ����ǿ�Ʒ���, Ĭ�� 60s.
-- project(Required): project����.
-- table(Required): table����.
-- fields(Required): ��source��Ӧ���ֶ������������source֮��.
-- partition(Optional)����Ϊ�������������ô���.
-    - ������֧�ֵ�����ģʽ:
-        - �̶�ֵ: partition ctime=20150804
-        - �ؼ���: partition ctime=${remote} ������remoteΪsource��ij�ֶΣ�
-        - ʱ���ʽ�ؼ���: partition ctime=${datetime.strftime('%Y%m%d')} ������datetimeΪsource��ijʱ���ʽ�ֶΣ����Ϊ%Y%m%d��ʽ��Ϊ�������ƣ�
-- time_format(Optional):
-    - ���ʹ��ʱ���ʽ�ؼ���Ϊ<partition>, �����ñ�����. ����: source[datetime]="29/Aug/2015:11:10:16 +0800",������<time_format>Ϊ"%d/%b/%Y:%H:%M:%S %z"
-- shard_number(Optional):ָ��shard���������������shard[0,shard_number-1]��Χ�ڵ�shard��д�����ݣ�����Ϊ����0��С��table��Ӧshard�������޵�����.
-
-## �ٷ���վ
----
-
-- [Fluentd User Guide](http://docs.fluentd.org/)
-
-## ����
----
-
-- [Sun Zongtao]()
-- [Cai Ying]()
-- [Dong Xiao](https://github.com/dongxiao1198)
-- [Yang Hongbo](https://github.com/hongbosoftware)
-
-## License
----
-
-licensed under the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0.html)