fluentd-plugin-aliyun-odps 0.0.1

data/lib/fluent/plugin/serialize/deserializer.rb

@@ -0,0 +1,32 @@
+#
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing,
+#software distributed under the License is distributed on an
+#"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#KIND, either express or implied.  See the License for the
+#specific language governing permissions and limitations
+#under the License.
+#
+require 'stringio'
+require 'protobuf'
+require_relative '../digest/crc32c'
+require_relative '../odps/odps_table'
+
+module OdpsDatahub
+  #TODO
+  class Deserializer
+
+    def deserialize( )
+    end
+
+  end
+end

data/lib/fluent/plugin/serialize/serializer.rb

@@ -0,0 +1,141 @@
+#
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing,
+#software distributed under the License is distributed on an
+#"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#KIND, either express or implied.  See the License for the
+#specific language governing permissions and limitations
+#under the License.
+#
+require 'stringio'
+require 'protobuf'
+require_relative '../exceptions'
+require_relative '../digest/crc32c'
+require_relative '../odps/odps_table'
+
+module OdpsDatahub
+
+  $TUNNEL_META_COUNT = 33554430 #  magic num 2^25-2
+  $TUNNEL_META_CHECKSUM = 33554431 #  magic num 2^25-1
+  $TUNNEL_END_RECORD = 33553408 #  maigc num 2^25-1024
+
+  class Serializer
+    def encodeBool(value)
+      [value ? 1 : 0].pack('C')
+    end
+
+    def encodeDouble(value)
+      [value].pack('E')
+    end
+
+    def encodeSInt64(value)
+      if value >= 0
+        ::Protobuf::Field::VarintField.encode(value << 1)
+      else
+        ::Protobuf::Field::VarintField.encode(~(value << 1))
+      end
+    end
+
+    def encodeUInt32(value)
+      return [value].pack('C') if value < 128
+      bytes = []
+      until value == 0
+        bytes << (0x80 | (value & 0x7f))
+        value >>= 7
+      end
+      bytes[-1] &= 0x7f
+      bytes.pack('C*')
+    end
+
+    def encodeDataTime(value)
+      self.encodeSInt64(value)
+    end
+
+    def encodeString(value)
+      value_to_encode = value.dup
+      value_to_encode.encode!(::Protobuf::Field::StringField::ENCODING, :invalid => :replace, :undef => :replace, :replace => "")
+      value_to_encode.force_encoding(::Protobuf::Field::BytesField::BYTES_ENCODING)
+      string_bytes = ::Protobuf::Field::VarintField.encode(value_to_encode.size)
+      string_bytes << value_to_encode
+    end
+
+    def encodeFixed64(value)
+      # we don't use 'Q' for pack/unpack. 'Q' is machine-dependent.
+      [value & 0xffff_ffff, value >> 32].pack('VV')
+    end
+
+    def encodeFixed32(value)
+      [value].pack('V')
+    end
+
+    def encodeFixedString(value)
+      [value].pack('V')
+    end
+
+    def writeTag(idx, type, stream)
+      key = (idx << 3) | type
+      stream << ::Protobuf::Field::VarintField.encode(key)
+    end
+
+    def serialize(upStream, recordList)
+      crc32cPack = ::Digest::CRC32c.new
+      if recordList.is_a?Array
+        recordList.each { |record|
+          crc32cRecord = ::Digest::CRC32c.new
+          schema = OdpsTableSchema.new
+          schema = record.getTableSchema
+          schema.mCols.each { | col |
+            cellValue = record.getValue(col.mIdx)
+            if cellValue == nil
+              next
+            end
+            crc32cRecord.update(encodeFixed32(col.mIdx + 1))
+            case col.mType
+              when $ODPS_BIGINT
+                crc32cRecord.update(encodeFixed64(cellValue))
+                writeTag(col.mIdx + 1, ::Protobuf::WireType::VARINT, upStream)
+                upStream.write(encodeSInt64(cellValue))
+              when $ODPS_DOUBLE
+                crc32cRecord.update(encodeDouble(cellValue))
+                writeTag(col.mIdx + 1, ::Protobuf::WireType::FIXED64, upStream)
+                upStream.write(encodeDouble(cellValue))
+              when $ODPS_BOOLEAN
+                crc32cRecord.update(encodeBool(cellValue))
+                writeTag(col.mIdx + 1, ::Protobuf::WireType::VARINT, upStream)
+                upStream.write(encodeBool(cellValue))
+              when $ODPS_DATETIME
+                crc32cRecord.update(encodeFixed64(cellValue))
+                writeTag(col.mIdx + 1, ::Protobuf::WireType::VARINT, upStream)
+                upStream.write(encodeDataTime(cellValue))
+              when $ODPS_STRING
+                crc32cRecord.update(cellValue)
+                writeTag(col.mIdx + 1, ::Protobuf::WireType::LENGTH_DELIMITED, upStream)
+                upStream.write(encodeString(cellValue))
+              else
+                raise OdpsDatahubException.new($INVALID_ARGUMENT, "invalid mType")
+            end
+          }
+          recordCrc = crc32cRecord.checksum.to_i
+          writeTag($TUNNEL_END_RECORD, ::Protobuf::WireType::VARINT, upStream)
+          upStream.write(encodeUInt32(recordCrc))
+          crc32cPack.update(encodeFixed32(recordCrc))
+        }
+        writeTag($TUNNEL_META_COUNT, ::Protobuf::WireType::VARINT, upStream)
+        upStream.write(encodeSInt64(recordList.size))
+        writeTag($TUNNEL_META_CHECKSUM, ::Protobuf::WireType::VARINT, upStream)
+        upStream.write(encodeUInt32(crc32cPack.checksum))
+      else
+        raise OdpsDatahubException.new($INVALID_ARGUMENT, "param must be a array")
+      end
+    end
+  end
+end

data/lib/fluent/plugin/stream_client.rb

@@ -0,0 +1,111 @@
+#
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing,
+#software distributed under the License is distributed on an
+#"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#KIND, either express or implied.  See the License for the
+#specific language governing permissions and limitations
+#under the License.
+#
+require 'json'
+require_relative 'exceptions'
+require_relative 'stream_writer'
+require_relative 'stream_reader'
+require_relative 'http/http_connection'
+require_relative 'conf/config'
+require_relative 'odps/odps_table'
+
+module OdpsDatahub
+  class StreamClient
+    attr_reader :mProject, :mTable, :mOdpsConfig, :mOdpsTableSchema, :mOdpsTable
+    def initialize(odpsConfig, project, table)
+      @mOdpsConfig = odpsConfig
+      @mProject = project
+      @mTable = table
+      @mShards = Array.new
+      if @mProject == nil or @mProject == ""
+        @mProject = @mOdpsConfig.defaultProjectName
+      end
+      @mOdpsTable = OdpsTable.new(@mOdpsConfig, @mProject, @mTable)
+      header = Hash.new
+      param = Hash.new
+      param[$PARAM_QUERY] = "meta"
+      conn = HttpConnection.new(@mOdpsConfig, header, param, getResource, "GET")
+      res = conn.getResponse
+      jsonTableMeta = JSON.parse(res.body)
+      if res.code != "200"
+        raise OdpsDatahubException.new(jsonTableMeta["Code"], "initialize failed because " + jsonTableMeta["Message"])
+      end
+      @mOdpsTableSchema = OdpsTableSchema.new(jsonTableMeta["Schema"])
+    end
+
+    #get partitions and return an array like :[{"time"=>"2016", "place"=>"china2"},{"time"=>"2015", "place"=>"china"}]
+    def getPartitionList
+      @mOdpsTable.getPartitionList
+    end
+
+    #ptStr ex: 'dt=20150805,hh=08,mm=24'
+    #call add partiton if not exsits
+    def addPartition(ptStr)
+      @mOdpsTable.addPartition(ptStr)
+    end
+
+    def getOdpsTableSchema
+      return @mOdpsTableSchema
+    end
+
+    def createStreamWriter(shardId = nil)
+      StreamWriter.new(@mOdpsConfig, @mProject, @mTable,getResource, shardId)
+    end
+
+    def createStreamArrayWriter(shardId = nil)
+      StreamWriter.new(@mOdpsConfig, @mProject, @mTable,getResource, shardId,  @mOdpsTableSchema)
+    end
+
+    #return json like [{"ShardId": "0","State": "loaded"},{"ShardId": "1","State": "loaded"}]
+    def getShardStatus
+      header = Hash.new
+      param = Hash.new
+      param[$PARAM_CURR_PROJECT] = @mProject
+      param[$PARAM_SHARD_STATUS] = ""
+
+      conn = HttpConnection.new(@mOdpsConfig, header, param, getResource + "/shards", "GET")
+      res = conn.getResponse
+      json_obj = JSON.parse(res.body)
+      if res.code != "200"
+        raise OdpsDatahubException.new(json_obj["Code"], "getShardStatus failed because " + json_obj["Message"])
+      end
+      return json_obj["ShardStatus"]
+    end
+
+    def loadShard(idx)
+      if idx < 0
+        raise OdpsDatahubException.new($INVALID_ARGUMENT, "loadShard num invalid")
+      end
+      header = Hash.new
+      param = Hash.new
+      param[$PARAM_CURR_PROJECT] = @mProject
+      param[$PARAM_SHARD_NUMBER] = idx
+      conn = HttpConnection.new(@mOdpsConfig, header, param, getResource + "/shards", "POST")
+      res = conn.getResponse
+      if res.code != "200"
+        json_obj = JSON.parse(res.body)
+        raise OdpsDatahubException.new(json_obj["Code"], "loadShard failed because " + json_obj["Message"])
+      end
+    end
+
+    protected
+    def getResource
+      return "/projects/" + @mProject + "/tables/" + @mTable
+    end
+  end
+end

data/lib/fluent/plugin/stream_reader.rb

@@ -0,0 +1,53 @@
+#
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing,
+#software distributed under the License is distributed on an
+#"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#KIND, either express or implied.  See the License for the
+#specific language governing permissions and limitations
+#under the License.
+#
+require 'net/http'
+require 'stringio'
+require 'digest/md5'
+require 'zlib'
+require_relative 'http/http_connection'
+require_relative 'serialize/deserializer'
+require_relative 'conf/config'
+require_relative 'odps/xstream_pack.pb'
+require_relative 'odps/odps_table'
+
+module OdpsDatahub
+  class StreamReader
+    attr_reader :mProject, :mTable, :mPath, :mShardId, :mPackId, :mReadMode, :mSchema, :mPackStream
+    def initialize(project, table, shardId, path, schema, packId = PackType.FIRST_PACK_ID)
+      @mProject = project
+      @mTable = table
+      @mPath = path
+      @mShardId = shardId
+      @mSchema = schema
+      @mPackId = packId
+      @mReadMode = ReadMode.SEEK_BEGIN
+    end
+    #TODO
+    def read #return a pack stream of this pack
+      if mPackStream != nil
+        mPackStream = getPack
+      end
+
+    end
+    #TODO
+    def getPack #get cur pack stream
+
+    end
+  end
+end

data/lib/fluent/plugin/stream_writer.rb

@@ -0,0 +1,152 @@
+#
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing,
+#software distributed under the License is distributed on an
+#"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#KIND, either express or implied.  See the License for the
+#specific language governing permissions and limitations
+#under the License.
+#
+require 'net/http'
+require 'stringio'
+require 'digest/md5'
+require 'zlib'
+require_relative 'exceptions'
+require_relative 'http/http_connection'
+require_relative 'serialize/serializer'
+require_relative 'conf/config'
+require_relative 'odps/xstream_pack.pb'
+require_relative 'odps/odps_table'
+
+module OdpsDatahub
+  class StreamWriter
+    attr_reader :mRecordList, :mProject, :mTable, :mPath, :mShardId, :mUpStream, :mOdpsConfig
+    def initialize(odpsConfig, project, table, path, shardId = nil, odpsSchema = nil)
+      @mOdpsConfig = odpsConfig
+      @mProject = project
+      @mTable = table
+      @mPath = path
+      @mShardId = shardId
+      @mSchema = odpsSchema
+      reload
+    end
+
+    def reload
+      @mUpStream = ::StringIO.new
+      @mRecordList = Array.new
+      @mUpStream.set_encoding(::Protobuf::Field::BytesField::BYTES_ENCODING)
+    end
+
+    def write(recordList, partition = "")
+      if recordList.is_a?Array
+        recordList.each{ |value|
+          #handle RecordList
+          if value.is_a?OdpsTableRecord
+            @mRecordList.push(value)
+          #handle ArrayList
+          elsif value.is_a?Array and @mSchema != nil and value.size == @mSchema.getColumnCount
+            record = convert2Record(value)
+            @mRecordList.push(record)
+          else
+            raise OdpsDatahubException.new($INVALID_ARGUMENT, "write an error type")
+          end
+        }
+      else
+        raise OdpsDatahubException.new($INVALID_ARGUMENT, "write param must be a array")
+      end
+
+      serializer = Serializer.new
+      serializer.serialize(@mUpStream, @mRecordList)
+
+      if @mUpStream.length == 0
+        raise OdpsDatahubException.new($INVALID_ARGUMENT, "mRecordList is empty")
+      end
+      header = Hash.new
+      param = Hash.new
+      param[$PARAM_CURR_PROJECT] = @mProject
+      #TODO partition format
+      param[$PARAM_PARTITION] = partition
+      param[$PARAM_RECORD_COUNT] = @mRecordList.size.to_s
+      header[$CONTENT_ENCODING] = "deflate"
+      header[$CONTENT_TYPE] = "application/octet-stream"
+=begin version 4
+      pack = OdpsDatahub::XStreamPack.new
+      pack.pack_data = Zlib::Deflate.deflate(@mUpStream.string)
+      pack.pack_meta = ""
+      upstream = ::StringIO.new
+      pack.serialize_to(upstream)
+      header[$CONTENT_MD5] = Digest::MD5.hexdigest(upstream.string)
+      header[$CONTENT_LENGTH] = upstream.length.to_s
+
+      conn = HttpConnection.new(@mOdpsConfig, header, param, @mPath + "/shards/" + @mShardId.to_s, "PUT", upstream)
+=end
+      #version 3
+      upStream = Zlib::Deflate.deflate(@mUpStream.string)
+      header[$CONTENT_MD5] = Digest::MD5.hexdigest(upStream)
+      header[$CONTENT_LENGTH] = upStream.length.to_s
+      #MAX_LENGTH 2048KB
+      if upStream.length > $MAX_PACK_SIZE
+        raise OdpsDatahubException.new($PACK_SIZE_EXCEED, "pack size:" + upStream.length.to_s)
+      end
+      if @mShardId != nil
+        conn = HttpConnection.new(@mOdpsConfig, header, param, @mPath + "/shards/" + @mShardId.to_s, "PUT", upStream)
+      else
+        conn = HttpConnection.new(@mOdpsConfig, header, param, @mPath + "/shards", "PUT", upStream)
+      end
+
+      reload
+      res = conn.getResponse
+      json_obj = JSON.parse(res.body)
+      if res.code != "200"
+        raise OdpsDatahubException.new(json_obj["Code"], "write failed because " + json_obj["Message"])
+      end
+    end
+
+    private
+    def convert2Record(value)
+      if not value.is_a?Array
+        raise OdpsDatahubException.new($INVALID_ARGUMENT, "param for convert2Record must be a array")
+      end
+
+      if value.count != @mSchema.getColumnCount
+        raise OdpsDatahubException.new($SCHEMA_NOT_MATCH, "column counts are not equal between value and schema")
+      end
+
+      record = OdpsTableRecord.new(@mSchema)
+      i = 0
+      while i < value.count do
+        type = @mSchema.getColumnType(i)
+        if value[i] == nil
+          record.setNullValue(i)
+          i += 1
+          next
+        end
+        case type
+          when $ODPS_BIGINT
+            record.setBigInt(i, value[i])
+          when $ODPS_BOOLEAN
+            record.setBoolean(i, value[i])
+          when $ODPS_DATETIME
+            record.setDateTime(i, value[i])
+          when $ODPS_DOUBLE
+            record.setDouble(i, value[i])
+          when $ODPS_STRING
+            record.setString(i, value[i])
+          else
+            raise OdpsDatahubException.new($INVALID_ARGUMENT, "unsupported schema type")
+        end
+        i += 1
+      end
+      return record
+    end
+  end
+end