aliyun-odps 0.1.0 → 0.4.0

data/lib/aliyun/odps/tunnel/download_session.rb

@@ -0,0 +1,98 @@
+require 'odps_protobuf'
+require 'aliyun/odps/tunnel/snappy_reader'
+
+module Aliyun
+  module Odps
+    class DownloadSession < Struct::Base
+      property :project, Project, required: true
+      property :client, Client, required: true
+
+      property :download_id, String, required: true
+      property :table_name, String, required: true
+      property :partition_spec, String
+      property :record_count, Integer
+      property :status, String
+      property :owner, String
+      property :initiated, DateTime
+      property :schema, Hash
+
+      # Download data
+      #
+      # @see http://repo.aliyun.com/api-doc/Tunnel/get_table_download_id/index.html Get Download Block ID
+      #
+      # @param start [String] specify start download row number
+      # @param count [String] specify download row count
+      # @param columns [Array] specify columns need download in array
+      # @param encoding [String] specify the data compression format, supported value: raw, deflate
+      #
+      # @return [Raw Data] return the raw data from ODPS
+      def download(start, count, columns, encoding = 'raw')
+        path = "/projects/#{project.name}/tables/#{table_name}"
+
+        query = build_download_query(start, count, columns)
+        headers = build_download_headers(encoding)
+
+        resp = client.get(path, query: query, headers: headers)
+        protobufed2records(resp.parsed_response, resp.headers['content-encoding'], columns)
+      end
+
+      private
+
+      def build_download_query(start, count, columns)
+        query = {
+          data: true,
+          downloadid: download_id,
+          columns: columns.join(','),
+          rowrange: "(#{start},#{count})"
+        }
+        query[:partition] = partition_spec if partition_spec
+        query
+      end
+
+      def protobufed2records(data, encoding, columns)
+        data = uncompress_data(data, encoding)
+        deserializer = OdpsProtobuf::Deserializer.new
+        schema = build_schema_with(columns)
+        deserializer.deserialize(data, schema)
+      rescue
+        raise RecordNotMatchSchemaError.new(columns, schema)
+      end
+
+      def uncompress_data(data, encoding)
+        case encoding
+        when 'deflate'
+          data
+        when 'x-snappy-framed'
+          SnappyReader.uncompress(data)
+        else
+          data
+        end
+      end
+
+      def build_download_headers(encoding)
+        headers = { 'x-odps-tunnel-version' => TableTunnels::TUNNEL_VERSION }
+        set_accept_encoding(headers, encoding)
+        headers
+      end
+
+      def set_accept_encoding(headers, encoding)
+        case encoding.to_s.downcase
+        when 'deflate'
+          headers['Accept-Encoding'] = 'deflate'
+        when 'snappy'
+          headers['Accept-Encoding'] = 'x-snappy-framed'
+        when 'raw'
+          headers.delete('Accept-Encoding')
+        else
+          fail ValueNotSupportedError.new(:encoding, TableTunnels::SUPPORTED_ENCODING)
+        end
+      end
+
+      def build_schema_with(columns)
+        {
+          'columns' => schema['columns'].select { |column| columns.include?(column['name']) }
+        }
+      end
+    end
+  end
+end

data/lib/aliyun/odps/tunnel/router.rb

@@ -0,0 +1,15 @@
+module Aliyun
+  module Odps
+    class TunnelRouter
+      def self.get_tunnel_endpoint(client, project_name)
+        host = client.get(
+          "/projects/#{project_name}/tunnel",
+          query: { service: true, curr_project: project_name }
+        ).parsed_response
+        "#{Aliyun::Odps.config.protocol}://#{host}"
+      rescue RequestError
+        nil
+      end
+    end
+  end
+end

data/lib/aliyun/odps/tunnel/snappy_reader.rb

@@ -0,0 +1,19 @@
+require 'zlib'
+require 'odps_protobuf'
+
+module Aliyun
+  class SnappyReader
+    def self.uncompress(data)
+      load_snappy
+
+      data.slice!(0, 18)
+      Snappy.inflate(data)
+    end
+
+    def self.load_snappy
+      require 'snappy'
+    rescue LoadError
+      raise 'Install snappy to support x-snappy-framed encoding: https://github.com/miyucy/snappy'
+    end
+  end
+end

data/lib/aliyun/odps/tunnel/snappy_writer.rb

@@ -0,0 +1,45 @@
+require 'snappy'
+require 'odps_protobuf'
+
+module Aliyun
+  class SnappyWriter
+    CHUNK_MAX = 65_536
+    COMPRESSION_THRESHOLD = 0.125
+    COMPRESSED_CHUNK = 0x00
+    UNCOMPRESSED_CHUNK = 0x01
+    STREAM_IDENTIFIER = 'sNaPpY'
+    IDENTIFIER_CHUNK = 0xff
+
+    def self.compress(data)
+      load_snappy
+
+      out = [[IDENTIFIER_CHUNK + (STREAM_IDENTIFIER.length << 8)].pack('<L'), STREAM_IDENTIFIER.force_encoding('ASCII-8BIT')]
+      (0..data.length).step(CHUNK_MAX) do |i|
+        chunk = data[i, CHUNK_MAX]
+        crc = masked_crc32c(chunk)
+        compressed_chunk = Snappy.deflate(chunk)
+        if compressed_chunk.length <= (1 - COMPRESSION_THRESHOLD) * chunk.length
+          chunk = compressed_chunk
+          chunk_type = COMPRESSED_CHUNK
+        else
+          chunk_type = UNCOMPRESSED_CHUNK
+        end
+        chunk.force_encoding('ASCII-8BIT')
+        out << [chunk_type + ((chunk.length + 4) << 8), crc].pack('<LL')
+        out << chunk
+      end
+      out.join('')
+    end
+
+    def self.masked_crc32c(data)
+      crc = OdpsProtobuf::CrcCalculator.calculate(StringIO.new(data))
+      (((crc >> 15) | (crc << 17)) + 0xa282ead8) & 0xffffffff
+    end
+
+    def self.load_snappy
+      require 'snappy'
+    rescue LoadError
+      raise 'Install snappy to support x-snappy-framed encoding: https://github.com/miyucy/snappy'
+    end
+  end
+end

data/lib/aliyun/odps/tunnel/table_tunnels.rb

@@ -0,0 +1,81 @@
+module Aliyun
+  module Odps
+    class TableTunnels < ServiceObject
+      TUNNEL_VERSION = '4'
+      SUPPORTED_ENCODING = %w(raw deflate snappy)
+
+      def client
+        config = Aliyun::Odps.config.dup
+        config.endpoint = TunnelRouter.get_tunnel_endpoint(project.client, project.name) || Aliyun::Odps.config.tunnel_endpoint
+        fail TunnelEndpointMissingError if config.endpoint.nil?
+        Aliyun::Odps::Client.new(config)
+      end
+
+      # Init Download Session
+      #
+      # @see http://repo.aliyun.com/api-doc/Tunnel/post_create_download_session/index.html Post Download Session
+      #
+      # @params table_name [String] specify table name
+      # @params partition [Hash] specify partition spec, format: { 'key1': 'value1', 'key2': 'value2' }
+      def init_download_session(table_name, partition = {})
+        path = "/projects/#{project.name}/tables/#{table_name}"
+        query = { downloads: true }
+        unless partition.empty?
+          query.merge!(partition: generate_partition_spec(partition))
+        end
+
+        resp = client.post(path, query: query)
+        result = resp.parsed_response
+        result = JSON.parse(result) if result.is_a?(String)
+
+        build_download_session(result, table_name, query['partition'])
+      end
+
+      # Init Upload Session
+      #
+      # @see http://repo.aliyun.com/api-doc/Tunnel/post_create_upload_session/index.html Post Upload Session
+      #
+      # @params table_name [String] specify table name
+      # @params partition [Hash] specify partition spec, format: { 'key1': 'value1', 'key2': 'value2' }
+      def init_upload_session(table_name, partition = {})
+        path = "/projects/#{project.name}/tables/#{table_name}"
+        query = { uploads: true }
+        unless partition.empty?
+          query.merge!(partition: generate_partition_spec(partition))
+        end
+
+        resp = client.post(path, query: query)
+        result = resp.parsed_response
+        result = JSON.parse(result) if result.is_a?(String)
+
+        build_upload_session(result, table_name, query['partition'])
+      end
+
+      private
+
+      def generate_partition_spec(partition)
+        partition.map { |k, v| "#{k}=#{v}" }.join(',')
+      end
+
+      def build_upload_session(result, table_name, partition_spec)
+        UploadSession.new(
+          result.merge(
+            project: project,
+            client: client,
+            table_name: table_name,
+            partition_spec: partition_spec
+          ))
+      end
+
+      def build_download_session(result, table_name, partition_spec)
+        DownloadSession.new(
+          result.merge(
+            project: project,
+            client: client,
+            table_name: table_name,
+            partition_spec: partition_spec
+          ))
+      end
+    end
+  end
+end

data/lib/aliyun/odps/tunnel/upload_block.rb

@@ -0,0 +1,9 @@
+module Aliyun
+  module Odps
+    class UploadBlock < Struct::Base
+      property :block_id, :String, required: true
+      property :create_time, :String
+      property :date, :String
+    end
+  end
+end

data/lib/aliyun/odps/tunnel/upload_session.rb

@@ -0,0 +1,132 @@
+require 'odps_protobuf'
+require 'aliyun/odps/tunnel/snappy_writer'
+
+module Aliyun
+  module Odps
+    class UploadSession < Struct::Base
+      property :project, :Project, required: true
+      property :client, :Client, required: true
+
+      property :upload_id, :String, required: true
+      property :table_name, :String, required: true
+      property :partition_spec, :String
+      property :status, :String
+      property :owner, :String
+      property :initiated, :DateTime
+      property :schema, :Hash
+      property :blocks, :Array, init_with: ->(value) do
+        value.map { |v| UploadBlock.new(v) }
+      end
+
+      alias_method :uploaded_block_list=, :blocks=
+
+      # Upload data with block id
+      #
+      # @see http://repo.aliyun.com/api-doc/Tunnel/put_create_upload_id/index.html Put Upload Block ID
+      #
+      # @param block_id [String] specify block_id for this upload, range in 0~19999, new block with replace with old with same blockid
+      # @param record_values [Array<Array>] specify the data, a array for your record, with order matched with your schema
+      # @param encoding [String] specify the data compression format, supported value: raw, deflate
+      #
+      # @return [true]
+      def upload(block_id, record_values, encoding = 'raw')
+        path = "/projects/#{project.name}/tables/#{table_name}"
+
+        query = { blockid: block_id, uploadid: upload_id }
+        query[:partition] = partition_spec if partition_spec
+
+        headers = build_upload_headers(encoding)
+        body = generate_upload_body(record_values, encoding)
+
+        !!client.put(path, query: query, headers: headers, body: body)
+      end
+
+      # reload this upload session
+      #
+      # @see http://repo.aliyun.com/api-doc/Tunnel/get_upload_session_status/index.html Get Upload Session Status
+      #
+      # @return [UploadSession]
+      def reload
+        path = "/projects/#{project.name}/tables/#{table_name}"
+
+        query = { uploadid: upload_id }
+        query.merge!(partition: partition_spec) if partition_spec
+
+        resp = client.get(path, query: query)
+
+        result = resp.parsed_response
+        attrs = result.is_a?(String) ? JSON.parse(result) : result
+
+        update_attrs(attrs)
+      end
+
+      # List uploaded blocks
+      #
+      # @return [Array<UploadBlock>]
+      def list_blocks
+        reload
+        blocks
+      end
+
+      # Complete the upload session
+      #
+      # @see http://repo.aliyun.com/api-doc/Tunnel/post_commit_upload_session/index.html Post commit Upload Session
+      #
+      # @return [true]
+      def complete
+        path = "/projects/#{project.name}/tables/#{table_name}"
+
+        query = { uploadid: upload_id }
+        query.merge!(partition: partition_spec) if partition_spec
+
+        headers = { 'x-odps-tunnel-version' => TableTunnels::TUNNEL_VERSION }
+
+        !!client.post(path, query: query, headers: headers)
+      end
+
+      private
+
+      def generate_upload_body(record_values, encoding)
+        serializer = OdpsProtobuf::Serializer.new
+        data = serializer.serialize(record_values, schema)
+        compress_data(data, encoding)
+      rescue
+        raise RecordNotMatchSchemaError.new(record_values, schema)
+      end
+
+      def compress_data(data, encoding)
+        case encoding
+        when 'raw'
+          data
+        when 'deflate'
+          require 'zlib'
+          Zlib::Deflate.deflate(data)
+        when 'snappy'
+          SnappyWriter.compress(data)
+        end
+      end
+
+      def set_content_encoding(headers, encoding)
+        case encoding.to_s.downcase
+        when 'deflate'
+          headers['Content-Encoding'] = 'deflate'
+        when 'snappy'
+          headers['Content-Encoding'] = 'x-snappy-framed'
+        when 'raw'
+          headers.delete('Content-Encoding')
+        else
+          fail ValueNotSupportedError.new(:encoding, TableTunnels::SUPPORTED_ENCODING)
+        end
+      end
+
+      def build_upload_headers(encoding)
+        headers = {
+          'x-odps-tunnel-version' => TableTunnels::TUNNEL_VERSION,
+          'Transfer-Encoding' => 'chunked'
+        }
+        set_content_encoding(headers, encoding)
+        headers
+      end
+    end
+  end
+end

data/lib/aliyun/odps/utils.rb

@@ -0,0 +1,102 @@
+require 'base64'
+require 'openssl'
+require 'digest'
+require 'gyoku'
+
+module Aliyun
+  module Odps
+    class Utils
+      class << self
+        # Calculate content length
+        #
+        # @return [Integer]
+        def content_size(content)
+          if content.respond_to?(:size)
+            content.size
+          elsif content.is_a?(IO)
+            content.stat.size
+          end
+        end
+
+        # HexDigest body with MD5
+        #
+        # @return [String]
+        def md5_hexdigest(body)
+          Digest::MD5.hexdigest(body).strip
+        end
+
+        # @example
+        #   # { 'a' => 1, 'c' => 3 }
+        #   Utils.hash_slice({ 'a' => 1, 'b' => 2, 'c' => 3 }, 'a', 'c')
+        #
+        # @return [Hash]
+        def hash_slice(hash, *selected_keys)
+          new_hash = {}
+          selected_keys.each { |k| new_hash[k] = hash[k] if hash.key?(k) }
+          new_hash
+        end
+
+        # Convert File or Bin data to bin data
+        #
+        # @return [Bin data]
+        def to_data(file_or_bin)
+          file_or_bin.respond_to?(:read) ? IO.binread(file_or_bin) : file_or_bin
+        end
+
+        def to_xml(hash, options = {}) # nodoc
+          %(<?xml version="1.0" encoding="UTF-8"?>#{Gyoku.xml(hash, options)})
+        end
+
+        # Dig values in deep hash
+        #
+        # @example
+        #   dig_value({ 'a' => { 'b' => { 'c' => 3 } } }, 'a', 'b', 'c')  # => 3
+        #
+        def dig_value(hash, *keys)
+          new_hash = hash.dup
+
+          keys.each do |key|
+            if new_hash.is_a?(Hash) && new_hash.key?(key)
+              new_hash = new_hash[key]
+            else
+              return nil
+            end
+          end
+          new_hash
+        end
+
+        # @see {http://apidock.com/rails/String/underscore String#underscore}
+        def underscore(str)
+          word = str.to_s.dup
+          word.gsub!(/::/, '/')
+          word.gsub!(/([A-Z\d]+)([A-Z][a-z])/, '\1_\2')
+          word.gsub!(/([a-z\d])([A-Z])/, '\1_\2')
+          word.tr!('-', '_')
+          word.downcase!
+          word
+        end
+
+        # Copy from {https://github.com/rails/rails/blob/14254d82a90b8aa4bd81f7eeebe33885bf83c378/activesupport/lib/active_support/core_ext/array/wrap.rb#L36 ActiveSupport::Array#wrap}
+        def wrap(object)
+          if object.nil?
+            []
+          elsif object.respond_to?(:to_ary)
+            object.to_ary || [object]
+          else
+            [object]
+          end
+        end
+
+        def stringify_keys!(hash)
+          hash.keys.each do |key|
+            hash[key.to_s] = hash.delete(key)
+          end
+        end
+
+        def generate_uuid(flag)
+          "#{flag}#{Time.now.strftime('%Y%m%d%H%M%S')}#{SecureRandom.hex(3)}"
+        end
+      end
+    end
+  end
+end