aliyun-odps 0.1.0 → 0.4.0

data/lib/aliyun/odps/configuration.rb

@@ -0,0 +1,16 @@
+require 'addressable/uri'
+
+module Aliyun
+  module Odps
+    class Configuration
+      attr_accessor :access_key, :secret_key, :endpoint, :tunnel_endpoint, :project, :options, :ssl_ca_file
+      def initialize
+        @options = {}
+      end
+
+      def protocol
+        Addressable::URI.parse(@endpoint).scheme
+      end
+    end
+  end
+end

data/lib/aliyun/odps/error.rb

@@ -0,0 +1,97 @@
+module Aliyun
+  module Odps
+    class Error < StandardError; end
+
+    # [Aliyun::Odps::RequestError] when Odps give a Non 2xx response
+    class RequestError < Error
+      # Error Code defined by Odps
+      attr_reader :code
+
+      # Error Message defined by Odps
+      attr_reader :message
+
+      # It's the UUID to uniquely identifies this request;
+      # When you can't solve the problem, you can request help from the ODPS development engineer with the RequestId.
+      attr_reader :request_id
+
+      # The Origin Httparty Response
+      attr_reader :origin_response
+
+      def initialize(response)
+        assign_error_code(response)
+        assign_request_id(response)
+        @origin_response = response
+        super("#{@request_id} - #{@code}: #{@message}")
+      end
+
+      def assign_error_code(response)
+        result = response.parsed_response
+        if result.key?('Error')
+          @code = result['Error']['Code']
+          @message = result['Error']['Message']
+        elsif result.key?('Code')
+          @code = result['Code']
+          @message = result['Message']
+        end
+      end
+
+      def assign_request_id(response)
+        @request_id = response.headers['x-odps-request-id']
+      end
+    end
+
+    class XmlElementMissingError < Error
+      def initialize(element)
+        super("Missing #{element} Element in xml")
+      end
+    end
+
+    class MissingProjectConfigurationError < Error
+      def initialize
+        super("Must config project first. Use Aliyun::Odps.configure {|config| config.project = 'your-project' }")
+      end
+    end
+
+    class PriorityInvalidError < Error
+      def initialize
+        super('Priority must more than or equal to zero.')
+      end
+    end
+
+    class InstanceTaskNotSuccessError < Error
+      def initialize(name, status, task_result)
+        super("Task #{name} #{status}: #{task_result}")
+      end
+    end
+
+    class InstanceNameInvalidError < Error
+      def initialize(name)
+        super("#{name} should match pattern: #{Instance::NAME_PATTERN}")
+      end
+    end
+
+    class TunnelEndpointMissingError < Error
+      def initialize
+        super("Tunnel Endpoint auto detect fail, Use Aliyun::Odps.configure {|config| config.tunnel_endpoint = 'your-project' } to config")
+      end
+    end
+
+    class ValueNotSupportedError < Error
+      def initialize(attr, supported_value)
+        super("#{attr} only support: #{Utils.wrap(supported_value).join(', ')} !!")
+      end
+    end
+
+    class ResourceMissingContentError < Error
+      def initialize
+        super('A Resource must exist file or table')
+      end
+    end
+
+    class RecordNotMatchSchemaError < Error
+      def initialize(values, schema)
+        super("#{values} not match with #{schema}")
+      end
+    end
+  end
+end

data/lib/aliyun/odps/http.rb

@@ -0,0 +1,138 @@
+require 'httparty'
+require 'addressable/uri'
+require 'aliyun/odps/error'
+
+module Aliyun
+  module Odps
+    class Http # nodoc
+      include HTTParty
+
+      class BetterXmlParser < HTTParty::Parser
+        protected
+
+        def xml
+          MultiXml.parse(body)
+        rescue
+          body
+        end
+      end
+      parser BetterXmlParser
+
+      attr_reader :config
+
+      def initialize(config)
+        @config = config
+      end
+
+      def get(uri, options = {})
+        request('GET', uri, options)
+      end
+
+      def put(uri, options = {})
+        headers = default_content_type.merge(options[:headers] || {})
+        request('PUT', uri, options.merge(headers: headers))
+      end
+
+      def post(uri, options = {})
+        headers = default_content_type.merge(options[:headers] || {})
+        request('POST', uri, options.merge(headers: headers))
+      end
+
+      def delete(uri, options = {})
+        headers = default_content_type.merge(options[:headers] || {})
+        request('DELETE', uri, options.merge(headers: headers))
+      end
+
+      def options(uri, options = {})
+        request('OPTIONS', uri, options)
+      end
+
+      def head(uri, options = {})
+        request('HEAD', uri, options)
+      end
+
+      private
+
+      def request(verb, resource, options = {})
+        query = options.fetch(:query, {})
+        headers = options.fetch(:headers, {})
+        body = options.delete(:body)
+
+        append_headers!(headers, verb, body, options.merge(path: resource))
+        path = config.endpoint + resource
+        options = { headers: headers, query: query, body: body }
+        append_options!(options, path)
+
+        wrap(self.class.__send__(verb.downcase, path, options))
+      end
+
+      def wrap(response)
+        case response.code
+        when 200..299
+          response
+        else
+          fail RequestError, response
+        end
+      end
+
+      def append_headers!(headers, verb, body, options)
+        append_default_headers!(headers)
+        append_body_headers!(headers, body)
+        append_authorization_headers!(headers, verb, options)
+      end
+
+      def append_options!(options, url)
+        options.merge!(uri_adapter: Addressable::URI)
+        if config.ssl_ca_file
+          options.merge!(ssl_ca_file: config.ssl_ca_file)
+        elsif url.start_with?('https://')
+          options.merge!(verify_peer: true)
+        end
+      end
+
+      def append_default_headers!(headers)
+        headers.merge!(default_headers)
+      end
+
+      def append_body_headers!(headers, body)
+        return headers unless body
+
+        unless headers.key?('Content-MD5')
+          headers.merge!('Content-MD5' => Utils.md5_hexdigest(body))
+        end
+
+        return if headers.key?('Content-Length')
+        headers.merge!('Content-Length' => Utils.content_size(body).to_s)
+      end
+
+      def append_authorization_headers!(headers, verb, options)
+        auth_key = get_auth_key(
+          options.merge(verb: verb, headers: headers, date: headers['Date'])
+        )
+        headers.merge!('Authorization' => auth_key)
+      end
+
+      def get_auth_key(options)
+        Authorization.get_authorization(config.access_key, config.secret_key, options)
+      end
+
+      def default_headers
+        {
+          'User-Agent' => user_agent,
+          'Date' => Time.now.utc.strftime('%a, %d %b %Y %H:%M:%S GMT')
+        }
+      end
+
+      def default_content_type
+        {
+          'Content-Type' => 'application/xml'
+        }
+      end
+
+      def user_agent
+        "aliyun-odps-sdk-ruby/#{Aliyun::Odps::VERSION} " \
+        "(#{RbConfig::CONFIG['host_os']} ruby-#{RbConfig::CONFIG['ruby_version']})"
+      end
+    end
+  end
+end

data/lib/aliyun/odps/list.rb

@@ -0,0 +1,40 @@
+require 'forwardable'
+
+module Aliyun
+  module Odps
+    # Wrap for simple array and give marker and max_items methods
+    class List
+      include Enumerable
+      extend Forwardable
+
+      attr_reader :marker, :max_items
+      def_delegators :@objects, :[], :each, :size, :inspect
+
+      def initialize(marker, max_items, objects)
+        @marker = marker
+        @max_items = max_items.to_i
+        @objects = objects
+      end
+
+      # Auto detect marker, max_items, values from result,
+      # build a object, where you can access marker, max_items, and values
+      #
+      # @example
+      #
+      #   Aliyun::Odps::List.build(result, %w(Projects Project)) do |hash|
+      #     Project.new(hash.merge(client: client))
+      #   end
+      #
+      # @return [List]
+      def self.build(result, keys, &_block)
+        top_key = keys.first
+        marker = Utils.dig_value(result, top_key, 'Marker')
+        max_items = Utils.dig_value(result, top_key, 'MaxItems')
+        objects = Utils.wrap(Utils.dig_value(result, *keys)).map do |hash|
+          yield hash
+        end
+        new(marker, max_items, objects)
+      end
+    end
+  end
+end

data/lib/aliyun/odps/model/function.rb

@@ -0,0 +1,16 @@
+module Aliyun
+  module Odps
+    class Function < Struct::Base
+      extend Aliyun::Odps::Modelable
+
+      property :name, String, required: true
+      property :owner, String
+      property :class_type, String
+      property :creation_time, DateTime
+      property :resources, Array
+      property :location, String
+
+      alias_method :alias=, :name=
+    end
+  end
+end

data/lib/aliyun/odps/model/functions.rb

@@ -0,0 +1,113 @@
+module Aliyun
+  module Odps
+    # Methods for Functions
+    class Functions < ServiceObject
+      # List Functions of project
+      #
+      # @see http://repo.aliyun.com/api-doc/Function/get_functions/index.html Get functions
+      #
+      # @param options [Hash] options
+      # @option options [String] :name specify function name
+      # @option options [String] :owner specify function owner
+      # @option options [String] :marker
+      # @option options [String] :maxitems (1000)
+      #
+      # @return [List]
+      def list(options = {})
+        Utils.stringify_keys!(options)
+        path = "/projects/#{project.name}/registration/functions"
+        query = Utils.hash_slice(options, 'name', 'owner', 'marker', 'maxitems')
+        result = client.get(path, query: query).parsed_response
+
+        Aliyun::Odps::List.build(result, %w(Functions Function)) do |hash|
+          Function.new(hash)
+        end
+      end
+
+      # Get Function
+      #
+      # @param name specify function name
+      #
+      # @return [Function]
+      def get(name)
+        path = "/projects/#{project.name}/registration/functions/#{name}"
+
+        result = client.get(path).parsed_response
+        Function.new(Utils.dig_value(result, 'Function'))
+      end
+      alias_method :function, :get
+
+      # Register function in project
+      #
+      # @see http://repo.aliyun.com/api-doc/Function/post_function/index.html Post function
+      #
+      # @param name [String] specify function name
+      # @param class_path [String] specify class Path used by function
+      # @param resources [Array<Model::Resource>] specify resources used by function
+      #
+      # @return [Function]
+      def create(name, class_path, resources = [])
+        path = "/projects/#{project.name}/registration/functions"
+
+        function = Function.new(
+          name: name,
+          class_type: class_path,
+          resources: resources
+        )
+
+        resp = client.post(path, body: build_create_body(function))
+
+        function.tap do |obj|
+          obj.location = resp.headers['Location']
+        end
+      end
+
+      # Update function in project
+      #
+      # @see http://repo.aliyun.com/api-doc/Function/put_function/index.html Put function
+      #
+      # @param name [String] specify function name
+      # @param class_path [String] specify class Path used by function
+      # @param resources [Array<Model::Resource>] specify resources used by function
+      #
+      # @return [true]
+      def update(name, class_path, resources = [])
+        path = "/projects/#{project.name}/registration/functions/#{name}"
+
+        function = Function.new(
+          name: name,
+          class_type: class_path,
+          resources: resources
+        )
+        !!client.put(path, body: build_create_body(function))
+      end
+
+      # Delete function in project
+      #
+      # @see http://repo.aliyun.com/api-doc/Function/delete_function/index.html Delete function
+      #
+      # @param name [String] specify function name
+      #
+      # @return [true]
+      def delete(name)
+        path = "/projects/#{project.name}/registration/functions/#{name}"
+        !!client.delete(path)
+      end
+
+      private
+
+      def build_create_body(function)
+        fail XmlElementMissingError, 'ClassType' if function.class_type.nil?
+        fail XmlElementMissingError, 'Resources' if function.resources.empty?
+
+        Utils.to_xml(
+          'Function' => {
+            'Alias' => function.name,
+            'ClassType' => function.class_type,
+            'Resources' => function.resources.map(&:to_hash)
+          }
+        )
+      end
+    end
+  end
+end

data/lib/aliyun/odps/model/instance.rb

@@ -0,0 +1,130 @@
+module Aliyun
+  module Odps
+    class Instance < Struct::Base
+      extend Aliyun::Odps::Modelable
+
+      NAME_PATTERN = /^([a-z]|[A-Z]){1,}([a-z]|[A-Z]|[\d]|_)*/
+
+      property :project, Project, required: true
+
+      property :name, String, required: true
+      property :owner, String
+      property :comment, String
+      property :priority, Integer
+      property :tasks, Array
+      property :status, String
+      property :start_time, DateTime
+      property :end_time, DateTime
+      property :location, String
+
+      # Get task detail of instance
+      #
+      # @see http://repo.aliyun.com/api-doc/Instance/get_instance_detail/index.html Get instance detail
+      #
+      # @params task_name [String] specify task name
+      #
+      # @return [Hash]
+      def task_detail(task_name)
+        path = "/projects/#{project.name}/instances/#{name}"
+        query = { instancedetail: true, taskname: task_name }
+        client.get(path, query: query).parsed_response
+      end
+
+      # Get task progress of instance
+      #
+      # @see http://repo.aliyun.com/api-doc/Instance/get_instance_progress/index.html Get instance progress
+      #
+      # @params task_name [String] specify task name
+      #
+      # @return [Hash]
+      def task_progress(task_name)
+        path = "/projects/#{project.name}/instances/#{name}"
+        query = { instanceprogress: true, taskname: task_name }
+        client.get(path, query: query).parsed_response['Progress']
+      end
+
+      # Get task summary of instance
+      #
+      # @see http://repo.aliyun.com/api-doc/Instance/get_instance_summary/index.html Get instance summary
+      #
+      # @params task_name [String] specify task name
+      #
+      # @return [Hash]
+      def task_summary(task_name)
+        path = "/projects/#{project.name}/instances/#{name}"
+        query = { instancesummary: true, taskname: task_name }
+        client.get(path, query: query).parsed_response
+      end
+
+      # Get task results
+      #
+      # @return [Hash<name, TaskResult>]
+      def task_results
+        path = "/projects/#{project.name}/instances/#{name}"
+        query = { result: true }
+        result = client.get(path, query: query).parsed_response
+        task_results = Utils.dig_value(result, 'Instance', 'Tasks', 'Task')
+        Hash[Utils.wrap(task_results).map { |v| [v['Name'], Aliyun::Odps::TaskResult.new(v)] }]
+      end
+
+      # Get tasks of instance
+      #
+      # @see http://repo.aliyun.com/api-doc/Instance/get_instance_task/index.html Get instance task
+      #
+      # @return [List]
+      def list_tasks
+        path = "/projects/#{project.name}/instances/#{name}"
+        query = { taskstatus: true }
+        result = client.get(path, query: query).parsed_response
+
+        keys = %w(Instance Tasks Task)
+        Utils.wrap(Utils.dig_value(result, *keys)).map do |hash|
+          InstanceTask.new(hash)
+        end
+      end
+
+      # Terminate the instance
+      #
+      # @see http://repo.aliyun.com/api-doc/Instance/put_instance_terminate/index.html Put instance terminated
+      #
+      # @return true
+      def terminate
+        path = "/projects/#{project.name}/instances/#{name}"
+
+        body = Utils.to_xml(
+          'Instance' => { 'Status' => 'Terminated' }
+        )
+        !!client.put(path, body: body)
+      end
+
+      # Get status
+      #
+      # @see http://repo.aliyun.com/api-doc/Instance/get_instance/index.html Get instance
+      #
+      # @return [String] Instance status: Suspended, Running, Terminated
+      def get_status
+        path = "/projects/#{project.name}/instances/#{name}"
+        result = client.get(path).parsed_response
+        Utils.dig_value(result, 'Instance', 'Status')
+      end
+
+      # Block process until instance success
+      #
+      # @raise [InstanceTaskNotSuccessError] if task not success
+      def wait_for_success(interval = 0.01)
+        wait_for_terminated(interval)
+
+        list_tasks.each do |task|
+          if task.status.upcase != 'SUCCESS'
+            fail InstanceTaskNotSuccessError.new(task.name, task.status, task_results[task.name].result)
+          end
+        end
+      end
+
+      # Block process until instance terminated
+      def wait_for_terminated(interval = 0.01)
+        sleep interval while get_status != 'Terminated'
+      end
+    end
+  end
+end