aliyun-odps 0.1.0 → 0.4.0

data/lib/aliyun/odps/model/table_partition.rb

@@ -0,0 +1,9 @@
+module Aliyun
+  module Odps
+    class TablePartition < Struct::Base
+      property :name, String, required: true
+      property :type, String, required: true, within: %w(bigint double boolean datetime string)
+      property :comment, String
+    end
+  end
+end

data/lib/aliyun/odps/model/table_partitions.rb

@@ -0,0 +1,90 @@
+module Aliyun
+  module Odps
+    class TablePartitions < ServiceObject
+      # List partitions for table
+      #
+      # @see http://repo.aliyun.com/api-doc/Table/get_table_partition/index.html Get table partitions
+      #
+      # @param options [Hash] options
+      # @option options [String] :marker specify marker for paginate
+      # @option options [String] :maxitems (1000) specify maxitems in this request
+      #
+      # @return [List]
+      def list(options = {})
+        Utils.stringify_keys!(options)
+        path = "/projects/#{project.name}/tables/#{master.name}"
+        query = {
+          partitions: true,
+          expectmarker: true
+        }.merge(Utils.hash_slice(options, 'marker', 'maxitems'))
+
+        result = client.get(path, query: query).parsed_response
+        Aliyun::Odps::List.build(result, %w(Partitions Partition)) do |hash|
+          build_table_partition(hash)
+        end
+      end
+
+      # Create Partition
+      #
+      # @param partition [Hash<name, value>] specify the partition you want to create
+      #
+      # @raise InstanceTaskNotSuccessError if instance task failed
+      #
+      # @return [Hash] the partition you create
+      def create(partition)
+        sql = generate_create_sql(partition)
+
+        task = InstanceTask.new(
+          name: 'SQLCreatePartitionTask',
+          type: 'SQL',
+          query: sql
+        )
+
+        instance = project.instances.create([task])
+
+        instance.wait_for_success
+
+        partition
+      end
+
+      # Delete Partition
+      #
+      # @params partitions [Hash<name, value>] specify the partition you want to delete
+      #
+      # @raise InstanceTaskNotSuccessError if instance task failed
+      #
+      # @return [true]
+      def delete(partition)
+        sql = generate_drop_sql(partition)
+
+        task = InstanceTask.new(
+          name: 'SQLDropPartitionTask',
+          type: 'SQL',
+          query: sql
+        )
+
+        instance = project.instances.create([task])
+
+        instance.wait_for_success
+
+        true
+      end
+
+      private
+
+      def generate_create_sql(partition)
+        spec = partition.map { |k, v| "#{k} = '#{v}'" }.join(',')
+        "ALTER TABLE #{project.name}.`#{master.name}` ADD PARTITION (#{spec});"
+      end
+
+      def generate_drop_sql(partition)
+        spec = partition.map { |k, v| "#{k} = '#{v}'" }.join(',')
+        "ALTER TABLE #{project.name}.`#{master.name}`DROP IF EXISTS PARTITION (#{spec});"
+      end
+
+      def build_table_partition(hash)
+        Hash[Utils.wrap(hash['Column']).map(&:values)]
+      end
+    end
+  end
+end

data/lib/aliyun/odps/model/table_schema.rb

@@ -0,0 +1,13 @@
+module Aliyun
+  module Odps
+    class TableSchema < Struct::Base
+      property :columns, Array, init_with: ->(value) do
+        value.map { |v| TableColumn.new(v) }
+      end
+
+      property :partitions, Array, init_with: ->(value) do
+        value.map { |v| TablePartition.new(v) }
+      end
+    end
+  end
+end

data/lib/aliyun/odps/model/tables.rb

@@ -0,0 +1,125 @@
+module Aliyun
+  module Odps
+    # Methods for Tables
+    class Tables < ServiceObject
+      # List tables in this project
+      #
+      # @see http://repo.aliyun.com/api-doc/Table/get_tables/index.html Get tables
+      #
+      # @params options [Hash] options
+      # @option options [String] :name specify the table name
+      # @option options [String] :owner specify the table owner
+      # @option options [String] :marker
+      # @option options [String] :maxitems (1000)
+      def list(options = {})
+        Utils.stringify_keys!(options)
+        path = "/projects/#{project.name}/tables"
+        query = Utils.hash_slice(options, 'name', 'owner', 'marker', 'maxitems')
+        query.merge!(tables: true, expectmarker: true)
+        result = client.get(path, query: query).parsed_response
+
+        Aliyun::Odps::List.build(result, %w(Tables Table)) do |hash|
+          build_table(hash)
+        end
+      end
+
+      # Get Table
+      #
+      # @see http://repo.aliyun.com/api-doc/Table/get_table/index.html Get Table
+      #
+      # @params name [String] specify the table name
+      def get(name)
+        path = "/projects/#{project.name}/tables/#{name}"
+        resp = client.get(path)
+
+        build_table(
+          Utils.dig_value(resp.parsed_response, 'Table')
+          .merge(
+            'creation_time' => resp.headers['x-odps-creation-time'],
+            'last_modified' => resp.headers['Last-Modified'],
+            'owner' => resp.headers['x-odps-owner']
+          )
+        )
+      end
+      alias_method :table, :get
+
+      # Create Table
+      #
+      # @params name [String] specify table name
+      # @params schema [Struct::TableSchema] specify table schema
+      # @params options [Hash] options
+      # @option options [String] :comment specify table comment
+      #
+      # @raise InstanceTaskNotSuccessError if instance task failed
+      #
+      # @return Table
+      def create(name, schema, options = {})
+        Utils.stringify_keys!(options)
+        table = Table.new(name: name, schema: schema, project: project)
+        table.comment = options['comment'] if options.key?('comment')
+
+        task = InstanceTask.new(name: 'SQLCreateTableTask', type: 'SQL', query: generate_create_sql(table))
+
+        instance = project.instances.create([task])
+        instance.wait_for_success
+        table
+      end
+
+      # Delete Table
+      #
+      # @params name [String]
+      #
+      # @raise InstanceTaskNotSuccessError if instance task failed
+      #
+      # @return true
+      def delete(name)
+        table = Table.new(name: name, project: project)
+
+        task = InstanceTask.new(
+          name: 'SQLDropTableTask',
+          type: 'SQL',
+          query: generate_drop_sql(table)
+        )
+
+        instance = project.instances.create([task])
+        instance.wait_for_success
+        true
+      end
+
+      private
+
+      def generate_create_sql(table)
+        sql = "CREATE TABLE #{project.name}.`#{table.name}`"
+        sql += generate_table_coumns_sql(table.schema.columns) if table.schema && table.schema.columns
+        sql += " COMMENT '#{table.comment}'" if table.comment
+        sql += generate_table_partitions_sql(table.schema.partitions) if table.schema && table.schema.partitions
+        sql += ';'
+        sql
+      end
+
+      def generate_table_coumns_sql(columns)
+        ' (' + columns.map do |column|
+          generate_table_column_sql(column)
+        end.join(', ') + ')'
+      end
+
+      def generate_table_partitions_sql(partitions)
+        ' PARTITIONED BY (' + partitions.map do |column|
+          generate_table_column_sql(column)
+        end.join(', ') + ')'
+      end
+
+      def generate_table_column_sql(column)
+        "`#{column.name}` #{column.type}" + (column.comment ? " COMMENT '#{column.comment}'" : '')
+      end
+
+      def generate_drop_sql(table)
+        "DROP TABLE IF EXISTS #{project.name}.`#{table.name}`;"
+      end
+
+      def build_table(result)
+        Table.new(result.merge(project: project))
+      end
+    end
+  end
+end

data/lib/aliyun/odps/model/task_result.rb

@@ -0,0 +1,9 @@
+module Aliyun
+  module Odps
+    class TaskResult < Struct::Base
+      property :name, String, required: true
+      property :type, String, required: true
+      property :result, Hash, required: true
+    end
+  end
+end

data/lib/aliyun/odps/modelable.rb

@@ -0,0 +1,16 @@
+require 'active_support/all'
+require 'aliyun/odps/service_object'
+
+module Aliyun
+  module Odps
+    module Modelable
+      def has_many(models, _opts = {})
+        mod = models.to_s.singularize
+        klass = "Aliyun::Odps::#{mod.camelize}s".constantize
+        define_method(models) do
+          klass.build(self)
+        end
+      end
+    end
+  end
+end

data/lib/aliyun/odps/project.rb

@@ -0,0 +1,47 @@
+require 'aliyun/odps/model/instances'
+require 'aliyun/odps/model/tables'
+require 'aliyun/odps/model/functions'
+require 'aliyun/odps/model/resources'
+require 'aliyun/odps/tunnel/table_tunnels'
+require 'aliyun/odps/client'
+
+module Aliyun
+  module Odps
+    class Project < Struct::Base
+      extend Aliyun::Odps::Modelable
+
+      # @!method instances
+      # @return [Instances]
+      has_many :instances
+
+      # @!method functions
+      # @return [Functions]
+      has_many :functions
+
+      # @!method tables
+      # @return [Tables]
+      has_many :tables
+
+      # @!method resources
+      # @return [Resources]
+      has_many :resources
+
+      # @!method table_tunnels
+      # @return [TableTunnels]
+      has_many :table_tunnels
+
+      property :client, Client, required: true
+
+      property :name, String, required: true
+      property :comment, String
+      property :project_group_name, String
+      property :state, String
+      property :clusters, Hash
+      property :property, String
+      property :properties, Hash
+    end
+  end
+end
+
+Dir[File.join(File.dirname(__FILE__), 'model/*.rb')].each { |f| require f }
+Dir[File.join(File.dirname(__FILE__), 'tunnel/*.rb')].each { |f| require f }

data/lib/aliyun/odps/service_object.rb

@@ -0,0 +1,27 @@
+module Aliyun
+  module Odps
+    class ServiceObject
+      def client
+        master.is_a?(Client) ? master : project.client
+      end
+
+      def project
+        @master.is_a?(Project) ? @master : @master.try(:project)
+      end
+
+      attr_reader :master
+
+      def initialize(master, _options = {})
+        @master = master
+      end
+
+      def self.service_pool
+        @service_pool ||= {}
+      end
+
+      def self.build(master, options = {})
+        service_pool[master] ||= new(master, options)
+      end
+    end
+  end
+end

data/lib/aliyun/odps/struct.rb

@@ -0,0 +1,126 @@
+module Aliyun
+  module Odps
+    module Struct
+      class Base
+        def initialize(attributes = {})
+          attributes.each do |key, value|
+            m = "#{Utils.underscore(key)}=".to_sym
+            send(m, value) if self.respond_to?(m)
+          end
+
+          validate_required
+        end
+
+        def validate_required
+          missing_attrs = []
+          (self.class.required_attrs || []).each do |attr|
+            missing_attrs.push(attr) if send(attr).nil?
+          end
+          fail "Missing attribute: #{missing_attrs.join(',')}" unless missing_attrs.empty?
+        end
+        private :validate_required
+
+        def update_attrs(attrs)
+          attrs.each do |k, v|
+            send("#{Utils.underscore(k)}=", v)
+          end
+          self
+        end
+
+        def client
+          project.client
+        end
+
+        class << self
+          attr_reader :required_attrs
+
+          # @!macro [attach] property
+          #   @!attribute [rw] $1
+          #   @return [$2]
+          #
+          # @example
+          #
+          #  property :name, String, required: true, init_with: proc {|value| value.upcase }, within: %w{value1 value2}
+          #
+          # @params options [Hash] options
+          # @option options [Boolean] :required required or optional
+          # @option options [block] :init_with block used to init attr
+          def property(attr, type, options = {})
+            @required_attrs ||= []
+            @required_attrs << attr.to_s if options[:required]
+
+            define_reader_method(attr)
+            define_writer_method(attr, type, options)
+          end
+
+          private
+
+          def define_reader_method(attr)
+            attr_reader attr
+          end
+
+          def define_writer_method(attr, type, options)
+            init_with_block =
+              if options.key?(:init_with) && options[:init_with].respond_to?(:call)
+                options[:init_with]
+              else
+                build_block_with_type(attr, type, options)
+              end
+            define_writer_method_with_block(attr, init_with_block)
+          end
+
+          def build_block_with_type(attr, type, options)
+            case type.to_s
+            when 'Integer'
+              build_integer_block(attr, options)
+            when 'DateTime'
+              build_datetime_block(attr, options)
+            when 'String'
+              build_string_block(attr, options)
+            else
+              build_default_block(attr, options)
+            end
+          end
+
+          def build_string_block(attr, options)
+            if options.key?(:within) && options[:within].is_a?(Array)
+              build_block_with_within(attr, options)
+            else
+              build_default_block(attr, options)
+            end
+          end
+
+          def build_block_with_within(attr, options)
+            proc do |value|
+              if options[:within].include?(value.to_s)
+                value
+              else
+                fail ValueNotSupportedError.new(attr, options[:within])
+              end
+            end
+          end
+
+          def build_integer_block(attr, options)
+            proc { |value| value.to_i }
+          end
+
+          def build_datetime_block(attr, options)
+            proc { |value| DateTime.parse(value) }
+          end
+
+          def build_default_block(attr, options)
+            proc { |value| value }
+          end
+
+          def define_writer_method_with_block(attr, block)
+            define_method("#{attr}=") do |value|
+              instance_eval do
+                instance_variable_set("@#{attr}", block.call(value))
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end