iij-dag-client 1.0.1

data/lib/dag/client/model/cluster_collection.rb

@@ -0,0 +1,76 @@
+module Dag
+  class ClusterCollection < Model
+    include Enumerable
+    include Dag::Client::ClusterValidation
+
+    def initialize(api)
+      super(api)
+    end
+
+    #
+    # == Parameters ==
+    # <tt>status</tt> - 'init', 'stopped', restarting, 'norm', 'failed', 'ptfailed', 'error', 'released'
+    def where(params)
+      validate_cluster_param_keys(params)
+
+      @status = params[:status] if params[:status]
+      @type = params[:type] if params[:type]
+      @cluster_name = params[:cluster_name] if params[:cluster_name].present?
+
+      self
+    end
+
+    def order(o)
+      result = o.downcase.to_s
+      unless ['asc', 'desc'].include?(result)
+        raise Dag::Client::ParameterInvalid.new("Invalid order condition: #{o}")
+      end
+
+      @order = result
+
+      self
+    end
+
+    def each
+      marker = nil
+      truncated = false
+      begin
+        cluster_info_list = @api.cluster_info_list(make_options(marker))
+        cluster_info_list['clusters'].each do |cluster_info|
+          yield Dag::Cluster.new(@api, cluster_info)
+        end
+        truncated = cluster_info_list['isTruncated']
+        marker = cluster_info_list['nextMarker']
+      end while truncated
+    end
+
+    private
+
+    def make_options(marker = nil)
+      options = { max: 100 }
+      if marker
+        options = options.merge(marker: marker)
+      end
+
+      if @order
+        options = options.merge(order: @order)
+      end
+
+      if @status
+        status = @status.respond_to?(:join) ? @status.join(",") : @status
+        options = options.merge(status: status)
+      end
+
+      if @type
+        options = options.merge(type: @type)
+      end
+
+      if @cluster_name
+        options = options.merge(cluster_name: @cluster_name)
+      end
+
+      options
+    end
+
+  end
+end

data/lib/dag/client/model/database.rb

@@ -0,0 +1,34 @@
+module Dag
+  class Database < Model
+    def initialize(api, cluster_name, db_name, cluster_status: nil)
+      super(api)
+
+      @cluster_name = cluster_name
+      @cluster_status = cluster_status
+      @db_name = db_name
+      @name = @db_name
+    end
+
+    attr_reader :cluster_name, :db_name, :name
+
+    def delete
+      @api.delete_database(@cluster_name, @name)
+    end
+
+    #
+    # == parameters ==
+    # * <tt>label</tt> - label
+    # * <tt>input_object_keys</tt> - array of object_keys. ex ["#{bucket}/${big_object1}", "#{bucket}/${big_object2}"]
+    # * <tt>input_format</tt> - 'csv' or 'tsv' or 'json'
+    # * <tt>schema</tt> - schema
+    def split(tbl_name, params)
+      split_info = @api.split_table(@cluster_name, @db_name, tbl_name, params)
+      job_id = split_info['queryId']
+
+      query_info = @api.query_info(job_id)
+      Dag::Job.new(@api, query_info)
+    end
+
+    include Dag::Client::Table
+  end
+end

data/lib/dag/client/model/database_collection.rb

@@ -0,0 +1,51 @@
+module Dag
+  class DatabaseCollection < Model
+    include Enumerable
+    include Dag::Client::ClusterValidation
+
+    attr_reader :cluster_name
+
+    def initialize(api, cluster_name, cluster_status: nil)
+      super(api)
+
+      @cluster_name = cluster_name
+      @cluster_status = cluster_status
+    end
+
+    def each
+      validate_cluster
+
+      find_from_api unless find?
+
+      @databases.each do |db_name|
+        yield Dag::Database.new(@api, @cluster_name, db_name)
+      end
+    end
+
+    def create(db_name)
+      validate_cluster
+
+      @api.create_database(@cluster_name, db_name)
+      Dag::Database.new(@api, @cluster_name, db_name)
+    end
+
+    private
+
+    def find_from_api
+      dbs = []
+      tmp = {}
+      params = { "max" => 100 }
+      begin
+        params.merge!({"marker" => tmp["nextMarker"]}) rescue nil
+        tmp = @api.database_list(@cluster_name , params: params)
+        dbs += tmp["databases"]
+      end while tmp["isTruncated"]
+
+      @databases = dbs
+    end
+
+    def find?
+      @databases.present?
+    end
+  end
+end

data/lib/dag/client/model/job.rb

@@ -0,0 +1,125 @@
+module Dag
+  class Job < Model
+    def initialize(api, job_info)
+      super(api)
+      update_parameters(job_info)
+    end
+
+    attr_reader :id, :status, :dsl, :process_engine, :start_at, :type, :cluster, :cluster_rebooted, :label, :stage, :progress,
+      :access_key_id, :query, :output_format, :output_resource_path, :job_id, :schema, :input_object_keys,
+      :input_format, :output_database, :output_table
+
+    def finished?
+      @status == 'finished'
+    end
+
+    def running?
+      @status == 'running'
+    end
+
+    def split?
+      @type == 'split'
+    end
+
+    def hive?
+      @type == 'select'
+    end
+
+    def cluster_rebooted?
+      !!@cluster_rebooted
+    end
+
+    def reload
+      job_info = @api.query_info(@id)
+      update_parameters(job_info)
+    end
+
+    def kill
+      validate_cancel_condition
+      @api.query_cancel(@id)
+    end
+
+    alias :cancel :kill
+
+    def download_urls(time_limit = 30)
+      raise Dag::Client::StatusInvalid.new("job status is not finished") unless finished?
+      expire_at = time_limit.minutes.since.to_i
+      object_uri = URI.parse(@output_resource_path)
+      bucket = object_uri.host
+      object_path = object_uri.path[1..-1]
+      object_path += '/' unless object_path.end_with? '/'
+      bucket_objects = @api.objects(bucket, prefix: object_path).objects
+      bucket_objects.map do |object|
+        path = if @api.force_path_style?
+                 "/#{bucket}/#{object}"
+               else
+                 "/#{object}"
+               end
+
+        parameters = {
+          "Expires" => expire_at,
+          "IIJGIOAccessKeyId" => @api.apikey,
+          "Signature" => @api.download_signature(expire_at, bucket, path)
+        }
+
+        uri = URI.parse(@api.storage_api)
+        url = if @api.force_path_style?
+                "http://#{uri.host}"
+              else
+                "http://#{bucket}.#{uri.host}"
+              end
+        url += ":#{uri.port}" unless uri.port == 80 || uri.port == 443
+
+        File.join(url, "#{path}?#{parameters.to_param}")
+      end
+    end
+
+    def log
+      validate_log_condition
+      log_info = @api.query_log(@id)
+      log_info ? log_info['log'] : ''
+    end
+
+    def validate_log_condition
+      if split?
+        raise Dag::Client::JobTypeInvalid.new("job type is not select")
+      end
+
+      if cluster_rebooted?
+        raise Dag::Client::ClusterRebooted.new("cluster is rebooted")
+      end
+    end
+
+    def validate_cancel_condition
+      unless running?
+        raise Dag::Client::StatusInvalid.new("job status is not running")
+      end
+    end
+
+    private
+
+    def update_parameters(job)
+      @id = job['id']
+      @status = job['status']
+      @process_engine = job['processEngine']
+      @dsl = job['dsl']
+      @cluster = job['clusterName']
+      @cluster_rebooted = job['clusterRebooted']
+      @start_at = Time.parse(job['startTime']) if job['startTime']
+      @access_key_id = job['accessKeyId']
+      @query = job['query']
+      @output_format = job['outputFormat']
+      @output_resource_path = job['outputResourcePath']
+      @type = job['type']
+      @label = job['label']
+      @stage = job['stage']
+      @progress = job['progress']
+      @job_id = job['jobId']
+      @schema = job['schema']
+      @input_object_keys = job['inputObjectKeys']
+      @input_format = job['inputFormat']
+      @output_database = job['outputDatabase']
+      @output_table = job['outputTable']
+    end
+  end
+end

data/lib/dag/client/model/job_collection.rb

@@ -0,0 +1,114 @@
+module Dag
+  class JobCollection < Model
+    include Enumerable
+    include Dag::Client::JobValidation
+
+    def initialize(api)
+      super(api)
+    end
+
+    def where(params = {})
+      validate_job_param_keys(params)
+
+      @status = params[:status] if params[:status].present?
+      @type = params[:type] if params[:type].present?
+      @cluster_name = params[:cluster_name] if params[:cluster_name].present?
+      @label = params[:label] if params[:label].present?
+      @cluster_rebooted = params[:cluster_rebooted]
+
+      self
+    end
+
+    def order(o)
+      result = o.downcase.to_s
+      unless ['asc', 'desc'].include?(result)
+        raise Dag::Client::ParameterInvalid.new("Invalid order condition: #{o}")
+      end
+
+      @order = result
+
+      self
+    end
+
+    def limit(number = 100)
+      @limit = number.to_i
+      max = 100
+
+      if number > max
+        @max = []
+        (number / max).times { @max << max }
+        if rem = (number % max)
+          @max << rem if rem != 0
+        end
+      end
+
+      self
+    end
+
+    def each
+      if @limit
+        count, i = 0, 0
+        total = @max ? @max.inject(:+) : @limit
+      end
+
+      marker = nil
+      truncated = false
+      begin
+        @limit = @max[i] if @max
+        job_info_list = @api.query_info_list(make_options(marker))
+        job_info_list['queries'].each do |job_info|
+          yield Dag::Job.new(@api, job_info)
+        end
+        truncated = job_info_list['isTruncated']
+        marker = job_info_list['nextMarker']
+
+        if @limit
+          i += 1
+          count += @limit
+          break if total <= count
+        end
+      end while truncated
+    end
+
+    private
+
+    def make_options(marker = nil)
+      options = { max: 100 }
+
+      if marker
+        options = options.merge(marker: marker)
+      end
+
+      if @limit
+        options = options.merge(max: @limit)
+      end
+
+      if @order
+        options = options.merge(order: @order)
+      end
+
+      if @status
+        status = @status.respond_to?(:join) ? @status.join(",") : @status
+        options = options.merge(status: status)
+      end
+
+      if @type
+        options = options.merge(type: @type)
+      end
+
+      if @cluster_name
+        options = options.merge(cluster_name: @cluster_name)
+      end
+
+      if @label
+        options = options.merge(label_prefix: @label)
+      end
+
+      unless @cluster_rebooted.nil?
+        options = options.merge(cluster_rebooted: @cluster_rebooted)
+      end
+
+      options
+    end
+  end
+end

data/lib/dag/client/model/object.rb

@@ -0,0 +1,56 @@
+module Dag
+  class Object < Model
+    def initialize(api, bucket_name, object_name, opts = {})
+      super(api)
+      @bucket = bucket_name
+      @name = object_name
+      @opts = opts
+    end
+
+    def name
+      @name
+    end
+
+    def size
+      @opts["Size"][0].to_i
+    end
+
+    def etag
+      @opts["ETag"][0]
+    end
+
+    def lastmodified
+      Time.parse(@opts["LastModified"][0])
+    end
+
+    def read(range = nil)
+      @api.get_object(@bucket, @name, range)
+    end
+
+    def write(data, options = {})
+      if data.is_a?(String)
+        data = StringIO.new(data)
+      elsif data.is_a?(Pathname)
+        data = File.open(data, 'rb')
+      elsif data.respond_to?(:read) && data.respond_to?(:eof?)
+      end
+
+      unless options[:multipart]
+        @api.create_object(@bucket, @name, options) do
+          data
+        end
+      else
+        @api.create_multipart_object(@bucket, @name, options) do
+          data
+        end
+      end
+      data.close
+
+      nil
+    end
+
+    def delete
+      @api.delete_object(@bucket, @name)
+    end
+  end
+end