queuery_client 0.8.0 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: d9bdfe61f21a506cef4c09057e13510305826234
-  data.tar.gz: '068d8a19aea9e7a13dd1c2b997848363e5a66217'
+SHA256:
+  metadata.gz: a5aa7c59714b2972a888c845beaaf38e220fdc0173cd746a0a73f510a1c69afd
+  data.tar.gz: c876ff24ba9b2d6f489d480de5fc55b74d08ab2aa0229c4cb2a74ca0914053f9
 SHA512:
-  metadata.gz: 6cdc5d84c7ce94b3ed1c835b00f20d7893b0f08776eac198bb58951bf2def275692a3fd87f4324bbb033150c26984c50f9918bce201f4aa9c7137a33926b66ea
-  data.tar.gz: 50bc3ebad07bd41bed5c1a4c554155478d554a93c2a2887eb7693c319d0ab9d7cb7d52266121285d81139f4a30be9079144aed9d47debb92e2805439065fc086
+  metadata.gz: 0c154295e3c4c676de39ee43b544ec458285044c0d797f7ec199c39589023019c7f407623745a53d2da6016a9eb1b1594240e2a1f74d0093025e9d3d73d0ab56
+  data.tar.gz: f7340f67c56865d1c179d062801087931db880a6f9ccbc3f5d3109438ab277a2902000e253bf6d45f8ceebf47b91f2d27fb6709e2fe305f90201d6e9fba206c7

data/lib/queuery_client.rb

@@ -3,7 +3,8 @@ require "queuery_client/configuration"
 require "queuery_client/basic_auth_garage_client"
 require "queuery_client/query_error"
 require "queuery_client/client"
-require "queuery_client/queuery_data_file_bundle"
+require "queuery_client/url_data_file_bundle"
+require "queuery_client/s3_data_file_bundle"
 
 module QueueryClient
   class << self

data/lib/queuery_client/client.rb

@@ -7,6 +7,7 @@ module QueueryClient
     def execute_query(select_stmt, values)
       garage_client.post("/v1/queries", q: select_stmt, values: values)
     end
+    alias start_query execute_query
 
     def get_query(id)
       garage_client.get("/v1/queries/#{id}", fields: '__default__,s3_prefix')
@@ -32,7 +33,7 @@ module QueueryClient
       query = query_and_wait(select_stmt, values)
       case query.status
       when 'success'
-        QueueryDataFileBundle.new(
+        UrlDataFileBundle.new(
           query.data_file_urls,
           s3_prefix: query.s3_prefix,
         )
@@ -41,13 +42,21 @@ module QueueryClient
       end
     end
 
+    # poll_result returns the results only if the query has already successed.
+    def poll_result(id)
+      query = get_query(id)
+      get_query_result(query)
+    end
+
     def garage_client
       @garage_client ||= BasicAuthGarageClient.new(
         endpoint: options.endpoint,
         path_prefix: '/',
         login: options.token,
         password: options.token_secret
-      )
+      ).tap do |client|
+        client.headers['Host'] = options.host_header if options.host_header
+      end
     end
 
     def options
@@ -57,5 +66,21 @@ module QueueryClient
     def default_options
       QueueryClient.configuration
     end
+
+    private
+
+    def get_query_result(query)
+      case query.status
+      when 'pending', 'running'
+        nil
+      when 'success'
+        UrlDataFileBundle.new(
+          query.data_file_urls,
+          s3_prefix: query.s3_prefix,
+        )
+      when 'failure'
+        raise QueryError.new(query.error)
+      end
+    end
   end
 end

data/lib/queuery_client/configuration.rb

@@ -1,5 +1,8 @@
 module QueueryClient
   class Configuration
+    REQUIRED_KEYS = [:endpoint, :token, :token_secret]
+    OPTIONAL_KEYS = [:host_header]
+
     def initialize(options = {})
       @options = options
     end
@@ -12,11 +15,7 @@ module QueueryClient
       @options = nil
     end
 
-    [
-      :endpoint,
-      :token,
-      :token_secret,
-    ].each do |key|
+    REQUIRED_KEYS.each do |key|
       define_method(key) do
         options.fetch(key)
       end
@@ -26,6 +25,16 @@ module QueueryClient
       end
     end
 
+    OPTIONAL_KEYS.each do |key|
+      define_method(key) do
+        options[key]
+      end
+
+      define_method("#{key}=") do |value|
+        options[key] = value
+      end
+    end
+
     def merge(other)
       Configuration.new(to_h.merge(other.to_h))
     end

data/lib/queuery_client/data_file.rb

@@ -0,0 +1,32 @@
+require 'redshift_csv_file'
+require 'zlib'
+
+module QueueryClient
+  class DataFile
+    def data_object?
+      /\.csv(?:\.|\z)/ =~ File.basename(key)
+    end
+
+    def gzipped_object?
+      File.extname(key) == '.gz'
+    end
+
+    def each_row(&block)
+      return enum_for(:each_row) if !block_given?
+
+      f = open
+      begin
+        if gzipped_object?
+          f = Zlib::GzipReader.new(f)
+        end
+        RedshiftCsvFile.new(f).each do |row|
+          yield row
+        end
+      ensure
+        f.close
+      end
+
+      self
+    end
+  end
+end

data/lib/queuery_client/data_file_bundle.rb

@@ -0,0 +1,21 @@
+module QueueryClient
+  class DataFileBundle
+    # abstract data_files :: [DataFile]
+
+    def each_row(&block)
+      return enum_for(:each_row) if !block_given?
+
+      data_files.each do |file|
+        if file.data_object?
+          file.each_row do |row|
+            yield row
+          end
+        end
+      end
+
+      self
+    end
+
+    alias each each_row
+  end
+end

data/lib/queuery_client/s3_data_file.rb

@@ -0,0 +1,18 @@
+require 'queuery_client/data_file'
+require 'forwardable'
+
+module QueueryClient
+  class S3DataFile < DataFile
+    extend Forwardable
+
+    def initialize(object)
+      @object = object
+    end
+
+    def_delegators '@object', :url, :key, :presigned_url
+
+    def open
+      @object.get.body
+    end
+  end
+end

data/lib/queuery_client/s3_data_file_bundle.rb

@@ -0,0 +1,28 @@
+require 'queuery_client/data_file_bundle'
+require 'queuery_client/s3_data_file'
+require 'aws-sdk-s3'
+require 'logger'
+
+module QueueryClient
+  class S3DataFileBundle < DataFileBundle
+    def initialize(bucket, prefix, s3_client: nil, logger: Logger.new($stderr))
+      @s3_client = s3_client || Aws::S3::Client.new   # Use env to inject credentials
+      @bucket = bucket
+      @prefix = prefix
+      @logger = logger
+    end
+
+    attr_reader :bucket
+    attr_reader :prefix
+    attr_reader :logger
+
+    def url
+      "s3://#{@bucket}/#{@prefix}"
+    end
+
+    def data_files
+      b = Aws::S3::Resource.new(client: @s3_client).bucket(@bucket)
+      b.objects(prefix: @prefix).map {|obj| S3DataFile.new(obj) }
+    end
+  end
+end

data/lib/queuery_client/url_data_file.rb

@@ -0,0 +1,27 @@
+require 'queuery_client/data_file'
+require 'net/http'
+require 'stringio'
+
+module QueueryClient
+  class UrlDataFile < DataFile
+    def initialize(url)
+      @url = url
+    end
+
+    attr_reader :url
+
+    def key
+      @url.path
+    end
+
+    def open
+      http = Net::HTTP.new(@url.host, @url.port)
+      http.use_ssl = (@url.scheme.downcase == 'https')
+      content = http.start {
+        res = http.get(@url.request_uri)
+        res.body
+      }
+      StringIO.new(content)
+    end
+  end
+end

data/lib/queuery_client/url_data_file_bundle.rb

@@ -0,0 +1,33 @@
+require 'queuery_client/data_file_bundle'
+require 'queuery_client/url_data_file'
+require 'uri'
+require 'logger'
+
+module QueueryClient
+  class UrlDataFileBundle < DataFileBundle
+    def initialize(urls, s3_prefix:, logger: Logger.new($stderr))
+      raise ArgumentError, 'no URL given' if urls.empty?
+      @data_files = urls.map {|url| UrlDataFile.new(URI.parse(url)) }
+      @s3_prefix = s3_prefix
+      @logger = logger
+    end
+
+    attr_reader :data_files
+    attr_reader :s3_prefix
+    attr_reader :logger
+
+    def url
+      uri = data_files.first.url.dup
+      uri.query = nil
+      uri.path = File.dirname(uri.path)
+      uri.to_s
+    end
+
+    def direct(bucket_opts = {}, bundle_opts = {})
+      s3_uri = URI.parse(s3_prefix)
+      bucket = s3_uri.host
+      prefix = s3_uri.path[1..-1] # trim heading slash
+      S3DataFileBundle.new(bucket, prefix)
+    end
+  end
+end

data/lib/queuery_client/version.rb

@@ -1,3 +1,3 @@
 module QueueryClient
-  VERSION = "0.8.0"
+  VERSION = "1.0.3"
 end

data/queuery_client.gemspec

@@ -29,8 +29,9 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency 'garage_client'
-  spec.add_dependency "redshift-connector-data_file", ">= 7.1"
+  spec.add_dependency 'redshift_csv_file'
+  spec.add_dependency 'aws-sdk-s3'
   spec.add_development_dependency "bundler", "~> 1.13"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "pry"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: queuery_client
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 1.0.3
 platform: ruby
 authors:
 - Hidekazu Kobayashi
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-03-22 00:00:00.000000000 Z
+date: 2020-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: garage_client
@@ -25,19 +25,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: redshift-connector-data_file
+  name: redshift_csv_file
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '7.1'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '7.1'
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -58,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -97,11 +111,15 @@ files:
 - lib/queuery_client/basic_auth_garage_client.rb
 - lib/queuery_client/client.rb
 - lib/queuery_client/configuration.rb
+- lib/queuery_client/data_file.rb
+- lib/queuery_client/data_file_bundle.rb
 - lib/queuery_client/query_error.rb
-- lib/queuery_client/queuery_data_file_bundle.rb
+- lib/queuery_client/s3_data_file.rb
+- lib/queuery_client/s3_data_file_bundle.rb
+- lib/queuery_client/url_data_file.rb
+- lib/queuery_client/url_data_file_bundle.rb
 - lib/queuery_client/version.rb
 - queuery_client.gemspec
-- queuery_client.rb
 homepage: https://github.com/bricolages/queuery_client
 licenses:
 - MIT
@@ -122,8 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Client library for Queuery Redshift HTTP API

data/lib/queuery_client/queuery_data_file_bundle.rb

@@ -1,19 +0,0 @@
-require 'redshift_connector/url_data_file_bundle'
-
-module QueueryClient
-  class QueueryDataFileBundle < RedshiftConnector::UrlDataFileBundle
-    def initialize(url, s3_prefix:, **args)
-      super(url, **args)
-      @s3_prefix = s3_prefix
-    end
-
-    attr_reader :s3_prefix
-
-    def url
-      uri = data_files.first.url.dup
-      uri.query = nil
-      uri.path = File.dirname(uri.path)
-      uri.to_s
-    end
-  end
-end

data/queuery_client.rb

@@ -1,64 +0,0 @@
-require 'uri'
-require 'net/http'
-
-class QueueryClient
-  class << self
-    attr_accessor :host
-    attr_accessor :port
-
-    def query(select_stmt)
-      client = new(
-        host: host,
-        port: port,
-      )
-
-      res = client.query_and_wait(select_stmt)
-      RedshiftConnector::UrlDataFileBundle.new(res['data_objects'])
-    end
-  end
-
-  def initialize(host:, port: 80)
-    @host = host
-    @port = port
-  end
-
-  def request(req)
-    res = Net::HTTP.new(@host, @port).request(req)
-    JSON.parse(res.body)
-  end
-
-  def request_post(path, params = {})
-    post_req = Net::HTTP::Post.new(path)
-    post_req.form_data = params
-    request(post_req)
-  end
-
-  def request_get(path)
-    get_req = Net::HTTP::Get.new(path)
-    request(get_req)
-  end
-
-  def query(select_stmt)
-    request_post('/queries', { q: select_stmt })
-  end
-
-  def status(job_id)
-    request_get("/queries/#{job_id}")
-  end
-
-  def wait_for(job_id)
-    loop do
-      res = status(job_id)
-      case res['status']
-      when 'success', 'failed'
-        return res
-      end
-      sleep 1
-    end
-  end
-
-  def query_and_wait(select_stmt)
-    res = query(select_stmt)
-    wait_for(res['job_id'])
-  end
-end