queuery_client 0.9.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 312f2cfd1168a1af9b581ca43dcaad9e46ac7377
-  data.tar.gz: e0e6c8247b14c92f9145a7155f18d619cdb8541d
+SHA256:
+  metadata.gz: c873a091add34619c21ef63bd324a41205dfdc4f509a9b4574a2a31c2c121e4b
+  data.tar.gz: 5b184dba90213ba0f022302ae2dc99e59ec491189e4c7f65c5c05b24d3497ec3
 SHA512:
-  metadata.gz: 5e20bbf161970e050375276242206cc1206d733c83691c8f86a66d45a78a0fd475f351134f9a1f8ddeadcfa88222b67a148e3ebbd6de80daa52eb3209208fdc7
-  data.tar.gz: 73306d35879474ba8d92ad077f4d6f47279041332979abe4d28302ae2d2e964795b69d22d2f44b9ca005021dce7c1ba53fdd34681dc3ac77024102c686b7dd20
+  metadata.gz: 59fb860a84e25e604f160cd8209aecebd742b1a4e36fd84e3591bc4b7a0bc4f6c8ecb3482eb001c22eae379bec109ebf78ebe78a9f435a1c99b05f5e86299a56
+  data.tar.gz: a56489e0fa77068fc548f9b4c98fc9215f962c80ec2fe197f1958e05538992884ddc7aaa76bd2f86c43980db9c31437f24485f8371ea4739c867ce7837bd7c45

data/lib/queuery_client.rb

@@ -3,7 +3,8 @@ require "queuery_client/configuration"
 require "queuery_client/basic_auth_garage_client"
 require "queuery_client/query_error"
 require "queuery_client/client"
-require "queuery_client/queuery_data_file_bundle"
+require "queuery_client/url_data_file_bundle"
+require "queuery_client/s3_data_file_bundle"
 
 module QueueryClient
   class << self

data/lib/queuery_client/client.rb

@@ -32,7 +32,7 @@ module QueueryClient
       query = query_and_wait(select_stmt, values)
       case query.status
       when 'success'
-        QueueryDataFileBundle.new(
+        UrlDataFileBundle.new(
           query.data_file_urls,
           s3_prefix: query.s3_prefix,
         )

data/lib/queuery_client/data_file.rb

@@ -0,0 +1,26 @@
+require 'redshift_csv_file'
+require 'zlib'
+
+module QueueryClient
+  class DataFile
+    def data_object?
+      /\.csv(?:\.|\z)/ =~ File.basename(key)
+    end
+
+    def gzipped_object?
+      File.extname(key) == '.gz'
+    end
+
+    def each_row(&block)
+      f = open
+      begin
+        if gzipped_object?
+          f = Zlib::GzipReader.new(f)
+        end
+        RedshiftCsvFile.new(f).each(&block)
+      ensure
+        f.close
+      end
+    end
+  end
+end

data/lib/queuery_client/data_file_bundle.rb

@@ -0,0 +1,15 @@
+module QueueryClient
+  class DataFileBundle
+    # abstract data_files :: [DataFile]
+
+    def each_row(&block)
+      data_files.each do |file|
+        if file.data_object?
+          file.each_row(&block)
+        end
+      end
+    end
+
+    alias each each_row
+  end
+end

data/lib/queuery_client/s3_data_file.rb

@@ -0,0 +1,18 @@
+require 'queuery_client/data_file'
+require 'forwardable'
+
+module QueueryClient
+  class S3DataFile < DataFile
+    extend Forwardable
+
+    def initialize(object)
+      @object = object
+    end
+
+    def_delegators '@object', :url, :key, :presigned_url
+
+    def open
+      @object.get.body
+    end
+  end
+end

data/lib/queuery_client/s3_data_file_bundle.rb

@@ -0,0 +1,28 @@
+require 'queuery_client/data_file_bundle'
+require 'queuery_client/s3_data_file'
+require 'aws-sdk-s3'
+require 'logger'
+
+module QueueryClient
+  class S3DataFileBundle < DataFileBundle
+    def initialize(bucket, prefix, s3_client: nil, logger: Logger.new($stderr))
+      @s3_client = s3_client || Aws::S3::Client.new   # Use env to inject credentials
+      @bucket = bucket
+      @prefix = prefix
+      @logger = logger
+    end
+
+    attr_reader :bucket
+    attr_reader :prefix
+    attr_reader :logger
+
+    def url
+      "s3://#{@bucket}/#{@prefix}"
+    end
+
+    def data_files
+      b = Aws::S3::Resource.new(client: @s3_client).bucket(@bucket)
+      b.objects(prefix: @prefix).map {|obj| S3DataFile.new(obj) }
+    end
+  end
+end

data/lib/queuery_client/url_data_file.rb

@@ -0,0 +1,27 @@
+require 'queuery_client/data_file'
+require 'net/http'
+require 'stringio'
+
+module QueueryClient
+  class UrlDataFile < DataFile
+    def initialize(url)
+      @url = url
+    end
+
+    attr_reader :url
+
+    def key
+      @url.path
+    end
+
+    def open
+      http = Net::HTTP.new(@url.host, @url.port)
+      http.use_ssl = (@url.scheme.downcase == 'https')
+      content = http.start {
+        res = http.get(@url.request_uri)
+        res.body
+      }
+      StringIO.new(content)
+    end
+  end
+end

data/lib/queuery_client/url_data_file_bundle.rb

@@ -0,0 +1,32 @@
+require 'queuery_client/data_file_bundle'
+require 'queuery_client/url_data_file'
+require 'uri'
+require 'logger'
+
+module QueueryClient
+  class UrlDataFileBundle < DataFileBundle
+    def initialize(urls, s3_prefix:, logger: Logger.new($stderr))
+      raise ArgumentError, 'no URL given' if urls.empty?
+      @data_files = urls.map {|url| UrlDataFile.new(URI.parse(url)) }
+      @s3_prefix = s3_prefix
+      @logger = logger
+    end
+
+    attr_reader :data_files
+    attr_reader :s3_prefix
+
+    def url
+      uri = data_files.first.url.dup
+      uri.query = nil
+      uri.path = File.dirname(uri.path)
+      uri.to_s
+    end
+
+    def direct(bucket_opts = {}, bundle_opts = {})
+      s3_uri = URI.parse(s3_prefix)
+      bucket = s3_uri.host
+      prefix = s3_uri.path[1..-1] # trim heading slash
+      S3DataFileBundle.new(bucket, prefix)
+    end
+  end
+end

data/lib/queuery_client/version.rb

@@ -1,3 +1,3 @@
 module QueueryClient
-  VERSION = "0.9.0"
+  VERSION = "1.0.0"
 end

data/queuery_client.gemspec

@@ -29,7 +29,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency 'garage_client'
-  spec.add_dependency "redshift-connector-data_file", ">= 7.1"
+  spec.add_dependency 'redshift_csv_file'
+  spec.add_dependency 'aws-sdk-s3'
   spec.add_development_dependency "bundler", "~> 1.13"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "pry"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: queuery_client
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Hidekazu Kobayashi
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-04-04 00:00:00.000000000 Z
+date: 2019-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: garage_client
@@ -25,19 +25,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: redshift-connector-data_file
+  name: redshift_csv_file
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '7.1'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '7.1'
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -97,11 +111,15 @@ files:
 - lib/queuery_client/basic_auth_garage_client.rb
 - lib/queuery_client/client.rb
 - lib/queuery_client/configuration.rb
+- lib/queuery_client/data_file.rb
+- lib/queuery_client/data_file_bundle.rb
 - lib/queuery_client/query_error.rb
-- lib/queuery_client/queuery_data_file_bundle.rb
+- lib/queuery_client/s3_data_file.rb
+- lib/queuery_client/s3_data_file_bundle.rb
+- lib/queuery_client/url_data_file.rb
+- lib/queuery_client/url_data_file_bundle.rb
 - lib/queuery_client/version.rb
 - queuery_client.gemspec
-- queuery_client.rb
 homepage: https://github.com/bricolages/queuery_client
 licenses:
 - MIT
@@ -122,8 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Client library for Queuery Redshift HTTP API

data/lib/queuery_client/queuery_data_file_bundle.rb

@@ -1,30 +0,0 @@
-require 'redshift_connector/url_data_file_bundle'
-require 'redshift_connector/s3_bucket'
-require 'redshift_connector/s3_data_file_bundle'
-require 'uri'
-
-module QueueryClient
-  class QueueryDataFileBundle < RedshiftConnector::UrlDataFileBundle
-    def initialize(url, s3_prefix:, **args)
-      super(url, **args)
-      @s3_prefix = s3_prefix
-    end
-
-    attr_reader :s3_prefix
-
-    def url
-      uri = data_files.first.url.dup
-      uri.query = nil
-      uri.path = File.dirname(uri.path)
-      uri.to_s
-    end
-
-    def direct(bucket_opts = {}, bundle_opts = {})
-      s3_uri = URI.parse(s3_prefix)
-      bucket_name = s3_uri.host
-      prefix = s3_uri.path[1..-1] # trim heading slash
-      bucket = RedshiftConnector::S3Bucket.new(bucket: bucket_name, prefix: prefix)
-      RedshiftConnector::S3DataFileBundle.new(bucket, prefix, format: :redshift_csv)
-    end
-  end
-end

data/queuery_client.rb

@@ -1,64 +0,0 @@
-require 'uri'
-require 'net/http'
-
-class QueueryClient
-  class << self
-    attr_accessor :host
-    attr_accessor :port
-
-    def query(select_stmt)
-      client = new(
-        host: host,
-        port: port,
-      )
-
-      res = client.query_and_wait(select_stmt)
-      RedshiftConnector::UrlDataFileBundle.new(res['data_objects'])
-    end
-  end
-
-  def initialize(host:, port: 80)
-    @host = host
-    @port = port
-  end
-
-  def request(req)
-    res = Net::HTTP.new(@host, @port).request(req)
-    JSON.parse(res.body)
-  end
-
-  def request_post(path, params = {})
-    post_req = Net::HTTP::Post.new(path)
-    post_req.form_data = params
-    request(post_req)
-  end
-
-  def request_get(path)
-    get_req = Net::HTTP::Get.new(path)
-    request(get_req)
-  end
-
-  def query(select_stmt)
-    request_post('/queries', { q: select_stmt })
-  end
-
-  def status(job_id)
-    request_get("/queries/#{job_id}")
-  end
-
-  def wait_for(job_id)
-    loop do
-      res = status(job_id)
-      case res['status']
-      when 'success', 'failed'
-        return res
-      end
-      sleep 1
-    end
-  end
-
-  def query_and_wait(select_stmt)
-    res = query(select_stmt)
-    wait_for(res['job_id'])
-  end
-end