queuery_client 1.0.3 → 1.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a5aa7c59714b2972a888c845beaaf38e220fdc0173cd746a0a73f510a1c69afd
-  data.tar.gz: c876ff24ba9b2d6f489d480de5fc55b74d08ab2aa0229c4cb2a74ca0914053f9
+  metadata.gz: 11b32a624cd5e1125520cc7f1d68d6048f551ebbdfbdb34a2d64520cb3ac89c2
+  data.tar.gz: a41d2d8a3148035caadaa435ee98a49db50be5382286c67c78d5faeba157dc1b
 SHA512:
-  metadata.gz: 0c154295e3c4c676de39ee43b544ec458285044c0d797f7ec199c39589023019c7f407623745a53d2da6016a9eb1b1594240e2a1f74d0093025e9d3d73d0ab56
-  data.tar.gz: f7340f67c56865d1c179d062801087931db880a6f9ccbc3f5d3109438ab277a2902000e253bf6d45f8ceebf47b91f2d27fb6709e2fe305f90201d6e9fba206c7
+  metadata.gz: 4c9303a966bc816b6fc95eede740ae9cc804f95f8538984726d5f54f96e2b2b9f985e9dc7ecf831a59fb9a334dd9759217c782386b68d0523336153649081f55
+  data.tar.gz: 61227ed70ae5620421ccee2445c901a6c6875dd0b9c9758a847b6a0091821e2b5f9906f211118c22419a0a3454a4ab373047ecac346c876984ef064f5c4351ae

data/lib/queuery_client/client.rb

@@ -4,37 +4,44 @@ module QueueryClient
       @options = options
     end
 
-    def execute_query(select_stmt, values)
-      garage_client.post("/v1/queries", q: select_stmt, values: values)
+    def execute_query(select_stmt, values, query_options)
+      garage_client.post("/v1/queries", q: select_stmt, values: values, enable_metadata: query_options[:enable_cast])
     end
     alias start_query execute_query
 
-    def get_query(id)
-      garage_client.get("/v1/queries/#{id}", fields: '__default__,s3_prefix')
+    def get_query(id, query_options)
+      query_option_fields = build_query_option_fields(query_options)
+      garage_client.get("/v1/queries/#{id}", fields: '__default__,s3_prefix' + query_option_fields)
     end
 
-    def wait_for(id)
+    MAX_POLLING_INTERVAL = 30
+
+    def wait_for(id, query_options)
+      n = 0
       loop do
-        query = get_query(id)
+        query = get_query(id, query_options)
         case query.status
         when 'success', 'failed'
           return query
         end
-        sleep 3
+        n += 1
+        sleep [3 * n, MAX_POLLING_INTERVAL].min
       end
     end
 
-    def query_and_wait(select_stmt, values)
-      query = execute_query(select_stmt, values)
-      wait_for(query.id)
+    def query_and_wait(select_stmt, values, query_options)
+      query = execute_query(select_stmt, values, query_options)
+      wait_for(query.id, query_options)
     end
 
-    def query(select_stmt, values)
-      query = query_and_wait(select_stmt, values)
+    def query(select_stmt, values, **query_options)
+      query = query_and_wait(select_stmt, values, query_options)
+      manifest_file_url = query.manifest_file_url if query_options[:enable_cast]
       case query.status
       when 'success'
         UrlDataFileBundle.new(
           query.data_file_urls,
+          manifest_file_url,
           s3_prefix: query.s3_prefix,
         )
       when 'failed'
@@ -76,11 +83,27 @@ module QueueryClient
       when 'success'
         UrlDataFileBundle.new(
           query.data_file_urls,
+          nil,
           s3_prefix: query.s3_prefix,
         )
       when 'failure'
         raise QueryError.new(query.error)
       end
     end
+
+    def build_query_option_fields(query_options)
+      enable_query_options = query_options.select{ |name, v| name if v }.keys
+      return '' if enable_query_options.empty?
+      query_option_fields = enable_query_options.map{ |option_name| convert_field_name(option_name) }
+      ',' + query_option_fields.join(',')
+    end
+
+    def convert_field_name(option_name)
+      case option_name
+        when :enable_cast
+          'manifest_file_url'
+        # add another option here if you need
+      end
+    end
   end
 end

data/lib/queuery_client/data_file.rb

@@ -2,7 +2,7 @@ require 'redshift_csv_file'
 require 'zlib'
 
 module QueueryClient
-  class DataFile
+  class DataFile # abstract class
     def data_object?
       /\.csv(?:\.|\z)/ =~ File.basename(key)
     end

data/lib/queuery_client/data_file_bundle.rb

@@ -1,6 +1,12 @@
+require 'queuery_client/redshift_data_type'
+require 'date'
+require 'time'
+
 module QueueryClient
   class DataFileBundle
     # abstract data_files :: [DataFile]
+    # abstract manifest_file :: ManifestFile
+    # abstract def has_manifest?
 
     def each_row(&block)
       return enum_for(:each_row) if !block_given?
@@ -8,14 +14,18 @@ module QueueryClient
       data_files.each do |file|
         if file.data_object?
           file.each_row do |row|
-            yield row
+            if has_manifest?
+              yield RedshiftDataType.type_cast(row, manifest_file)
+            else
+              yield row
+            end
           end
         end
       end
 
       self
     end
-
     alias each each_row
+
   end
 end

data/lib/queuery_client/manifest_file.rb

@@ -0,0 +1,20 @@
+require 'json'
+
+module QueueryClient
+  class ManifestFile # abstract class
+    def manifest_object?
+      /\.manifest(?:\.|\z)/ =~ File.basename(key)
+    end
+
+    def column_types
+      @column_types ||=
+        begin
+          f = open
+          j = JSON.load(f)
+          j['schema']['elements'].map{|x| x['type']['base']}
+        ensure
+          f.close
+        end
+    end
+  end
+end

data/lib/queuery_client/redshift_data_type.rb

@@ -0,0 +1,38 @@
+
+module QueueryClient
+  module RedshiftDataType
+    FALSE_VALUES = [
+      false, 0,
+      "0", :"0",
+      "f", :f,
+      "F", :F,
+      "false", :false,
+      "FALSE", :FALSE,
+      "off", :off,
+      "OFF", :OFF,
+    ].to_set.freeze
+
+    def self.type_cast(row, manifest_file)
+      row.zip(manifest_file.column_types).map do |value, type|
+        next nil if (value == '' and type != 'character varing') # null becomes '' on unload
+
+        case type
+        when 'smallint', 'integer', 'bigint'
+          value.to_i
+        when 'numeric', 'double precision'
+          value.to_f
+        when 'character', 'character varying'
+          value
+        when 'timestamp without time zone', 'timestamp with time zone'
+          value # Ruby does not have a class without timezone
+        when 'date'
+          Date.parse(value)
+        when 'boolean'
+          FALSE_VALUES.include?(value) ? false : true
+        else
+          raise "not support data type: #{type}"
+        end
+      end
+    end
+  end
+end

data/lib/queuery_client/s3_data_file_bundle.rb

@@ -1,5 +1,6 @@
 require 'queuery_client/data_file_bundle'
 require 'queuery_client/s3_data_file'
+require 'queuery_client/s3_manifest_file'
 require 'aws-sdk-s3'
 require 'logger'
 
@@ -22,7 +23,23 @@ module QueueryClient
 
     def data_files
       b = Aws::S3::Resource.new(client: @s3_client).bucket(@bucket)
-      b.objects(prefix: @prefix).map {|obj| S3DataFile.new(obj) }
+      b.objects(prefix: @prefix)
+        .select {|obj| obj.key.include?('_part_') }
+        .map {|obj| S3DataFile.new(obj) }
+    end
+
+    def manifest_file
+      b = Aws::S3::Resource.new(client: @s3_client).bucket(@bucket)
+      obj = b.object("#{@prefix}manifest")
+      if obj.exists?
+        S3ManifestFile.new(obj)
+      else
+        nil
+      end
+    end
+
+    def has_manifest?
+      !manifest_file.nil?
     end
   end
 end

data/lib/queuery_client/s3_manifest_file.rb

@@ -0,0 +1,18 @@
+require 'queuery_client/manifest_file'
+require 'forwardable'
+
+module QueueryClient
+  class S3ManifestFile < ManifestFile
+    extend Forwardable
+
+    def initialize(object)
+      @object = object
+    end
+
+    def_delegators '@object', :url, :key, :presigned_url
+
+    def open
+      @object.get.body
+    end
+  end
+end

data/lib/queuery_client/url_data_file_bundle.rb

@@ -1,18 +1,21 @@
 require 'queuery_client/data_file_bundle'
 require 'queuery_client/url_data_file'
+require 'queuery_client/url_manifest_file'
 require 'uri'
 require 'logger'
 
 module QueueryClient
   class UrlDataFileBundle < DataFileBundle
-    def initialize(urls, s3_prefix:, logger: Logger.new($stderr))
-      raise ArgumentError, 'no URL given' if urls.empty?
-      @data_files = urls.map {|url| UrlDataFile.new(URI.parse(url)) }
+    def initialize(data_urls, manifest_url, s3_prefix:, logger: Logger.new($stderr))
+      raise ArgumentError, 'no URL given' if data_urls.empty?
+      @data_files = data_urls.map {|url| UrlDataFile.new(URI.parse(url)) }
+      @manifest_file = UrlManifestFile.new(URI.parse(manifest_url)) if manifest_url
       @s3_prefix = s3_prefix
       @logger = logger
     end
 
     attr_reader :data_files
+    attr_reader :manifest_file
     attr_reader :s3_prefix
     attr_reader :logger
 
@@ -29,5 +32,9 @@ module QueueryClient
       prefix = s3_uri.path[1..-1] # trim heading slash
       S3DataFileBundle.new(bucket, prefix)
     end
+
+    def has_manifest?
+      !@manifest_file.nil?
+    end
   end
 end

data/lib/queuery_client/url_manifest_file.rb

@@ -0,0 +1,28 @@
+require 'queuery_client/manifest_file'
+require 'net/http'
+require 'stringio'
+require 'json'
+
+module QueueryClient
+  class UrlManifestFile < ManifestFile
+    def initialize(url)
+      @url = url
+    end
+
+    attr_reader :url
+
+    def key
+      @url.path
+    end
+
+    def open
+      http = Net::HTTP.new(@url.host, @url.port)
+      http.use_ssl = (@url.scheme.downcase == 'https')
+      content = http.start {
+        res = http.get(@url.request_uri)
+        res.body
+      }
+      StringIO.new(content)
+    end
+  end
+end

data/lib/queuery_client/version.rb

@@ -1,3 +1,3 @@
 module QueueryClient
-  VERSION = "1.0.3"
+  VERSION = "1.1.4"
 end

data/lib/queuery_client.rb

@@ -16,8 +16,8 @@ module QueueryClient
       configuration.instance_eval(&block)
     end
 
-    def query(select_stmt, values = [])
-      Client.new.query(select_stmt, values)
+    def query(select_stmt, values = [], enable_cast: false)
+      Client.new.query(select_stmt, values, enable_cast: enable_cast)
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: queuery_client
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.1.4
 platform: ruby
 authors:
 - Hidekazu Kobayashi
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-08-04 00:00:00.000000000 Z
+date: 2021-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: garage_client
@@ -94,7 +94,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - hidekazu-kobayashi@cookpad.com
 executables: []
@@ -113,11 +113,15 @@ files:
 - lib/queuery_client/configuration.rb
 - lib/queuery_client/data_file.rb
 - lib/queuery_client/data_file_bundle.rb
+- lib/queuery_client/manifest_file.rb
 - lib/queuery_client/query_error.rb
+- lib/queuery_client/redshift_data_type.rb
 - lib/queuery_client/s3_data_file.rb
 - lib/queuery_client/s3_data_file_bundle.rb
+- lib/queuery_client/s3_manifest_file.rb
 - lib/queuery_client/url_data_file.rb
 - lib/queuery_client/url_data_file_bundle.rb
+- lib/queuery_client/url_manifest_file.rb
 - lib/queuery_client/version.rb
 - queuery_client.gemspec
 homepage: https://github.com/bricolages/queuery_client
@@ -125,7 +129,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -140,8 +144,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.1.6
+signing_key:
 specification_version: 4
 summary: Client library for Queuery Redshift HTTP API
 test_files: []