queuery_client 1.0.3 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a5aa7c59714b2972a888c845beaaf38e220fdc0173cd746a0a73f510a1c69afd
-  data.tar.gz: c876ff24ba9b2d6f489d480de5fc55b74d08ab2aa0229c4cb2a74ca0914053f9
+  metadata.gz: 2a5b41cb805e7fc0864fd695adc519c6b7212c294383935af40301d88b5b8d63
+  data.tar.gz: d4e970371d23e36bf40d4cafdf9e7da0d886c25bb0882ce9e12fc2a971a0e2c0
 SHA512:
-  metadata.gz: 0c154295e3c4c676de39ee43b544ec458285044c0d797f7ec199c39589023019c7f407623745a53d2da6016a9eb1b1594240e2a1f74d0093025e9d3d73d0ab56
-  data.tar.gz: f7340f67c56865d1c179d062801087931db880a6f9ccbc3f5d3109438ab277a2902000e253bf6d45f8ceebf47b91f2d27fb6709e2fe305f90201d6e9fba206c7
+  metadata.gz: f61f86445e64fca2863ba474a6cfb27e87af761516b3fbb5394adbe6f61e23e4f9d81778cded02dc512bf495bb8b711099962b5fd63154880bcc3e746bf25f61
+  data.tar.gz: ee0f09b1e92729056857117a7fdf4fa422d13bae5168f521e6d2cd8ca8adc2152334c59fd826aff916369ee8db811c792a49087ddd08dfdb283c0a31445bab78

data/lib/queuery_client/client.rb

@@ -4,18 +4,19 @@ module QueueryClient
       @options = options
     end
 
-    def execute_query(select_stmt, values)
-      garage_client.post("/v1/queries", q: select_stmt, values: values)
+    def execute_query(select_stmt, values, query_options)
+      garage_client.post("/v1/queries", q: select_stmt, values: values, enable_metadata: query_options[:enable_cast])
     end
     alias start_query execute_query
 
-    def get_query(id)
-      garage_client.get("/v1/queries/#{id}", fields: '__default__,s3_prefix')
+    def get_query(id, query_options)
+      query_option_fields = build_query_option_fields(query_options)
+      garage_client.get("/v1/queries/#{id}", fields: '__default__,s3_prefix' + query_option_fields)
     end
 
-    def wait_for(id)
+    def wait_for(id, query_options)
       loop do
-        query = get_query(id)
+        query = get_query(id, query_options)
         case query.status
         when 'success', 'failed'
           return query
@@ -24,17 +25,19 @@ module QueueryClient
       end
     end
 
-    def query_and_wait(select_stmt, values)
-      query = execute_query(select_stmt, values)
-      wait_for(query.id)
+    def query_and_wait(select_stmt, values, query_options)
+      query = execute_query(select_stmt, values, query_options)
+      wait_for(query.id, query_options)
     end
 
-    def query(select_stmt, values)
-      query = query_and_wait(select_stmt, values)
+    def query(select_stmt, values, **query_options)
+      query = query_and_wait(select_stmt, values, query_options)
+      manifest_file_url = query.manifest_file_url if query_options[:enable_cast]
       case query.status
       when 'success'
         UrlDataFileBundle.new(
           query.data_file_urls,
+          manifest_file_url,
           s3_prefix: query.s3_prefix,
         )
       when 'failed'
@@ -76,11 +79,27 @@ module QueueryClient
       when 'success'
         UrlDataFileBundle.new(
           query.data_file_urls,
+          nil,
           s3_prefix: query.s3_prefix,
         )
       when 'failure'
         raise QueryError.new(query.error)
       end
     end
+
+    def build_query_option_fields(query_options)
+      enable_query_options = query_options.select{ |name, v| name if v }.keys
+      return '' if enable_query_options.empty?
+      query_option_fields = enable_query_options.map{ |option_name| convert_field_name(option_name) }
+      ',' + query_option_fields.join(',')
+    end
+
+    def convert_field_name(option_name)
+      case option_name
+        when :enable_cast
+          'manifest_file_url'
+        # add another option here if you need
+      end
+    end
   end
 end

data/lib/queuery_client/data_file.rb

@@ -2,7 +2,7 @@ require 'redshift_csv_file'
 require 'zlib'
 
 module QueueryClient
-  class DataFile
+  class DataFile # abstract class
     def data_object?
       /\.csv(?:\.|\z)/ =~ File.basename(key)
     end

data/lib/queuery_client/data_file_bundle.rb

@@ -1,6 +1,12 @@
+require 'queuery_client/redshift_data_type'
+require 'date'
+require 'time'
+
 module QueueryClient
   class DataFileBundle
     # abstract data_files :: [DataFile]
+    # abstract manifest_file :: ManifestFile
+    # abstract def has_manifest?
 
     def each_row(&block)
       return enum_for(:each_row) if !block_given?
@@ -8,14 +14,18 @@ module QueueryClient
       data_files.each do |file|
         if file.data_object?
           file.each_row do |row|
-            yield row
+            if has_manifest?
+              yield RedshiftDataType.type_cast(row, manifest_file)
+            else
+              yield row
+            end
           end
         end
       end
 
       self
     end
-
     alias each each_row
+
   end
 end

data/lib/queuery_client/manifest_file.rb

@@ -0,0 +1,20 @@
+require 'json'
+
+module QueueryClient
+  class ManifestFile # abstract class
+    def manifest_object?
+      /\.manifest(?:\.|\z)/ =~ File.basename(key)
+    end
+
+    def column_types
+      @column_types ||=
+        begin
+          f = open
+          j = JSON.load(f)
+          j['schema']['elements'].map{|x| x['type']['base']}
+        ensure
+          f.close
+        end
+    end
+  end
+end

data/lib/queuery_client/redshift_data_type.rb

@@ -0,0 +1,27 @@
+
+module QueueryClient
+  module RedshiftDataType
+    def self.type_cast(row, manifest_file)
+      row.zip(manifest_file.column_types).map do |value, type|
+        next nil if (value == '' and type != 'character varing') # null becomes '' on unload
+
+        case type
+        when 'smallint', 'integer', 'bigint'
+          value.to_i
+        when 'numeric', 'double precision'
+          value.to_f
+        when 'character', 'character varying'
+          value
+        when 'timestamp without time zone', 'timestamp with time zone'
+          Time.parse(value)
+        when 'date'
+          Date.parse(value)
+        when 'boolean'
+          value == 'true' ? true : false
+        else
+          raise "not support data type: #{type}"
+        end
+      end
+    end
+  end
+end

data/lib/queuery_client/s3_data_file_bundle.rb

@@ -1,5 +1,6 @@
 require 'queuery_client/data_file_bundle'
 require 'queuery_client/s3_data_file'
+require 'queuery_client/s3_manifest_file'
 require 'aws-sdk-s3'
 require 'logger'
 
@@ -24,5 +25,15 @@ module QueueryClient
       b = Aws::S3::Resource.new(client: @s3_client).bucket(@bucket)
       b.objects(prefix: @prefix).map {|obj| S3DataFile.new(obj) }
     end
+
+    def manifest_file
+      b = Aws::S3::Resource.new(client: @s3_client).bucket(@bucket)
+      obj = b.object("#{@prefix}manifest")
+      S3ManifestFile.new(obj)
+    end
+
+    def has_manifest?
+      !manifest_file.nil?
+    end
   end
 end

data/lib/queuery_client/s3_manifest_file.rb

@@ -0,0 +1,18 @@
+require 'queuery_client/manifest_file'
+require 'forwardable'
+
+module QueueryClient
+  class S3ManifestFile < ManifestFile
+    extend Forwardable
+
+    def initialize(object)
+      @object = object
+    end
+
+    def_delegators '@object', :url, :key, :presigned_url
+
+    def open
+      @object.get.body
+    end
+  end
+end

data/lib/queuery_client/url_data_file_bundle.rb

@@ -1,18 +1,21 @@
 require 'queuery_client/data_file_bundle'
 require 'queuery_client/url_data_file'
+require 'queuery_client/url_manifest_file'
 require 'uri'
 require 'logger'
 
 module QueueryClient
   class UrlDataFileBundle < DataFileBundle
-    def initialize(urls, s3_prefix:, logger: Logger.new($stderr))
-      raise ArgumentError, 'no URL given' if urls.empty?
-      @data_files = urls.map {|url| UrlDataFile.new(URI.parse(url)) }
+    def initialize(data_urls, manifest_url, s3_prefix:, logger: Logger.new($stderr))
+      raise ArgumentError, 'no URL given' if data_urls.empty?
+      @data_files = data_urls.map {|url| UrlDataFile.new(URI.parse(url)) }
+      @manifest_file = UrlManifestFile.new(URI.parse(manifest_url)) if manifest_url
       @s3_prefix = s3_prefix
       @logger = logger
     end
 
     attr_reader :data_files
+    attr_reader :manifest_file
     attr_reader :s3_prefix
     attr_reader :logger
 
@@ -29,5 +32,9 @@ module QueueryClient
       prefix = s3_uri.path[1..-1] # trim heading slash
       S3DataFileBundle.new(bucket, prefix)
     end
+
+    def has_manifest?
+      !@manifest_file.nil?
+    end
   end
 end

data/lib/queuery_client/url_manifest_file.rb

@@ -0,0 +1,28 @@
+require 'queuery_client/manifest_file'
+require 'net/http'
+require 'stringio'
+require 'json'
+
+module QueueryClient
+  class UrlManifestFile < ManifestFile
+    def initialize(url)
+      @url = url
+    end
+
+    attr_reader :url
+
+    def key
+      @url.path
+    end
+
+    def open
+      http = Net::HTTP.new(@url.host, @url.port)
+      http.use_ssl = (@url.scheme.downcase == 'https')
+      content = http.start {
+        res = http.get(@url.request_uri)
+        res.body
+      }
+      StringIO.new(content)
+    end
+  end
+end

data/lib/queuery_client/version.rb

@@ -1,3 +1,3 @@
 module QueueryClient
-  VERSION = "1.0.3"
+  VERSION = "1.1.0"
 end

data/lib/queuery_client.rb

@@ -16,8 +16,8 @@ module QueueryClient
       configuration.instance_eval(&block)
     end
 
-    def query(select_stmt, values = [])
-      Client.new.query(select_stmt, values)
+    def query(select_stmt, values = [], enable_cast: false)
+      Client.new.query(select_stmt, values, enable_cast: enable_cast)
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: queuery_client
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.1.0
 platform: ruby
 authors:
 - Hidekazu Kobayashi
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-08-04 00:00:00.000000000 Z
+date: 2021-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: garage_client
@@ -94,7 +94,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - hidekazu-kobayashi@cookpad.com
 executables: []
@@ -113,11 +113,15 @@ files:
 - lib/queuery_client/configuration.rb
 - lib/queuery_client/data_file.rb
 - lib/queuery_client/data_file_bundle.rb
+- lib/queuery_client/manifest_file.rb
 - lib/queuery_client/query_error.rb
+- lib/queuery_client/redshift_data_type.rb
 - lib/queuery_client/s3_data_file.rb
 - lib/queuery_client/s3_data_file_bundle.rb
+- lib/queuery_client/s3_manifest_file.rb
 - lib/queuery_client/url_data_file.rb
 - lib/queuery_client/url_data_file_bundle.rb
+- lib/queuery_client/url_manifest_file.rb
 - lib/queuery_client/version.rb
 - queuery_client.gemspec
 homepage: https://github.com/bricolages/queuery_client
@@ -125,7 +129,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -140,8 +144,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.1.6
+signing_key:
 specification_version: 4
 summary: Client library for Queuery Redshift HTTP API
 test_files: []