redshift_connector 8.0.0 → 8.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6dfd6aa65bfc44fca5d3e0dc7e042911344be4213764e267136f7259b3529a90
-  data.tar.gz: f12016b3044c1199e0b0e3bba4c8e8826e6a72b80d40088d849110dd4624607f
+  metadata.gz: 9ceccb9545b9d59e71696a1adc38413e7685f1ea191805200f08d47ae2b301eb
+  data.tar.gz: 8e0c019d98d823b59614f02b14dd6b47e5b9adbe5483923269091f895b818a78
 SHA512:
-  metadata.gz: 68e8a3265255b168cbb40b1b200902922fb9641035533cf561ab69feee1d825d15dd999f3d0fb9dfbfc0b00349be662d713c1ca5ac3da418d49c44ad03771e20
-  data.tar.gz: 59cf47cc0aaec30d8025497dc11347904d50f9be8a5e68086d9c2952f773619b27be3c8ed5edf8a9f6708afe370ba1dfac7cb48c1602c7cbbb23d33e0457b054
+  metadata.gz: 76dda7e7a2fb23794fb056bf0aaa81aff5ecbb3d98bb46b70903de6ceb68d51073597baceeefac88fa9a4ed04df100a4ebec1b5e2cee23eeeed7a485ba7a992b
+  data.tar.gz: ec3ac1e0fb7416cd5c6216ee5b3df9c8c0bb6a4e28383effc99607bbba564f22aee2c7d025fcd7518d69165ec2599fbc2090e8c08c523e983f8f7fe0776838fd

data/RELEASE.md

@@ -1,5 +1,24 @@
 # Release Note
 
+## version 8.1.3
+- [fix] Mishandling time zone for timestamp type.
+
+## version 8.1.2
+- [new] Add type cast option instead of filter option for #transport_delta and #transport_all.
+
+## version 8.1.1
+- [fix] Boolean type cast was reverse.
+
+## version 8.1.0
+
+- [new] Add type cast option for queuery.
+
+## version 8.0.1
+
+- [fix] Loosen pg version restriction to allow >0.18 to support Rails 5.
+  Note that Redshift is PostgreSQL 8 compatible and pg >0.18 does not support PostgreSQL 8.
+  Now you can use any version of pg and most version of pg works with Redshift, but you must take a risk.
+
 ## version 8.0.0
 
 - [INCOMPATIBLE] This library is renamed to "redshift_connector".  Just modify your Gemfile from "redshift-connector" to "redshift_connector".

data/lib/redshift_connector/active_record_exporter.rb

@@ -4,7 +4,8 @@ require 'redshift_connector/logger'
 
 module RedshiftConnector
   class ActiveRecordExporter
-    def initialize(ds:, query:, bundle_params:, enable_sort: false, logger: RedshiftConnector.logger)
+    def initialize(ds:, query:, bundle_params:, enable_sort: false, enable_cast: false, logger: RedshiftConnector.logger)
+      raise ArgumentError, "ActiveRecordExporter does not support type cast" if enable_cast
       @ds = ds
       @query = query
       @bundle_params = bundle_params

data/lib/redshift_connector/connector.rb

@@ -49,13 +49,17 @@ module RedshiftConnector
         upsert_columns: nil,
         bucket: nil,
         txn_id: nil,
-        filter:,
+        filter: nil,
+        enable_cast: false,
         logger: RedshiftConnector.logger,
         quiet: false
     )
       unless src_table and dest_table
         raise ArgumentError, "missing :table, :src_table or :dest_table"
       end
+      raise ArgumentError, "filter and enable_cast are exclusive" if filter and enable_cast
+      raise ArgumentError, "either filter or enable_cast is required" unless filter or enable_cast
+
       logger = NullLogger.new if quiet
       bundle_params = DataFileBundleParams.new(
         bucket: bucket,
@@ -70,6 +74,7 @@ module RedshiftConnector
         table: src_table,
         columns: columns,
         condition: condition,
+        enable_cast: enable_cast,
         logger: logger
       )
       importer = Importer.for_delta_upsert(
@@ -119,10 +124,14 @@ module RedshiftConnector
         columns:,
         bucket: nil,
         txn_id: nil,
-        filter:,
+        filter: nil,
+        enable_cast: false,
         logger: RedshiftConnector.logger,
         quiet: false
     )
+    raise ArgumentError, "filter and enable_cast are exclusive" if filter and enable_cast
+    raise ArgumentError, "either filter or enable_cast is required" unless filter or enable_cast
+
       logger = NullLogger.new if quiet
       bundle_params = DataFileBundleParams.new(
         bucket: bucket,
@@ -136,6 +145,7 @@ module RedshiftConnector
         schema: schema,
         table: src_table,
         columns: columns,
+        enable_cast: enable_cast,
         logger: logger
       )
       importer = Importer.for_rebuild(

data/lib/redshift_connector/data_file_bundle_reader.rb

@@ -1,4 +1,5 @@
 require 'redshift_connector/logger'
+require 'redshift_connector/redshift_data_type'
 require 'forwardable'
 
 module RedshiftConnector
@@ -22,7 +23,14 @@ module RedshiftConnector
 
     def each_row(&block)
       each_object do |obj|
-        obj.each_row(&block)
+        if @bundle.respond_to?(:has_manifest?) && @bundle.has_manifest?
+          obj.each_row do |row|
+            yield RedshiftDataType.type_cast(row, @bundle.manifest_file)
+          end
+        else
+          obj.each_row(&block)
+        end
+
       end
     end
 
@@ -68,5 +76,6 @@ module RedshiftConnector
       yield buf unless buf.empty?
     end
     private :do_each_batch
+
   end
 end

data/lib/redshift_connector/exporter_builder.rb

@@ -8,14 +8,14 @@ module RedshiftConnector
       @exporter_class = exporter_class
     end
 
-    def build_for_table_delta(schema:, table:, condition:, columns:, bundle_params:, logger: RedshiftConnector.logger)
+    def build_for_table_delta(schema:, table:, condition:, columns:, enable_cast:, bundle_params:, logger: RedshiftConnector.logger)
       query = DeltaQuery.new(schema: schema, table: table, columns: columns, condition: condition)
-      @exporter_class.new(ds: @ds, query: query, bundle_params: bundle_params, logger: logger)
+      @exporter_class.new(ds: @ds, query: query, bundle_params: bundle_params, enable_cast: enable_cast, logger: logger)
     end
 
-    def build_for_table(schema:, table:, columns:, bundle_params:, logger: RedshiftConnector.logger)
+    def build_for_table(schema:, table:, columns:, enable_cast:, bundle_params:, logger: RedshiftConnector.logger)
       query = SelectAllQuery.new(schema: schema, table: table, columns: columns)
-      @exporter_class.new(ds: @ds, query: query, bundle_params: bundle_params, logger: logger)
+      @exporter_class.new(ds: @ds, query: query, bundle_params: bundle_params, enable_cast: enable_cast, logger: logger)
     end
 
     def build_for_query(
@@ -23,8 +23,10 @@ module RedshiftConnector
         table:,
         bucket: nil,
         query:,
+        query_params: [],
         txn_id: "#{Time.now.strftime('%Y%m%d_%H%M%S')}_#{$$}",
         enable_sort: false,
+        enable_cast: false,
         logger: RedshiftConnector.logger,
         quiet: false
     )
@@ -39,8 +41,10 @@ module RedshiftConnector
       @exporter_class.new(
         ds: @ds,
         query: ArbitraryQuery.new(query),
+        query_params: query_params,
         bundle_params: bundle_params,
         enable_sort: enable_sort,
+        enable_cast: enable_cast,
         logger: logger
       )
     end

data/lib/redshift_connector/redshift_data_type.rb

@@ -0,0 +1,38 @@
+
+module RedshiftConnector
+  module RedshiftDataType
+    FALSE_VALUES = [
+      false, 0,
+      "0", :"0",
+      "f", :f,
+      "F", :F,
+      "false", :false,
+      "FALSE", :FALSE,
+      "off", :off,
+      "OFF", :OFF,
+    ].to_set.freeze
+
+    def self.type_cast(row, manifest_file)
+      row.zip(manifest_file.column_types).map do |value, type|
+        next nil if (value == '' and type != 'character varing') # null becomes '' on unload
+
+        case type
+        when 'smallint', 'integer', 'bigint'
+          value.to_i
+        when 'numeric', 'double precision'
+          value.to_f
+        when 'character', 'character varying'
+          value
+        when 'timestamp without time zone', 'timestamp with time zone'
+          value # Ruby does not have a class without timezone
+        when 'date'
+          Date.parse(value)
+        when 'boolean'
+          FALSE_VALUES.include?(value) ? false : true
+        else
+          raise "not support data type: #{type}"
+        end
+      end
+    end
+  end
+end

data/lib/redshift_connector/version.rb

@@ -1,3 +1,3 @@
 module RedshiftConnector
-  VERSION = '8.0.0'
+  VERSION = '8.1.3'
 end

data/redshift_connector.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
   s.required_ruby_version = '>= 2.1.0'
   s.add_dependency 'activerecord'
   s.add_dependency 'activerecord-redshift'
-  s.add_dependency 'pg', '~> 0.18.0'
+  s.add_dependency 'pg', '>= 0.18.0'
   s.add_dependency 'activerecord-import'
   s.add_dependency 'redshift_csv_file', '~> 1.0'
   s.add_dependency 'aws-sdk-s3', '~> 1.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redshift_connector
 version: !ruby/object:Gem::Version
-  version: 8.0.0
+  version: 8.1.3
 platform: ruby
 authors:
 - Minero Aoki
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-07-23 00:00:00.000000000 Z
+date: 2021-11-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -42,14 +42,14 @@ dependencies:
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.18.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.18.0
 - !ruby/object:Gem::Dependency
@@ -159,6 +159,7 @@ files:
 - lib/redshift_connector/reader/exception.rb
 - lib/redshift_connector/reader/redshift_csv.rb
 - lib/redshift_connector/reader/tsv.rb
+- lib/redshift_connector/redshift_data_type.rb
 - lib/redshift_connector/s3_bucket.rb
 - lib/redshift_connector/s3_data_file.rb
 - lib/redshift_connector/s3_data_file_bundle.rb
@@ -168,7 +169,7 @@ homepage: https://github.com/bricolages/redshift_connector
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -183,8 +184,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.6
+signing_key:
 specification_version: 4
 summary: Redshift bulk data connector
 test_files: []