activerecord-copy 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3b4dcb1725736ffe1229685b13945ebf201578f5
-  data.tar.gz: d0f4faf9fb27cf0e9712853caddad91140071018
+  metadata.gz: 050aa5f637d6f6d6a155e4d9f90c30062d62398e
+  data.tar.gz: b6c02d2fc46f8bfd2021b60ece7f42a19d2822ab
 SHA512:
-  metadata.gz: f15a8391484bd16250b6b4ff6285df87c5c682817bed3a8efa90add86993edd98ae25192532415edd17e3f573804f1b5f65b8c6364a160b60d76e5a6b4d928a5
-  data.tar.gz: 9a40ea85b320b12c9648bd00b5938e9f98f930b88d30d3621bb02e05cee3ad7807c21ccb9fd28ba18a88d7ddcdcab40d54d236411458c313e5e9ef5dd8bbf0db
+  metadata.gz: 7ffb2c84b4fa363f904dfd82cccb99116d96227b72b5cd871a6c647057b6fa229ad7f1c2ff3645e3ad1a124c7c83ce25486a0203837fab62a8b02745d186f833
+  data.tar.gz: 851ad2c3cfc6a986dbc184e2672b369cf83d3dadc8b33ee4709ff58f1451c8c86d82be1bd7f3d9ba8c5a8dd66c2f8cf0ee3897f2e422f6b42d0bf9028713eb82

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 1.1.0       2018-05-24
+
+* Add support for range data types
+* Fix bugs in NUMERIC data type encoding
+
+
 ## 1.0.1       2017-07-22
 
 * Explicitly include ActiveRecord to ensure copy_from_client gets defined

data/lib/activerecord-copy/encode_for_copy.rb

@@ -1,7 +1,23 @@
 require 'tempfile'
 require 'stringio'
+require 'ipaddr'
 
 module ActiveRecordCopy
+  class IntermediateBuffer
+    attr_reader :bytes
+    def initialize
+      @bytes = ''
+    end
+
+    def write(b)
+      @bytes += b
+    end
+
+    def size
+      @bytes.size
+    end
+  end
+
   class EncodeForCopy
     def initialize(options = {})
       @options = options
@@ -62,19 +78,9 @@ module ActiveRecordCopy
       io.write(buf)
     end
 
-    def encode_field(io, field, index, depth = 0)
-      # Nil is an exception in that any kind of field type can have a nil value transmitted
-      if field.nil?
-        io.write([-1].pack(PACKED_UINT_32))
-        return
-      end
-
-      if field.is_a?(Array) && ![:json, :jsonb].include?(@column_types[index])
-        encode_array(io, field, index)
-        return
-      end
-
-      case @column_types[index]
+    # Primitive types that can also appear in ranges/arrays/etc
+    def write_simple_field(io, field, type)
+      case type
       when :bigint
         buf = [field.to_i].pack(PACKED_UINT_64)
         write_field(io, buf)
@@ -89,6 +95,32 @@ module ActiveRecordCopy
       when :float
         buf = [field].pack(PACKED_FLOAT_64)
         write_field(io, buf)
+      when :timestamp, :timestamptz
+        buf = [(field.tv_sec * 1_000_000 + field.tv_usec - POSTGRES_EPOCH_TIME).to_i].pack(PACKED_UINT_64)
+        write_field(io, buf)
+      when :date
+        buf = [(field - Date.new(2000, 1, 1)).to_i].pack(PACKED_UINT_32)
+        write_field(io, buf)
+      else
+        raise Exception, "Unsupported simple type: #{type}"
+      end
+    end
+
+    def encode_field(io, field, index, depth = 0)
+      # Nil is an exception in that any kind of field type can have a nil value transmitted
+      if field.nil?
+        io.write([-1].pack(PACKED_UINT_32))
+        return
+      end
+
+      if field.is_a?(Array) && ![:json, :jsonb].include?(@column_types[index])
+        encode_array(io, field, index)
+        return
+      end
+
+      case @column_types[index]
+      when :bigint, :integer, :smallint, :numeric, :float
+        write_simple_field(io, field, @column_types[index])
       when :uuid
         buf = [field.delete('-')].pack(PACKED_HEX_STRING)
         write_field(io, buf)
@@ -101,6 +133,8 @@ module ActiveRecordCopy
         write_field(io, buf)
       when :jsonb
         encode_jsonb(io, field)
+      when :int4range, :int8range, :numrange, :tsrange, :tstzrange, :daterange
+        encode_range(io, field, @column_types[index])
       else
         encode_based_on_input(io, field, index, depth)
       end
@@ -135,13 +169,26 @@ module ActiveRecordCopy
         io.write([hash_io.pos].pack(PACKED_UINT_32)) # size of hstore data
         io.write(hash_io.string)
       when Time
-        buf = [(field.tv_sec * 1_000_000 + field.tv_usec - POSTGRES_EPOCH_TIME).to_i].pack(PACKED_UINT_64)
-        write_field(io, buf)
+        write_simple_field(io, field, :timestamp)
       when Date
-        buf = [(field - Date.new(2000, 1, 1)).to_i].pack(PACKED_UINT_32)
-        write_field(io, buf)
+        write_simple_field(io, field, :date)
       when IPAddr
         encode_ip_addr(io, field)
+      when Range
+        range_type = case field.begin
+                     when Integer
+                       :int4range
+                     when Float
+                       :numrange
+                     when Time
+                       :tstzrange
+                     when Date
+                       :daterange
+                     else
+                       raise Exception, "Unsupported range input type #{field.begin.class.name} for index #{index}"
+                     end
+
+        encode_range(io, field, range_type)
       else
         raise Exception, "Unsupported Format: #{field.class.name}"
       end
@@ -220,6 +267,49 @@ module ActiveRecordCopy
       io.write(ip_addr.hton)
     end
 
+    # From the Postgres source:
+    # Binary representation: The first byte is the flags, then the lower bound
+    # (if present), then the upper bound (if present).  Each bound is represented
+    # by a 4-byte length header and the binary representation of that bound (as
+    # returned by a call to the send function for the subtype).
+    RANGE_LB_INC = 0x02 # lower bound is inclusive
+    RANGE_UB_INC = 0x04 # upper bound is inclusive
+    RANGE_LB_INF = 0x08 # lower bound is -infinity
+    RANGE_UB_INF = 0x10 # upper bound is +infinity
+    def encode_range(io, range, range_type)
+      field_data_type = case range_type
+                        when :int4range
+                          :integer
+                        when :int8range
+                          :bigint
+                        when :numrange
+                          :numeric
+                        when :tsrange
+                          :timestamp
+                        when :tstzrange
+                          :timestamptz
+                        when :daterange
+                          :date
+                        else
+                          raise Exception, "Unsupported range type: #{range_type}"
+                        end
+      flags = 0
+      flags |= RANGE_LB_INC # Ruby ranges always include the lower bound
+      flags |= RANGE_UB_INC unless range.exclude_end?
+      flags |= RANGE_LB_INF if range.begin.respond_to?(:infinite?) && range.begin.infinite?
+      flags |= RANGE_UB_INF if range.end.respond_to?(:infinite?) && range.end.infinite?
+      tmp_io = IntermediateBuffer.new
+      tmp_io.write([flags].pack(PACKED_UINT_8))
+      if range.begin && (!range.begin.respond_to?(:infinite?) || !range.begin.infinite?)
+        write_simple_field(tmp_io, range.begin, field_data_type)
+      end
+      if range.end && (!range.end.respond_to?(:infinite?) || !range.end.infinite?)
+        write_simple_field(tmp_io, range.end, field_data_type)
+      end
+      io.write([tmp_io.size].pack(PACKED_UINT_32))
+      io.write(tmp_io.bytes)
+    end
+
     def encode_jsonb(io, field)
       buf = field.to_json.encode(UTF_8_ENCODING)
       io.write([1 + buf.bytesize].pack(PACKED_UINT_32))
@@ -227,7 +317,19 @@ module ActiveRecordCopy
       io.write(buf)
     end
 
-    NUMERIC_DEC_DIGITS = 4 # NBASE=10000
+    NUMERIC_NBASE = 10000
+    def base10_to_base10000(intval)
+      digits = []
+      loop do
+        newintval = intval / NUMERIC_NBASE
+        digits << intval - newintval * NUMERIC_NBASE
+        intval = newintval
+        break if intval == 0
+      end
+      digits
+    end
+
+    NUMERIC_DEC_DIGITS = 4
     def encode_numeric(io, field)
       float_str = field.to_s
       digits_base10 = float_str.scan(/\d/).map(&:to_i)
@@ -235,8 +337,11 @@ module ActiveRecordCopy
       sign          = field < 0.0 ? 0x4000 : 0
       dscale        = digits_base10.size - weight_base10
 
-      digits_before_decpoint = digits_base10[0..weight_base10].reverse.each_slice(NUMERIC_DEC_DIGITS).map { |d| d.reverse.map(&:to_s).join.to_i }.reverse
-      digits_after_decpoint  = digits_base10[weight_base10..-1].each_slice(NUMERIC_DEC_DIGITS).map { |d| d.map(&:to_s).join.to_i }
+      int_part, frac_part = float_str.split('.')
+      frac_part += '0' * (NUMERIC_DEC_DIGITS - frac_part.size % NUMERIC_DEC_DIGITS) # Add trailing zeroes so digit calculations are correct
+
+      digits_before_decpoint = base10_to_base10000(int_part.to_i)
+      digits_after_decpoint = base10_to_base10000(frac_part.to_i).reverse
 
       weight = digits_before_decpoint.size - 1
       digits = digits_before_decpoint + digits_after_decpoint

data/lib/activerecord-copy/version.rb

@@ -1,3 +1,3 @@
 module ActiveRecordCopy
-  VERSION = '1.0.1'.freeze
+  VERSION = '1.1.0'.freeze
 end

data/spec/verify_data_formats_spec.rb

@@ -412,4 +412,27 @@ describe 'generating data' do
     # File.open('spec/fixtures/output.dat', 'w:ASCII-8BIT') {|out| out.write(str) }
     expect(str).to eq existing_data
   end
+
+  # CREATE TABLE test(i4r int4range, i8r int8range, nr numrange, tr tsrange, tzr tstzrange, dr daterange);
+  # INSERT INTO test VALUES ('[12, 14)', '[223372033854775802, 223372033854775810)', '[12.5,13.88211]', '[2010-01-01 15:20, 2010-01-01 15:30)', '[2018-05-24 00:00:00+00,)', '[2018-05-24,)');
+  # \copy test TO range_test.dat WITH (FORMAT BINARY);
+  it 'encodes range data correctly' do
+    encoder = ActiveRecordCopy::EncodeForCopy.new(column_types: { 0 => :int4range, 1 => :int8range, 2 => :numrange, 3 => :tsrange, 4 => :tstzrange, 5 => :daterange })
+    encoder.add([
+      12...14,
+      223372033854775802...223372033854775810,
+      12.5..13.88211,
+      Time.parse('2010-01-01 15:20+00')...Time.parse('2010-01-01 15:30+00'),
+      Time.parse('2018-05-24 00:00:00+00')...Float::INFINITY,
+      Date.parse('2018-05-24')...Float::INFINITY
+    ])
+    encoder.close
+    io = encoder.get_io
+    existing_data = filedata('range_test.dat')
+    str = io.read
+    expect(io.class.name).to eq 'StringIO'
+    str.force_encoding('ASCII-8BIT')
+    #File.open('spec/fixtures/output.dat', 'w:ASCII-8BIT') {|out| out.write(str) }
+    expect(str).to eq existing_data
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-copy
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - Lukas Fittl
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-23 00:00:00.000000000 Z
+date: 2018-05-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -103,6 +103,7 @@ files:
 - spec/fixtures/just_an_array2.dat
 - spec/fixtures/multiline_hstore.dat
 - spec/fixtures/output.dat
+- spec/fixtures/range_test.dat
 - spec/fixtures/timestamp.dat
 - spec/fixtures/timestamp_9.3.dat
 - spec/fixtures/timestamp_big.dat
@@ -135,7 +136,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Convenient methods to load data quickly into Postgres
@@ -168,6 +169,7 @@ test_files:
 - spec/fixtures/just_an_array2.dat
 - spec/fixtures/multiline_hstore.dat
 - spec/fixtures/output.dat
+- spec/fixtures/range_test.dat
 - spec/fixtures/timestamp.dat
 - spec/fixtures/timestamp_9.3.dat
 - spec/fixtures/timestamp_big.dat