activerecord-copy 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c58fb95429ac77c914ddd9bdcc6980c3a4c99f87
+  data.tar.gz: 51e89df1b40433574b1dc3472ef8e823f15627fd
+SHA512:
+  metadata.gz: 078585c52f4266c53000067f0a78a1d7c94b8f73dfc52e9ad9e9fa7bed7c35e499f7ee317fbb6089cfcfe99e802b59549d87910308a3ab89c2e5ffce8326612e
+  data.tar.gz: 0d95073114fa52abfd5ef597eab4fd62e6fef155c4a039fd552f02d6b34a08c822e508a78747d49a15554c05bbafe100e3483ceab139a7083d89a3c2e495f4c6

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rubocop.yml

@@ -0,0 +1,40 @@
+require: rubocop-rspec
+
+AllCops:
+  TargetRubyVersion: 2.3
+
+LineLength:
+  Enabled: false
+
+FrozenStringLiteralComment:
+  Enabled: false
+
+ClassLength:
+  Enabled: false
+
+RescueModifier:
+  Enabled: false
+
+AccessorMethodName:
+  Enabled: false
+
+Documentation:
+  Enabled: false
+
+AbcSize:
+  Enabled: false
+
+CyclomaticComplexity:
+  Enabled: false
+
+PerceivedComplexity:
+  Enabled: false
+
+MethodLength:
+  Enabled: false
+
+BlockNesting:
+  Enabled: false
+
+RSpec/DescribeClass:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,9 @@
+language: ruby
+rvm:
+ - 2.0.0-p648
+ - 2.1.9
+ - 2.2.5
+ - 2.3.1
+ - 2.4.0-preview1
+ - rbx-3.20
+script: bundle exec rspec

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+# Changelog
+
+## 1.0.0       2017-07-22
+
+* Initial release after fork from pg_data_encoder

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in pg_data_encoder.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,49 @@
+Copyright (c) 2017 Lukas Fittl
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+---
+
+pg_data_encoder work is:
+
+Copyright (c) 2012 Pete Brumm
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,40 @@
+# activerecord-copy [ ![](https://img.shields.io/gem/v/activerecord-copy.svg)](https://rubygems.org/gems/activerecord-copy) [ ![](https://img.shields.io/gem/dt/activerecord-copy.svg)](https://rubygems.org/gems/activerecord-copy)
+
+Library to assist using binary COPY into PostgreSQL with activerecord.
+
+Binary copy functionality is based on [pg_data_encoder](https://github.com/pbrumm/pg_data_encoder),
+but modified to support additional types, and to prefer column type specifications
+over inferred data types.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'activerecord-copy'
+
+## Usage
+
+```ruby
+class MyModel < ApplicationRecord
+  include ActiveRecordCopy
+end
+
+MyModel.copy_from_client do
+
+end
+```    
+
+## Authors
+
+* [Lukas Fittl](https://github.com/lfittl)
+
+Credits to [Pete Brumm](https://github.com/pbrumm) who wrote pg_data_encoder and
+which this library repurposes.
+
+## LICENSE
+
+Copyright (c) 2017, Lukas Fittl <lukas@fittl><br>
+activerecord-copy is licensed under the MIT license, see LICENSE file for details.
+
+pg_data_encoder is Copyright (c) 2012, Pete Brumm<br>
+pg_data_encoder is included under the terms of the MIT license, see LICENSE file for details.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/activerecord-copy.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'activerecord-copy/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = 'activerecord-copy'
+  gem.version       = ActiveRecordCopy::VERSION
+  gem.authors       = ['Lukas Fittl']
+  gem.email         = ['lukas@fittl.com']
+  gem.description   = 'Supports binary COPY into PostgreSQL with activerecord'
+  gem.summary       = 'Convenient methods to load data quickly into Postgres'
+  gem.homepage      = 'https://github.com/lfittl/activerecord-copy'
+
+  gem.license       = 'MIT'
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+
+  gem.add_dependency('activerecord', '>= 3.1')
+
+  gem.add_development_dependency('rspec', '>= 2.12.0')
+  gem.add_development_dependency('rspec-core', '>= 2.12.0')
+end

data/lib/activerecord-copy.rb

@@ -0,0 +1,92 @@
+require 'activerecord-copy/version'
+
+require 'activerecord-copy/constants'
+require 'activerecord-copy/exception'
+require 'activerecord-copy/temp_buffer'
+
+require 'activerecord-copy/encode_for_copy'
+require 'activerecord-copy/decoder'
+
+require 'json'
+
+require 'active_support'
+
+module ActiveRecordCopy
+  module CopyFromClient
+    extend ActiveSupport::Concern
+
+    class CopyHandler
+      def initialize(columns:, model_class:, table_name:)
+        @columns      = columns
+        @model_class  = model_class
+        @connection   = model_class.connection.raw_connection
+        @table_name   = table_name
+        @column_types = columns.map do |c|
+          column = model_class.columns_hash[c.to_s]
+          raise format('Could not find column %s on %s', c, model_class.table_name) if column.nil?
+
+          if column.type == :integer
+            if column.limit == 8
+              :bigint
+            elsif column.limit == 2
+              :smallint
+            else
+              :integer
+            end
+          else
+            column.type
+          end
+        end
+
+        reset
+      end
+
+      def <<(row)
+        @encoder.add row
+      end
+
+      def close
+        run_copy
+      end
+
+      private
+
+      def run_copy
+        io = @encoder.get_io
+
+        @connection.copy_data %{COPY #{@table_name}("#{@columns.join('","')}") FROM STDIN BINARY} do
+          begin
+            while chunk = io.readpartial(10_240) # rubocop:disable Lint/AssignmentInCondition
+              @connection.put_copy_data chunk
+            end
+          rescue EOFError # rubocop:disable Lint/HandleExceptions
+          end
+        end
+
+        @encoder.remove
+        reset
+
+        nil
+      end
+
+      def reset
+        @encoder = ActiveRecordCopy::EncodeForCopy.new column_types: @column_types
+        @row_count = 0
+      end
+    end
+
+    class_methods do
+      def copy_from_client(columns, table_name: nil, &_block)
+        table_name ||= quoted_table_name
+        handler = CopyHandler.new(columns: columns, model_class: self, table_name: table_name)
+        yield(handler)
+        handler.close
+        true
+      end
+    end
+  end
+end
+
+if defined?(ActiveRecord::Base)
+  ActiveRecord::Base.extend(ActiveRecordCopy::CopyFromClient)
+end

data/lib/activerecord-copy/constants.rb

@@ -0,0 +1,18 @@
+module ActiveRecordCopy
+  PACKED_UINT_8 = 'C'.freeze # 8-bit unsigned (unsigned char)
+  PACKED_UINT_16 = 'n'.freeze # 16-bit unsigned, network (big-endian) byte order
+  PACKED_UINT_32 = 'N'.freeze # 32-bit unsigned, network (big-endian) byte order
+  PACKED_UINT_64 = 'Q>'.freeze # 64-bit unsigned, big endian
+  PACKED_FLOAT_64 = 'G'.freeze # double-precision, network (big-endian) byte order
+  PACKED_HEX_STRING = 'H*'.freeze # hex string (high nibble first)
+
+  INT_TYPE_OID = 23
+  TEXT_TYPE_OID = 25
+  UUID_TYPE_OID = 2950
+  VARCHAR_TYPE_OID = 1043
+
+  ASCII_8BIT_ENCODING = 'ASCII-8BIT'.freeze
+  UTF_8_ENCODING = 'UTF-8'.freeze
+
+  POSTGRES_EPOCH_TIME = (Time.utc(2000, 1, 1).to_f * 1_000_000).to_i
+end

data/lib/activerecord-copy/decoder.rb

@@ -0,0 +1,176 @@
+require 'tempfile'
+require 'stringio'
+
+module ActiveRecordCopy
+  class Decoder
+    def initialize(options = {})
+      @options = options
+      @closed = false
+      if options[:column_types].is_a?(Array)
+        map = {}
+        options[:column_types].each_with_index do |c, i|
+          map[i] = c
+        end
+        options[:column_types] = map
+      else
+        options[:column_types] ||= {}
+      end
+      @io = nil
+    end
+
+    def read_line
+      return nil if @closed
+      setup_io unless @io
+      row = []
+      bytes = @io.read(2)
+      # p bytes
+      column_count = bytes.unpack(PACKED_UINT_16).first
+      if column_count == 65_535
+        @closed = true
+        return nil
+      end
+      # @io.write([row.size].pack(PACKED_UINT_32))
+      0.upto(column_count - 1).each do |index|
+        field = decode_field(@io)
+        row[index] = if field.nil?
+                       field
+                     elsif @options[:column_types][index]
+                       map_field(field, @options[:column_types][index])
+                     else
+                       field
+                     end
+      end
+      row
+    end
+
+    def each
+      loop do
+        result = read_line
+        break unless result
+        yield result
+        break if @closed
+      end
+    end
+
+    private
+
+    def setup_io
+      if @options[:file]
+        @io = File.open(@options[:file], 'r:' + ASCII_8BIT_ENCODING)
+      elsif !@options[:io].nil?
+        @io = @options[:io]
+      else
+        raise 'NO io present'
+      end
+      header = "PGCOPY\n\377\r\n\0".force_encoding(ASCII_8BIT_ENCODING)
+      result = @io.read(header.bytesize)
+      raise 'invalid format' if result != header
+      # p @io.read(10)
+
+      @io.read(2) # blank
+      @io.read(6) # blank
+    end
+
+    def decode_field(io)
+      bytes = io.read(4)
+
+      if bytes == "\xFF\xFF\xFF\xFF".force_encoding(ASCII_8BIT_ENCODING)
+        return nil
+      else
+        io.read(bytes.unpack(PACKED_UINT_32).first)
+      end
+    end
+
+    def map_field(data, type)
+      # p [type, data]
+
+      case type
+      when :int, :integer
+        data.unpack(PACKED_UINT_32).first
+      when :bytea
+        data
+      when :bigint
+        data.unpack(PACKED_UINT_64).first
+      when :float, :double
+        data.unpack(PACKED_FLOAT_64).first
+      when :boolean
+        v = data.unpack(PACKED_UINT_8).first
+        v == 1
+      when :string, :text, :character
+        data.force_encoding(UTF_8_ENCODING)
+      when :json
+        JSON.load(data)
+      when :uuid
+        r = data.unpack('H*').first
+        "#{r[0..7]}-#{r[8..11]}-#{r[12..15]}-#{r[16..19]}-#{r[20..-1]}"
+      when :uuid_raw
+        r = data.unpack('H*').first
+      when :array, :"integer[]", :"uuid[]", :"character[]"
+        io = StringIO.new(data)
+        io.read(4) # unknown
+        io.read(4) # unknown
+        atype_raw = io.read(4)
+        return [] if atype_raw.nil?
+        atype = atype_raw.unpack(PACKED_UINT_32).first # string type?
+        return [] if io.pos == io.size
+        size = io.read(4).unpack(PACKED_UINT_32).first
+        io.read(4) # should be 1 for dimension
+        # p [atype, size]
+        # p data
+        case atype
+        when UUID_TYPE_OID
+          0.upto(size - 1).map do
+            io.read(4) # size
+            r = io.read(16).unpack(PACKED_HEX_STRING).first
+            "#{r[0..7]}-#{r[8..11]}-#{r[12..15]}-#{r[16..19]}-#{r[20..-1]}"
+          end
+        when TEXT_TYPE_OID, VARCHAR_TYPE_OID
+          0.upto(size - 1).map do
+            size = io.read(4).unpack(PACKED_UINT_32).first
+            io.read(size)
+          end
+        when INT_TYPE_OID
+          0.upto(size - 1).map do
+            size = io.read(4).unpack(PACKED_UINT_32).first
+            bytes = io.read(size)
+            bytes.unpack(PACKED_UINT_32).first
+          end
+        else
+          raise "Unsupported Array type #{atype}"
+        end
+      when :hstore, :hash
+        io = StringIO.new(data)
+        fields = io.read(4).unpack(PACKED_UINT_32).first
+        h = {}
+
+        0.upto(fields - 1).each do
+          key_size = io.read(4).unpack(PACKED_UINT_32).first
+          key = io.read(key_size).force_encoding("UTF-8")
+          value_size = io.read(4).unpack(PACKED_UINT_32).first
+          if value_size == 4294967295 # nil  "\xFF\xFF\xFF\xFF"
+            value = nil
+          else
+            value = io.read(value_size)
+            value = value.force_encoding("UTF-8") if !value.nil?
+          end
+          h[key] = value
+        end
+        raise "remaining hstore bytes!" if io.pos != io.size
+        h
+      when :time, :timestamp
+        d = data.unpack("L!>").first
+        Time.at((d + POSTGRES_EPOCH_TIME) / 1_000_000.0).utc
+      when :date
+        # couldn't find another way to get signed network byte order
+        m = 0b0111_1111_1111_1111_1111_1111_1111_1111
+        d = data.unpack(PACKED_UINT_32).first
+        d = (d & m) - m - 1 if data.bytes[0] & 0b1000_0000 > 0 # negative number
+
+        # p [data, d, Date.jd(d + Date.new(2000,1,1).jd)]
+        Date.jd(d + Date.new(2000, 1, 1).jd)
+      else
+        raise "Unsupported format #{type}"
+      end
+    end
+  end
+end