postgres-copy 1.5.0 → 1.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ad654ff2a933f1e41f23e6347f0b20daa3e16455caea9a6107ce4d50c39598f8
-  data.tar.gz: e972e684f928c0afdd7fdd096ade3d28d293604ef20f42a959097234f37f7317
+  metadata.gz: bfe11c5653d850edd47a3df54bcbb83600f820b35a297f05d363b6e89b8b27ff
+  data.tar.gz: 25a187d205ca18ad00e9a35de9a3fabc400137a56af61fce58eac5f7713491af
 SHA512:
-  metadata.gz: e57d3d4d0bd7a0bef175dad625c77013f2616222ef9ef086c449cdcbd8f58e4bbde77625e6f70ab55149683becac6e15d8c2cbf84c09c707f1a39cbef0417b05
-  data.tar.gz: b8768a3cf765aa7d6c01eb5138439e20c1140f6b2e772e96d169635832fe14a2780e438732503f719eba7f0b9bcf0e6cf6da4c7241d3d42e734d4433f65799e6
+  metadata.gz: bf7bdfb2fc14b24ede0973c7e573ed30507c77c205cfe87aa13f9302ad31d988360bcebec83b5ee1c83d266c14d8b42728fb17d5a07c222babc3aab5438562ce
+  data.tar.gz: b0f085dbbd1fbd684a00d97eda965f7074d52b8e8f0fe07657f33a5a5fcbd9ab4041a71956e640c5b40140c125675f20479044e1aea5b7bea1d1aba9d51e4227

data/.github/workflows/ruby.yml

@@ -33,6 +33,10 @@ jobs:
           --health-retries 5
 
     runs-on: ubuntu-latest
+    name: test (ruby v${{ matrix.ruby }})
+    strategy:
+      matrix:
+        ruby: ["2.7", "3.0", "3.1"]
 
     steps:
     - uses: actions/checkout@v2
@@ -41,7 +45,8 @@ jobs:
     # change this to (see https://github.com/ruby/setup-ruby#versioning):
       uses: ruby/setup-ruby@v1
       with:
-        ruby-version: 2.7
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true
     - name: Install dependencies
       run: bundle install
     - name: Run tests

data/Gemfile.lock

@@ -1,73 +1,31 @@
 PATH
   remote: .
   specs:
-    postgres-copy (1.5.0)
+    postgres-copy (1.6.1)
       activerecord (>= 5.1)
       pg (>= 0.17)
-      responders
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (6.0.3.2)
-      actionview (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
-      rack (~> 2.0, >= 2.0.8)
-      rack-test (>= 0.6.3)
-      rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actionview (6.0.3.2)
-      activesupport (= 6.0.3.2)
-      builder (~> 3.1)
-      erubi (~> 1.4)
-      rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activemodel (6.0.3.2)
-      activesupport (= 6.0.3.2)
-    activerecord (6.0.3.2)
-      activemodel (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
-    activesupport (6.0.3.2)
+    activemodel (7.0.4)
+      activesupport (= 7.0.4)
+    activerecord (7.0.4)
+      activemodel (= 7.0.4)
+      activesupport (= 7.0.4)
+    activesupport (7.0.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
-      zeitwerk (~> 2.2, >= 2.2.2)
-    builder (3.2.4)
-    concurrent-ruby (1.1.6)
-    crass (1.0.6)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    concurrent-ruby (1.1.10)
     diff-lcs (1.4.4)
-    erubi (1.9.0)
-    i18n (1.8.3)
+    i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    loofah (2.6.0)
-      crass (~> 1.0.2)
-      nokogiri (>= 1.5.9)
-    method_source (1.0.0)
-    mini_portile2 (2.4.0)
-    minitest (5.14.1)
-    nokogiri (1.10.10)
-      mini_portile2 (~> 2.4.0)
-    pg (1.2.3)
-    rack (2.2.3)
-    rack-test (1.1.0)
-      rack (>= 1.0, < 3)
-    rails-dom-testing (2.0.3)
-      activesupport (>= 4.2.0)
-      nokogiri (>= 1.6)
-    rails-html-sanitizer (1.3.0)
-      loofah (~> 2.3)
-    railties (6.0.3.2)
-      actionpack (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
-      method_source
-      rake (>= 0.8.7)
-      thor (>= 0.20.3, < 2.0)
+    minitest (5.16.3)
+    pg (1.4.3)
     rake (11.2.2)
     rdoc (6.2.1)
-    responders (3.0.1)
-      actionpack (>= 5.0)
-      railties (>= 5.0)
     rspec (2.99.0)
       rspec-core (~> 2.99.0)
       rspec-expectations (~> 2.99.0)
@@ -76,11 +34,8 @@ GEM
     rspec-expectations (2.99.2)
       diff-lcs (>= 1.1.3, < 2.0)
     rspec-mocks (2.99.4)
-    thor (1.0.1)
-    thread_safe (0.3.6)
-    tzinfo (1.2.7)
-      thread_safe (~> 0.1)
-    zeitwerk (2.4.0)
+    tzinfo (2.0.5)
+      concurrent-ruby (~> 1.0)
 
 PLATFORMS
   ruby
@@ -93,4 +48,4 @@ DEPENDENCIES
   rspec (~> 2.12)
 
 BUNDLED WITH
-   2.1.2
+   2.2.22

data/README.md

@@ -200,6 +200,13 @@ Which will generate the following SQL command:
 COPY users (id, name) FROM '/tmp/users.dat' WITH BINARY
 ```
 
+To specify the encoding with which to read the file, set the :encoding option.
+This is useful for removing byte order marks when matching column headers.
+
+```ruby
+User.copy_from "/tmp/users_with_byte_order_mark.csv", :encoding => 'bom|utf-8'
+```
+
 
 ### Using the CSV Responder
 If you want to make the result of a COPY command available to download this gem provides a CSV responder that, in conjunction with [inherited_resources](https://github.com/josevalim/inherited_resources), is a very powerfull tool. BTW, do not try to use the responder without inherited_resources.

data/lib/postgres-copy/acts_as_copy_target.rb

@@ -1,5 +1,13 @@
 require 'csv'
 
+def get_file_mode mode, encoding = nil
+  if encoding
+    "#{mode}:#{encoding}"
+  else
+    mode
+  end
+end
+
 module PostgresCopy
   module ActsAsCopyTarget
     extend ActiveSupport::Concern
@@ -10,7 +18,7 @@ module PostgresCopy
     module CopyMethods
       # Copy data to a file passed as a string (the file path) or to lines that are passed to a block
       def copy_to path = nil, options = {}
-        options = {:delimiter => ",", :format => :csv, :header => true}.merge(options)
+        options = { delimiter: ",", format: :csv, header: true }.merge(options)
         options_string = if options[:format] == :binary
                            "BINARY"
                          else
@@ -72,7 +80,7 @@ module PostgresCopy
       # * You can map fields from the file to different fields in the table using a map in the options hash
       # * For further details on usage take a look at the README.md
       def copy_from path_or_io, options = {}
-        options = {:delimiter => ",", :format => :csv, :header => true, :quote => '"'}.merge(options)
+        options = { delimiter: ",", format: :csv, header: true, quote: '"' }.merge(options)
         options[:delimiter] = "\t" if options[:format] == :tsv
         options_string = if options[:format] == :binary
                            "BINARY"
@@ -83,7 +91,7 @@ module PostgresCopy
                            delimiter = options[:format] == :tsv ? "E'\t'" : "'#{options[:delimiter]}'"
                            "WITH (" + ["DELIMITER #{delimiter}", "QUOTE '#{quote}'", null, force_null, "FORMAT CSV"].compact.join(', ') + ")"
                          end
-        io = path_or_io.instance_of?(String) ? File.open(path_or_io, 'r') : path_or_io
+        io = path_or_io.instance_of?(String) ? File.open(path_or_io, get_file_mode('r', options[:encoding])) : path_or_io
 
         if options[:format] == :binary
           columns_list = options[:columns] || []
@@ -100,7 +108,7 @@ module PostgresCopy
                   quoted_table_name
                 end
 
-        columns_list = columns_list.map{|c| options[:map][c.to_s] } if options[:map]
+        columns_list = columns_list.map{|c| options[:map][c.to_s] || c.to_s } if options[:map]
         columns_string = columns_list.size > 0 ? "(\"#{columns_list.join('","')}\")" : ""
         connection.raw_connection.copy_data %{COPY #{table} #{columns_string} FROM STDIN #{options_string}} do
           if options[:format] == :binary
@@ -124,10 +132,10 @@ module PostgresCopy
               if line_buffer =~ /\n$/ || line_buffer =~ /\Z/
                 if block_given?
                   begin
-                    row = CSV.parse_line(line_buffer.strip, {:col_sep => options[:delimiter]})
+                    row = CSV.parse_line(line_buffer.strip, col_sep: options[:delimiter])
                     yield(row)
-                    next if row.all?{|f| f.nil? }
-                    line_buffer = CSV.generate_line(row, {:col_sep => options[:delimiter]})
+                    next if row.all?(&:nil?)
+                    line_buffer = CSV.generate_line(row, col_sep: options[:delimiter])
                   rescue CSV::MalformedCSVError
                     next
                   end

data/postgres-copy.gemspec

@@ -5,7 +5,7 @@ $:.unshift lib unless $:.include?(lib)
 
 Gem::Specification.new do |s|
   s.name                  = "postgres-copy"
-  s.version               = "1.5.0"
+  s.version               = "1.6.1"
   s.platform              = Gem::Platform::RUBY
   s.required_ruby_version = ">= 1.9.3"
   s.authors               = ["Diogo Biazus"]
@@ -22,7 +22,6 @@ Gem::Specification.new do |s|
 
   s.add_dependency "pg", ">= 0.17"
   s.add_dependency "activerecord", '>= 5.1'
-  s.add_dependency "responders"
   s.add_development_dependency "bundler"
   s.add_development_dependency "rdoc"
   s.add_development_dependency "rspec", "~> 2.12"

data/spec/copy_from_spec.rb

@@ -14,6 +14,12 @@ describe "COPY FROM" do
     TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'test data 1'}]
   end
 
+  it "should import from file if path is with a field_map" do
+    TestModel.copy_from File.expand_path('spec/fixtures/comma_with_header_to_map.csv'), map: {'ref' => 'id'}
+    TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'test data 1'}]
+  end
+
+
   it "should import from IO without field_map" do
     TestModel.copy_from File.open(File.expand_path('spec/fixtures/comma_with_header.csv'), 'r')
     TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'test data 1'}]
@@ -173,4 +179,10 @@ describe "COPY FROM" do
     TestModel.order(:id).first.attributes.should == {'id' => 1, 'data' => 'changed this data'}
     TestModel.count.should == 2
   end
+
+  it "should import csv headers with BOM when provided encoding option" do
+    TestModel.copy_from File.expand_path("spec/fixtures/comma_with_bom.csv"), :encoding => "bom|utf-8"
+    TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'test data 1'}]
+  end
+
 end

data/spec/fixtures/comma_with_bom.csv

@@ -0,0 +1,2 @@
+id,data
+1,"test data 1"

data/spec/fixtures/comma_with_header_to_map.csv

@@ -0,0 +1,2 @@
+ref,data
+1,test data 1

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: postgres-copy
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.6.1
 platform: ruby
 authors:
 - Diogo Biazus
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-18 00:00:00.000000000 Z
+date: 2022-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -38,20 +38,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '5.1'
-- !ruby/object:Gem::Dependency
-  name: responders
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -134,12 +120,14 @@ files:
 - spec/copy_to_spec.rb
 - spec/fixtures/2_col_binary_data.dat
 - spec/fixtures/comma_inside_field.csv
+- spec/fixtures/comma_with_bom.csv
 - spec/fixtures/comma_with_carriage_returns.csv
 - spec/fixtures/comma_with_empty_string.csv
 - spec/fixtures/comma_with_header.csv
 - spec/fixtures/comma_with_header_and_scope.csv
 - spec/fixtures/comma_with_header_empty_values_at_the_end.csv
 - spec/fixtures/comma_with_header_multi.csv
+- spec/fixtures/comma_with_header_to_map.csv
 - spec/fixtures/comma_without_header.csv
 - spec/fixtures/extra_field.rb
 - spec/fixtures/reserved_word_model.rb
@@ -179,7 +167,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Put COPY command functionality in ActiveRecord's model class