rcsv 0.2.1 → 0.3.1

data/.gitignore

@@ -6,3 +6,6 @@ ext/rcsv/rcsv.o
 lib/rcsv/rcsv.so
 lib/rcsv/rcsv.bundle
 .*swp
+Gemfile.lock
+.bundle
+vendor/

data/.travis.yml

@@ -1,10 +1,8 @@
 language: ruby
 rvm:
+  - ree
+  - 1.8.7
   - 1.9.3
-  - 1.9.2
   - 2.0.0
-  - 2.1.0
-  - 1.8.7
-  - ree
-env:
-  - JRUBY_OPTS="--server -Xcext.enabled=true -Xcompile.invokedynamic=false"
+  - 2.1.6
+  - 2.2.2

data/Gemfile

@@ -4,3 +4,4 @@ source 'https://rubygems.org'
 gemspec
 
 gem "rake-compiler", :group => :development
+gem "test-unit", :group => :test if RUBY_VERSION >= '2.2'

data/README.md

@@ -2,7 +2,7 @@
 
 [![Build Status](https://travis-ci.org/fiksu/rcsv.png)](https://travis-ci.org/fiksu/rcsv)
 
-Rcsv is a fast CSV parsing library for MRI Ruby. Tested on REE 1.8.7 and Ruby 1.9.3.
+Rcsv is a fast CSV parsing library for MRI Ruby. Tested on REE, 1.8.7, 1.9.4, 2.0.0, 2.1.6, 2.2.2.
 
 Contrary to many other gems that implement their own parsers, Rcsv uses libcsv 3.0.3 (http://sourceforge.net/projects/libcsv/). As long as libcsv's API is stable, getting Rcsv to use newer libcsv version is as simple as updating two files (csv.h and libcsv.c).
 
@@ -114,6 +114,10 @@ A boolean flag. If enabled, only parses columns that are listed in :columns. Dis
 An integer. Default is 1MiB (1024 * 1024).
 Specifies a number of bytes that are read at once, thus allowing to read drectly from IO-like objects (files, sockets etc).
 
+### :output_encoding
+A string. By default is auto-detected from the original CSV file.
+If specified, enforces the encoding of parsed string values. The default value keeps the encoding the same as in the original CSV file.
+
 
 ## Examples
 
@@ -164,10 +168,14 @@ That would display contents of each row without needing to put the whole parsed
 
 This way it is possible to read from a File directly, with a 20MiB buffer and parse lines one by one:
 
-    Rcsv.parse(File.open('/some/file.csv'), :buffer_size => 20 * 1024 * 1024) { |row|
+    some_csv_file = File.open('/some/file.csv')
+
+    Rcsv.parse(some_csv_file, :buffer_size => 20 * 1024 * 1024) { |row|
       puts row.inspect
     }
 
+    some_csv_file.close
+
 
 ## To do
 
@@ -187,5 +195,5 @@ This way it is possible to read from a File directly, with a 20MiB buffer and pa
 
 ## Credits
 
-* Maintainer: Arthur Pirogovski @arp
-* Contributors: Edward Slavich @eslavich
+* Maintainer: Artur Pyrogovskyi @arp
+* Contributors: Edward Slavich @eslavich, Ivan Zarea @minivan, @97jaz, David Price @dprice, Andrew Grim @stopdropandrew

data/RELNOTES

@@ -1,3 +1,16 @@
+Version 0.3.1
+ * Travis fixes
+ * Fixed older Ruby support in tests
+
+Version 0.3.0
+ * changed nil to be rendered as empty string by writer (by David Price)
+ * changed the default writer newline character from "\r\n" to platform's default (by Andrew Grim)
+ * improved writer quoting (by Andrew Grim)
+ * added :quote_char writer option (by @97jaz)
+ * added :output_encoding reader option (by David Price)
+ * added automatic encoding detection by reader (by David Price)
+ * updated supported Rubies list to REE, 1.8.7, 1.9.4, 2.0.0, 2.1.6, 2.2.2
+
 Version 0.2.1
   * removed a bunch of deprecation warnings (by Ivan Zarea)
   * added Ruby 2.1 support for Travis CI

data/ext/rcsv/rcsv.c

@@ -110,7 +110,7 @@ void end_of_field_callback(void * field, size_t field_size, void * data) {
       if (meta->current_col < meta->num_row_conversions) {
         switch (row_conversion){
           case 's': /* String */
-            parsed_field = ENCODED_STR_NEW(field_str, field_size, meta->encoding_index); 
+            parsed_field = ENCODED_STR_NEW(field_str, field_size, meta->encoding_index);
             break;
           case 'i': /* Integer */
             parsed_field = LL2NUM(atoll(field_str));
@@ -327,6 +327,12 @@ VALUE rcsv_raw_parse(VALUE ensure_container) {
     csv_set_delim(cp, (unsigned char)*StringValuePtr(option));
   }
 
+  /* :quote_char sets the character used for quoting data; default is double-quote (") */
+  option = rb_hash_aref(options, ID2SYM(rb_intern("quote_char")));
+  if (option != Qnil) {
+    csv_set_quote(cp, (unsigned char)*StringValuePtr(option));
+  }
+
   /* Specify how many rows to skip from the beginning of CSV */
   option = rb_hash_aref(options, ID2SYM(rb_intern("offset_rows")));
   if (option != Qnil) {
@@ -382,7 +388,7 @@ VALUE rcsv_raw_parse(VALUE ensure_container) {
 
   /* :row_conversions specifies Ruby types that CSV field values should be converted into.
      Each char of row_conversions string represents Ruby type for CSV field with matching position. */
-  option = rb_hash_aref(options, ID2SYM(rb_intern("row_conversions"))); 
+  option = rb_hash_aref(options, ID2SYM(rb_intern("row_conversions")));
   if (option != Qnil) {
     meta->num_row_conversions = RSTRING_LEN(option);
     meta->row_conversions = StringValuePtr(option);
@@ -390,7 +396,7 @@ VALUE rcsv_raw_parse(VALUE ensure_container) {
 
  /* Column names should be declared explicitly when parsing fields as Hashes */
   if (meta->row_as_hash) { /* Only matters for hash results */
-    option = rb_hash_aref(options, ID2SYM(rb_intern("column_names"))); 
+    option = rb_hash_aref(options, ID2SYM(rb_intern("column_names")));
     if (option == Qnil) {
       rb_raise(rcsv_parse_error, ":row_as_hash requires :column_names to be set.");
     } else {

data/lib/lib_csv.rb

@@ -34,6 +34,9 @@ class LibCsv
   attach_function :csv_set_delim, [:pointer, :uchar], :void
   attach_function :csv_get_delim, [:pointer], :uchar
 
+  attach_function :csv_set_quote, [:pointer, :uchar], :void
+  attach_function :csv_get_quote, [:pointer], :uchar
+
   attach_function :csv_error, [:pointer], :int
   attach_function :csv_strerror, [:int], :string
 
@@ -46,6 +49,10 @@ class LibCsv
       csv_set_delim(parser, options[:col_sep].ord)
     end
 
+    if options[:quote_char]
+      csv_set_quote(parser, options[:quote_char].ord)
+    end
+
     fail "Couldn't initialize libcsv" if result == -1
 
     result = [[]]
@@ -82,7 +89,7 @@ class LibCsv
     csv_fini(parser, end_of_field_callback, end_of_record_callback, nil)
     csv_free(parser)
     result.pop if result.last == []
-    
+
     return result
   end
 end

data/lib/rcsv.rb

@@ -2,6 +2,7 @@ require "rcsv/rcsv"
 require "rcsv/version"
 
 require "stringio"
+require "English"
 
 class Rcsv
 
@@ -30,6 +31,7 @@ class Rcsv
     raw_options = {}
 
     raw_options[:col_sep] = options[:column_separator] && options[:column_separator][0] || ','
+    raw_options[:quote_char] = options[:quote_char] && options[:quote_char][0] || '"'
     raw_options[:offset_rows] = options[:offset_rows] || 0
     raw_options[:nostrict] = options[:nostrict]
     raw_options[:parse_empty_fields_as] = options[:parse_empty_fields_as]
@@ -138,8 +140,16 @@ class Rcsv
   def initialize(write_options = {})
     @write_options = write_options
     @write_options[:column_separator] ||= ','
-    @write_options[:newline_delimiter] ||= "\r\n" # Making Excel happy...
+    @write_options[:newline_delimiter] ||= $INPUT_RECORD_SEPARATOR
     @write_options[:header] ||= false
+
+    @quote = '"'
+    @escaped_quote = @quote * 2
+    @quotable_chars = Regexp.new('[%s%s%s]' % [
+      Regexp.escape(@write_options[:column_separator]),
+      Regexp.escape(@write_options[:newline_delimiter]),
+      Regexp.escape(@quote)
+    ])
   end
 
   def write(io, &block)
@@ -161,9 +171,8 @@ class Rcsv
     max_index = row.size - 1
 
     row.each_with_index do |field, index|
-      unquoted_field = process(field, @write_options[:columns][index])
-      # TODO: a better quoting
-      csv_row << (unquoted_field.match(/,/) ? "\"#{unquoted_field}\"" : unquoted_field)
+      unquoted_field = process(field, @write_options[:columns] && @write_options[:columns][index])
+      csv_row << (unquoted_field.match(@quotable_chars) ? "\"#{unquoted_field.gsub(@quote, @escaped_quote)}\"" : unquoted_field)
       csv_row << column_separator unless index == max_index
     end
 
@@ -173,7 +182,8 @@ class Rcsv
   protected
 
   def process(field, column_options)
-    return case column_options[:formatter]
+    return '' if field.nil?
+    return case column_options && column_options[:formatter]
     when :strftime
       format = column_options[:format] || "%Y-%m-%d %H:%M:%S %z"
       field.strftime(format)

data/lib/rcsv/version.rb

@@ -1,3 +1,3 @@
 class Rcsv
-  VERSION = "0.2.1"
+  VERSION = "0.3.1"
 end

data/rcsv.gemspec

@@ -16,4 +16,7 @@ Gem::Specification.new do |gem|
   gem.name          = "rcsv"
   gem.require_paths = ["lib", "ext"]
   gem.version       = Rcsv::VERSION
+  if RUBY_VERSION >= '2.2'
+    gem.add_development_dependency "test-unit", "~> 3.0.8"
+  end
 end

data/test/test_rcsv_raw_parse.rb

@@ -32,17 +32,29 @@ class RcsvRawParseTest < Test::Unit::TestCase
     assert_equal('""C81E-=; **ECCB; .. 89', raw_parsed_tsv_data[3][6])
     assert_equal("Dallas\t TX", raw_parsed_tsv_data[888][13])
   end
-  
+
+  def test_rcsv_quote_char
+    csv = [
+      "F0A83489,69118080,,73,7008,2016-10-03,'''''C81E-=; **ECCB; .. 89','130,86',a3eb,1341-04-10,7612.699237971538,5b5e3fce-2ea5-4ca9-9749-90fd6dc8dd66,9,'Los Angeles, CA',e,6.047887837492023,f",
+      "48F4FAC9,11599213,,0,1897,2014-02-23,'''''ECCB-=; **A87F; .. 61','787,6',84de,1353-11-10,8078.704911344607,404d9d3e-963f-4199-a2f3-e71f6828b716,6,'Dallas, TX',,-6.684507609859605, 1"
+    ]
+    csv_data = StringIO.new(csv.join("\n"))
+    raw_parsed_csv_data = Rcsv.raw_parse(csv_data, :quote_char => "'")
+
+    assert_equal("''C81E-=; **ECCB; .. 89", raw_parsed_csv_data[0][6])
+    assert_equal('Dallas, TX', raw_parsed_csv_data[1][13])
+  end
+
   if String.instance_methods.include?(:encoding)
     def test_rcsv_output_encoding_default
       raw_parsed_csv_data = Rcsv.raw_parse(@csv_data)
-    
+
       assert_equal(raw_parsed_csv_data[0][2].encoding, Encoding::ASCII_8BIT)
     end
 
     def test_rcsv_output_encoding_utf8
       raw_parsed_csv_data = Rcsv.raw_parse(@csv_data, :output_encoding => "UTF-8")
-      
+
       assert_equal(raw_parsed_csv_data[0][2].encoding, Encoding::UTF_8)
     end
   else
@@ -52,7 +64,7 @@ class RcsvRawParseTest < Test::Unit::TestCase
       end
     end
   end
-    
+
   def test_buffer_size
     raw_parsed_csv_data = Rcsv.raw_parse(@csv_data, :buffer_size => 10)
 

data/test/test_rcsv_write.rb

@@ -47,9 +47,10 @@ class RcsvWriteTest < Test::Unit::TestCase
     }
 
     @data = [
-      [1, Date.parse('2012-11-11'), 100.234, true, 1, nil],
-      [nil, Date.parse('1970-01-02'), -0.1, :nyancat, 123.8891, 0],
-      [3, Date.parse('2012-12-12'), 0, 'sepulka', -122, 'zoop']
+      [1, Date.parse('2012-11-11'), 100.234, true, 1, true],
+      ['elephant', Date.parse('1970-01-02'), -0.1, :nyancat, 123.8891, 0],
+      [3, Date.parse('2012-12-12'), 0, 'sepulka', -122, 'zoop'],
+      [nil, nil, nil, nil, nil, nil]
     ]
 
     @writer = Rcsv.new(@options)
@@ -57,12 +58,12 @@ class RcsvWriteTest < Test::Unit::TestCase
 
   def test_rcsv_generate_header
     assert_equal(
-      "ID,Date,Money,Banana IDDQD,Hashformat,\r\n", @writer.generate_header
+      "ID,Date,Money,Banana IDDQD,Hashformat,\n", @writer.generate_header
     )
   end
 
   def test_rscv_generate_row
-    assert_equal("1,2012-11-11,$100.23,true,$1.00,false\r\n", @writer.generate_row(@data.first))
+    assert_equal("1,2012-11-11,$100.23,true,$1.00,true\n", @writer.generate_row(@data.first))
   end
 
   def test_rcsv_write
@@ -75,7 +76,7 @@ class RcsvWriteTest < Test::Unit::TestCase
     io.rewind
 
     assert_equal(
-      "ID,Date,Money,Banana IDDQD,Hashformat,\r\n1,2012-11-11,$100.23,true,$1.00,false\r\n,1970-01-02,$-0.10,nyancat,$123.89,false\r\n3,2012-12-12,$0.00,sepulka,$-122.00,true\r\n", io.read
+      "ID,Date,Money,Banana IDDQD,Hashformat,\n1,2012-11-11,$100.23,true,$1.00,true\nelephant,1970-01-02,$-0.10,nyancat,$123.89,false\n3,2012-12-12,$0.00,sepulka,$-122.00,true\n,,,,,\n", io.read
     )
   end
 
@@ -90,7 +91,34 @@ class RcsvWriteTest < Test::Unit::TestCase
     io.rewind
 
     assert_equal(
-      "1,2012-11-11,$100.23,true,$1.00,false\r\n,1970-01-02,$-0.10,nyancat,$123.89,false\r\n3,2012-12-12,$0.00,sepulka,$-122.00,true\r\n", io.read
+      "1,2012-11-11,$100.23,true,$1.00,true\nelephant,1970-01-02,$-0.10,nyancat,$123.89,false\n3,2012-12-12,$0.00,sepulka,$-122.00,true\n,,,,,\n", io.read
     )
   end
+
+  def test_generate_row__dont_require_columns
+    writer = Rcsv.new
+    assert_equal "1,2,3\n", writer.generate_row([1, 2, 3])
+  end
+
+  def test_generate_row__proper_escaping_for_quotes_and_newlines
+    writer = Rcsv.new
+    assert_equal "\"before quote \"\" after quote\",\"before newline \n after newline\"\n",
+                 writer.generate_row(["before quote \" after quote", "before newline \n after newline"])
+  end
+
+  def test_generate_row__should_be_able_to_parse_generated_csv
+    writer = Rcsv.new
+    quotable_strings = [
+      "before quote \" after quote",
+      "before newline \n after newline",
+      "before separator , after separator",
+      "separator , and quote \" oh my"
+    ]
+    assert_equal [quotable_strings], Rcsv.parse(writer.generate_row(quotable_strings), :header => :none)
+  end
+
+  def test_generate_row__should_handle_alternate_column_separators
+    writer = Rcsv.new(:column_separator => '|')
+    assert_equal "1|2|\"before pipe | after pipe\"\n", writer.generate_row([1, 2, 'before pipe | after pipe'])
+  end
 end

metadata

@@ -1,28 +1,37 @@
---- !ruby/object:Gem::Specification
+--- !ruby/object:Gem::Specification 
 name: rcsv
-version: !ruby/object:Gem::Version
-  version: 0.2.1
+version: !ruby/object:Gem::Version 
+  hash: 17
+  prerelease: 
+  segments: 
+  - 0
+  - 3
+  - 1
+  version: 0.3.1
 platform: ruby
-authors:
+authors: 
 - Arthur Pirogovski
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-06 00:00:00.000000000 Z
+
+date: 2015-06-17 00:00:00 Z
 dependencies: []
+
 description: A libcsv-based CSV parser for Ruby
-email:
+email: 
 - arthur@flyingtealeaf.com
 executables: []
-extensions:
+
+extensions: 
 - ext/rcsv/extconf.rb
 extra_rdoc_files: []
-files:
+
+files: 
 - .gitignore
 - .travis.yml
 - COPYING.LESSER
 - Gemfile
-- Gemfile.lock
 - LICENSE
 - README.md
 - RELNOTES
@@ -43,30 +52,39 @@ files:
 - test/test_rcsv_write.rb
 homepage: http://github.com/fiksu/rcsv
 licenses: []
-metadata: {}
+
 post_install_message: 
 rdoc_options: []
-require_paths:
+
+require_paths: 
 - lib
 - ext
-required_ruby_version: !ruby/object:Gem::Requirement
-  requirements:
-  - - '>='
-    - !ruby/object:Gem::Version
-      version: '0'
-required_rubygems_version: !ruby/object:Gem::Requirement
-  requirements:
-  - - '>='
-    - !ruby/object:Gem::Version
-      version: '0'
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
 requirements: []
+
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 1.8.24
 signing_key: 
-specification_version: 4
-summary: Fast CSV parsing library for MRI based on libcsv. Supports type conversion,
-  non-strict parsing and basic filtering.
-test_files:
+specification_version: 3
+summary: Fast CSV parsing library for MRI based on libcsv. Supports type conversion, non-strict parsing and basic filtering.
+test_files: 
 - test/test_rcsv.csv
 - test/test_rcsv_parse.rb
 - test/test_rcsv_raw_parse.rb

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: 0333dfa0687b5fff49a8102ddbea1368a8335919
-  data.tar.gz: 0c9b534944b188a82c7734b30aac28bdf7f38f80
-SHA512:
-  metadata.gz: 06e065ecb4755ec5acb43132b39672331ce4a5b7e52c656739d6e23b896c5930311232f6e1c7fa0b39357df6e41d138dcee9cb33b040c0e536c1f7f450f48f1a
-  data.tar.gz: 9c46cedf27acf68b9248d8e9b2329f5162bb099bd67585f6ddc61d85d784b83e20a644649a8666ad5b57d074a1e015f7d42f1c58013f559b5e84d0495a1d6e44

data/Gemfile.lock

@@ -1,18 +0,0 @@
-PATH
-  remote: .
-  specs:
-    rcsv (0.2.1)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    rake (0.9.2.2)
-    rake-compiler (0.8.1)
-      rake
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  rake-compiler
-  rcsv!