csv 3.1.9 → 3.3.2

data/lib/csv.rb

@@ -48,7 +48,7 @@
 #
 # === Interface
 #
-# * CSV now uses Hash-style parameters to set options.
+# * CSV now uses keyword parameters to set options.
 # * CSV no longer has generate_row() or parse_row().
 # * The old CSV's Reader and Writer classes have been dropped.
 # * CSV::open() is now more like Ruby's open().
@@ -70,7 +70,7 @@
 # == What is CSV, really?
 #
 # CSV maintains a pretty strict definition of CSV taken directly from
-# {the RFC}[http://www.ietf.org/rfc/rfc4180.txt]. I relax the rules in only one
+# {the RFC}[https://www.ietf.org/rfc/rfc4180.txt]. I relax the rules in only one
 # place and that is to make using this library easier. CSV will parse all valid
 # CSV.
 #
@@ -90,29 +90,19 @@
 # with any questions.
 
 require "forwardable"
-require "English"
 require "date"
+require "time"
 require "stringio"
 
 require_relative "csv/fields_converter"
-require_relative "csv/match_p"
+require_relative "csv/input_record_separator"
 require_relative "csv/parser"
 require_relative "csv/row"
 require_relative "csv/table"
 require_relative "csv/writer"
 
-using CSV::MatchP if CSV.const_defined?(:MatchP)
-
 # == \CSV
 #
-# === In a Hurry?
-#
-# If you are familiar with \CSV data and have a particular task in mind,
-# you may want to go directly to the:
-# - {Recipes for CSV}[doc/csv/recipes/recipes_rdoc.html].
-#
-# Otherwise, read on here, about the API: classes, methods, and constants.
-#
 # === \CSV Data
 #
 # \CSV (comma-separated values) data is a text representation of a table:
@@ -341,6 +331,7 @@ using CSV::MatchP if CSV.const_defined?(:MatchP)
 #     liberal_parsing:    false,
 #     nil_value:          nil,
 #     empty_value:        "",
+#     strip:              false,
 #     # For generating.
 #     write_headers:      nil,
 #     quote_empty:        true,
@@ -348,7 +339,6 @@ using CSV::MatchP if CSV.const_defined?(:MatchP)
 #     write_converters:   nil,
 #     write_nil_value:    nil,
 #     write_empty_value:  "",
-#     strip:              false,
 #   }
 #
 # ==== Options for Parsing
@@ -357,7 +347,9 @@ using CSV::MatchP if CSV.const_defined?(:MatchP)
 # - +row_sep+: Specifies the row separator; used to delimit rows.
 # - +col_sep+: Specifies the column separator; used to delimit fields.
 # - +quote_char+: Specifies the quote character; used to quote fields.
-# - +field_size_limit+: Specifies the maximum field size allowed.
+# - +field_size_limit+: Specifies the maximum field size + 1 allowed.
+#   Deprecated since 3.2.3. Use +max_field_size+ instead.
+# - +max_field_size+: Specifies the maximum field size allowed.
 # - +converters+: Specifies the field converters to be used.
 # - +unconverted_fields+: Specifies whether unconverted fields are to be available.
 # - +headers+: Specifies whether data contains headers,
@@ -366,8 +358,9 @@ using CSV::MatchP if CSV.const_defined?(:MatchP)
 # - +header_converters+: Specifies the header converters to be used.
 # - +skip_blanks+: Specifies whether blanks lines are to be ignored.
 # - +skip_lines+: Specifies how comments lines are to be recognized.
-# - +strip+: Specifies whether leading and trailing whitespace are
-#   to be stripped from fields..
+# - +strip+: Specifies whether leading and trailing whitespace are to be
+#   stripped from fields. This must be compatible with +col_sep+; if it is not,
+#   then an +ArgumentError+ exception will be raised.
 # - +liberal_parsing+: Specifies whether \CSV should attempt to parse
 #   non-compliant data.
 # - +nil_value+: Specifies the object that is to be substituted for each null (no-text) field.
@@ -513,7 +506,7 @@ using CSV::MatchP if CSV.const_defined?(:MatchP)
 #  [" 1 ", #<struct CSV::FieldInfo index=1, line=2, header=nil>]
 #  [" baz ", #<struct CSV::FieldInfo index=0, line=3, header=nil>]
 #  [" 2 ", #<struct CSV::FieldInfo index=1, line=3, header=nil>]
-# Each CSV::Info object shows:
+# Each CSV::FieldInfo object shows:
 # - The 0-based field index.
 # - The 1-based line index.
 # - The field header, if any.
@@ -529,6 +522,7 @@ using CSV::MatchP if CSV.const_defined?(:MatchP)
 # - <tt>:float</tt>: converts each \String-embedded float into a true \Float.
 # - <tt>:date</tt>: converts each \String-embedded date into a true \Date.
 # - <tt>:date_time</tt>: converts each \String-embedded date-time into a true \DateTime
+# - <tt>:time</tt>: converts each \String-embedded time into a true \Time
 # .
 # This example creates a converter proc, then stores it:
 #   strip_converter = proc {|field| field.strip }
@@ -547,6 +541,14 @@ using CSV::MatchP if CSV.const_defined?(:MatchP)
 #
 # There is no such storage structure for write headers.
 #
+# In order for the parsing methods to access stored converters in non-main-Ractors, the
+# storage structure must be made shareable first.
+# Therefore, <tt>Ractor.make_shareable(CSV::Converters)</tt> and
+# <tt>Ractor.make_shareable(CSV::HeaderConverters)</tt> must be called before the creation
+# of Ractors that use the converters stored in these structures. (Since making the storage
+# structures shareable involves freezing them, any custom converters that are to be used
+# must be added first.)
+#
 # ===== Converter Lists
 #
 # A _converter_ _list_ is an \Array that may include any assortment of:
@@ -631,6 +633,7 @@ using CSV::MatchP if CSV.const_defined?(:MatchP)
 #   [:numeric, [:integer, :float]]
 #   [:date, Proc]
 #   [:date_time, Proc]
+#   [:time, Proc]
 #   [:all, [:date_time, :numeric]]
 #
 # Each of these converters transcodes values to UTF-8 before attempting conversion.
@@ -675,6 +678,15 @@ using CSV::MatchP if CSV.const_defined?(:MatchP)
 #   csv = CSV.parse_line(data, converters: :date_time)
 #   csv # => [#<DateTime: 2020-05-07T14:59:00-05:00 ((2458977j,71940s,0n),-18000s,2299161j)>, "x"]
 #
+# Converter +time+ converts each field that Time::parse accepts:
+#   data = '2020-05-07T14:59:00-05:00,x'
+#   # Without the converter
+#   csv = CSV.parse_line(data)
+#   csv # => ["2020-05-07T14:59:00-05:00", "x"]
+#   # With the converter
+#   csv = CSV.parse_line(data, converters: :time)
+#   csv # => [2020-05-07 14:59:00 -0500, "x"]
+#
 # Converter +:numeric+ converts with both +:date_time+ and +:numeric+..
 #
 # As seen above, method #convert adds \converters to a \CSV instance,
@@ -705,7 +717,7 @@ using CSV::MatchP if CSV.const_defined?(:MatchP)
 # Header converters operate only on headers (and not on other rows).
 #
 # There are three ways to use header \converters;
-# these examples use built-in header converter +:dowhcase+,
+# these examples use built-in header converter +:downcase+,
 # which downcases each parsed header.
 #
 # - Option +header_converters+ with a singleton parsing method:
@@ -846,6 +858,15 @@ class CSV
     end
   end
 
+  # The error thrown when the parser encounters invalid encoding in CSV.
+  class InvalidEncodingError < MalformedCSVError
+    attr_reader :encoding
+    def initialize(encoding, line_number)
+      @encoding = encoding
+      super("Invalid byte sequence in #{encoding}", line_number)
+    end
+  end
+
   #
   # A FieldInfo Struct contains details about a field's position in the data
   # source it was read from.  CSV will pass this Struct to some blocks that make
@@ -855,19 +876,19 @@ class CSV
   # <b><tt>index</tt></b>::  The zero-based index of the field in its row.
   # <b><tt>line</tt></b>::   The line of the data source this row is from.
   # <b><tt>header</tt></b>:: The header for the column, when available.
+  # <b><tt>quoted?</tt></b>:: True or false, whether the original value is quoted or not.
   #
-  FieldInfo = Struct.new(:index, :line, :header)
+  FieldInfo = Struct.new(:index, :line, :header, :quoted?)
 
   # A Regexp used to find and convert some common Date formats.
   DateMatcher     = / \A(?: (\w+,?\s+)?\w+\s+\d{1,2},?\s+\d{2,4} |
                             \d{4}-\d{2}-\d{2} )\z /x
-  # A Regexp used to find and convert some common DateTime formats.
+  # A Regexp used to find and convert some common (Date)Time formats.
   DateTimeMatcher =
     / \A(?: (\w+,?\s+)?\w+\s+\d{1,2}\s+\d{1,2}:\d{1,2}:\d{1,2},?\s+\d{2,4} |
-            \d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2} |
-            # ISO-8601
+            # ISO-8601 and RFC-3339 (space instead of T) recognized by (Date)Time.parse
             \d{4}-\d{2}-\d{2}
-              (?:T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?(?:[+-]\d{2}(?::\d{2})|Z)?)?)?
+              (?:[T\s]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?(?:[+-]\d{2}(?::\d{2})|Z)?)?)?
         )\z /x
 
   # The encoding used by all converters.
@@ -903,6 +924,14 @@ class CSV
         f
       end
     },
+    time: lambda { |f|
+      begin
+        e = f.encode(ConverterEncoding)
+        e.match?(DateTimeMatcher) ? Time.parse(e) : f
+      rescue  # encoding conversion or parse errors
+        f
+      end
+    },
     all:       [:date_time, :numeric],
   }
 
@@ -917,8 +946,10 @@ class CSV
     symbol:   lambda { |h|
       h.encode(ConverterEncoding).downcase.gsub(/[^\s\w]+/, "").strip.
                                            gsub(/\s+/, "_").to_sym
-    }
+    },
+    symbol_raw: lambda { |h| h.encode(ConverterEncoding).to_sym }
   }
+
   # Default values for method options.
   DEFAULT_OPTIONS = {
     # For both parsing and generating.
@@ -927,6 +958,7 @@ class CSV
     quote_char:         '"',
     # For parsing.
     field_size_limit:   nil,
+    max_field_size:     nil,
     converters:         nil,
     unconverted_fields: nil,
     headers:            false,
@@ -937,6 +969,7 @@ class CSV
     liberal_parsing:    false,
     nil_value:          nil,
     empty_value:        "",
+    strip:              false,
     # For generating.
     write_headers:      nil,
     quote_empty:        true,
@@ -944,7 +977,6 @@ class CSV
     write_converters:   nil,
     write_nil_value:    nil,
     write_empty_value:  "",
-    strip:              false,
   }.freeze
 
   class << self
@@ -957,6 +989,8 @@ class CSV
     # Creates or retrieves cached \CSV objects.
     # For arguments and options, see CSV.new.
     #
+    # This API is not Ractor-safe.
+    #
     # ---
     #
     # With no block given, returns a \CSV object.
@@ -992,7 +1026,7 @@ class CSV
     def instance(data = $stdout, **options)
       # create a _signature_ for this method call, data object and options
       sig = [data.object_id] +
-            options.values_at(*DEFAULT_OPTIONS.keys.sort_by { |sym| sym.to_s })
+            options.values_at(*DEFAULT_OPTIONS.keys)
 
       # fetch or create the instance for this signature
       @@instances ||= Hash.new
@@ -1006,65 +1040,227 @@ class CSV
     end
 
     # :call-seq:
-    #   filter(**options) {|row| ... }
-    #   filter(in_string, **options) {|row| ... }
-    #   filter(in_io, **options) {|row| ... }
-    #   filter(in_string, out_string, **options) {|row| ... }
-    #   filter(in_string, out_io, **options) {|row| ... }
-    #   filter(in_io, out_string, **options) {|row| ... }
-    #   filter(in_io, out_io, **options) {|row| ... }
-    #
-    # Reads \CSV input and writes \CSV output.
-    #
-    # For each input row:
-    # - Forms the data into:
-    #   - A CSV::Row object, if headers are in use.
-    #   - An \Array of Arrays, otherwise.
-    # - Calls the block with that object.
-    # - Appends the block's return value to the output.
+    #   filter(in_string_or_io, **options) {|row| ... } -> array_of_arrays or csv_table
+    #   filter(in_string_or_io, out_string_or_io, **options) {|row| ... } -> array_of_arrays or csv_table
+    #   filter(**options) {|row| ... } -> array_of_arrays or csv_table
     #
-    # Arguments:
-    # * \CSV source:
-    #   * Argument +in_string+, if given, should be a \String object;
-    #     it will be put into a new StringIO object positioned at the beginning.
-    #   * Argument +in_io+, if given, should be an IO object that is
-    #     open for reading; on return, the IO object will be closed.
-    #   * If neither  +in_string+ nor +in_io+ is given,
-    #     the input stream defaults to {ARGF}[https://ruby-doc.org/core/ARGF.html].
-    # * \CSV output:
-    #   * Argument +out_string+, if given, should be a \String object;
-    #     it will be put into a new StringIO object positioned at the beginning.
-    #   * Argument +out_io+, if given, should be an IO object that is
-    #     ppen for writing; on return, the IO object will be closed.
-    #   * If neither +out_string+ nor +out_io+ is given,
-    #     the output stream defaults to <tt>$stdout</tt>.
-    # * Argument +options+ should be keyword arguments.
-    #   - Each argument name that is prefixed with +in_+ or +input_+
-    #     is stripped of its prefix and is treated as an option
-    #     for parsing the input.
-    #     Option +input_row_sep+ defaults to <tt>$INPUT_RECORD_SEPARATOR</tt>.
-    #   - Each argument name that is prefixed with +out_+ or +output_+
-    #     is stripped of its prefix and is treated as an option
-    #     for generating the output.
-    #     Option +output_row_sep+ defaults to <tt>$INPUT_RECORD_SEPARATOR</tt>.
-    #   - Each argument not prefixed as above is treated as an option
-    #     both for parsing the input and for generating the output.
-    #   - See {Options for Parsing}[#class-CSV-label-Options+for+Parsing]
-    #     and {Options for Generating}[#class-CSV-label-Options+for+Generating].
+    # - Parses \CSV from a source (\String, \IO stream, or ARGF).
+    # - Calls the given block with each parsed row:
+    #   - Without headers, each row is an \Array.
+    #   - With headers, each row is a CSV::Row.
+    # - Generates \CSV to an output (\String, \IO stream, or STDOUT).
+    # - Returns the parsed source:
+    #   - Without headers, an \Array of \Arrays.
+    #   - With headers, a CSV::Table.
     #
-    # Example:
-    #   in_string = "foo,0\nbar,1\nbaz,2\n"
+    # When +in_string_or_io+ is given, but not +out_string_or_io+,
+    # parses from the given +in_string_or_io+
+    # and generates to STDOUT.
+    #
+    # \String input without headers:
+    #
+    #   in_string = "foo,0\nbar,1\nbaz,2"
+    #   CSV.filter(in_string) do |row|
+    #     row[0].upcase!
+    #     row[1] = - row[1].to_i
+    #   end # => [["FOO", 0], ["BAR", -1], ["BAZ", -2]]
+    #
+    # Output (to STDOUT):
+    #
+    #   FOO,0
+    #   BAR,-1
+    #   BAZ,-2
+    #
+    # \String input with headers:
+    #
+    #   in_string = "Name,Value\nfoo,0\nbar,1\nbaz,2"
+    #   CSV.filter(in_string, headers: true) do |row|
+    #     row[0].upcase!
+    #     row[1] = - row[1].to_i
+    #   end # => #<CSV::Table mode:col_or_row row_count:4>
+    #
+    # Output (to STDOUT):
+    #
+    #   Name,Value
+    #   FOO,0
+    #   BAR,-1
+    #   BAZ,-2
+    #
+    # \IO stream input without headers:
+    #
+    #   File.write('t.csv', "foo,0\nbar,1\nbaz,2")
+    #   File.open('t.csv') do |in_io|
+    #     CSV.filter(in_io) do |row|
+    #       row[0].upcase!
+    #       row[1] = - row[1].to_i
+    #     end
+    #   end # => [["FOO", 0], ["BAR", -1], ["BAZ", -2]]
+    #
+    # Output (to STDOUT):
+    #
+    #   FOO,0
+    #   BAR,-1
+    #   BAZ,-2
+    #
+    # \IO stream input with headers:
+    #
+    #   File.write('t.csv', "Name,Value\nfoo,0\nbar,1\nbaz,2")
+    #   File.open('t.csv') do |in_io|
+    #     CSV.filter(in_io, headers: true) do |row|
+    #       row[0].upcase!
+    #       row[1] = - row[1].to_i
+    #     end
+    #   end # => #<CSV::Table mode:col_or_row row_count:4>
+    #
+    # Output (to STDOUT):
+    #
+    #   Name,Value
+    #   FOO,0
+    #   BAR,-1
+    #   BAZ,-2
+    #
+    # When both +in_string_or_io+ and +out_string_or_io+ are given,
+    # parses from +in_string_or_io+ and generates to +out_string_or_io+.
+    #
+    # \String output without headers:
+    #
+    #   in_string = "foo,0\nbar,1\nbaz,2"
     #   out_string = ''
     #   CSV.filter(in_string, out_string) do |row|
-    #     row[0] = row[0].upcase
-    #     row[1] *= 4
-    #   end
-    #   out_string # => "FOO,0000\nBAR,1111\nBAZ,2222\n"
+    #     row[0].upcase!
+    #     row[1] = - row[1].to_i
+    #   end # => [["FOO", 0], ["BAR", -1], ["BAZ", -2]]
+    #   out_string # => "FOO,0\nBAR,-1\nBAZ,-2\n"
+    #
+    # \String output with headers:
+    #
+    #   in_string = "Name,Value\nfoo,0\nbar,1\nbaz,2"
+    #   out_string = ''
+    #   CSV.filter(in_string, out_string, headers: true) do |row|
+    #     row[0].upcase!
+    #     row[1] = - row[1].to_i
+    #   end # => #<CSV::Table mode:col_or_row row_count:4>
+    #   out_string # => "Name,Value\nFOO,0\nBAR,-1\nBAZ,-2\n"
+    #
+    # \IO stream output without headers:
+    #
+    #   in_string = "foo,0\nbar,1\nbaz,2"
+    #   File.open('t.csv', 'w') do |out_io|
+    #     CSV.filter(in_string, out_io) do |row|
+    #       row[0].upcase!
+    #       row[1] = - row[1].to_i
+    #     end
+    #   end # => [["FOO", 0], ["BAR", -1], ["BAZ", -2]]
+    #   File.read('t.csv') # => "FOO,0\nBAR,-1\nBAZ,-2\n"
+    #
+    # \IO stream output with headers:
+    #
+    #   in_string = "Name,Value\nfoo,0\nbar,1\nbaz,2"
+    #   File.open('t.csv', 'w') do |out_io|
+    #     CSV.filter(in_string, out_io, headers: true) do |row|
+    #       row[0].upcase!
+    #       row[1] = - row[1].to_i
+    #     end
+    #   end # => #<CSV::Table mode:col_or_row row_count:4>
+    #   File.read('t.csv') # => "Name,Value\nFOO,0\nBAR,-1\nBAZ,-2\n"
+    #
+    # When neither +in_string_or_io+ nor +out_string_or_io+ given,
+    # parses from {ARGF}[rdoc-ref:ARGF]
+    # and generates to STDOUT.
+    #
+    # Without headers:
+    #
+    #   # Put Ruby code into a file.
+    #   ruby = <<-EOT
+    #     require 'csv'
+    #     CSV.filter do |row|
+    #       row[0].upcase!
+    #       row[1] = - row[1].to_i
+    #     end
+    #   EOT
+    #   File.write('t.rb', ruby)
+    #   # Put some CSV into a file.
+    #   File.write('t.csv', "foo,0\nbar,1\nbaz,2")
+    #   # Run the Ruby code with CSV filename as argument.
+    #   system(Gem.ruby, "t.rb", "t.csv")
+    #
+    # Output (to STDOUT):
+    #
+    #   FOO,0
+    #   BAR,-1
+    #   BAZ,-2
+    #
+    # With headers:
+    #
+    #   # Put Ruby code into a file.
+    #   ruby = <<-EOT
+    #     require 'csv'
+    #     CSV.filter(headers: true) do |row|
+    #       row[0].upcase!
+    #       row[1] = - row[1].to_i
+    #     end
+    #   EOT
+    #   File.write('t.rb', ruby)
+    #   # Put some CSV into a file.
+    #   File.write('t.csv', "Name,Value\nfoo,0\nbar,1\nbaz,2")
+    #   # Run the Ruby code with CSV filename as argument.
+    #   system(Gem.ruby, "t.rb", "t.csv")
+    #
+    # Output (to STDOUT):
+    #
+    #   Name,Value
+    #   FOO,0
+    #   BAR,-1
+    #   BAZ,-2
+    #
+    # Arguments:
+    #
+    # * Argument +in_string_or_io+ must be a \String or an \IO stream.
+    # * Argument +out_string_or_io+ must be a \String or an \IO stream.
+    # * Arguments <tt>**options</tt> must be keyword options.
+    #
+    #   - Each option defined as an {option for parsing}[#class-CSV-label-Options+for+Parsing]
+    #     is used for parsing the filter input.
+    #   - Each option defined as an {option for generating}[#class-CSV-label-Options+for+Generating]
+    #     is used for generator the filter input.
+    #
+    # However, there are three options that may be used for both parsing and generating:
+    # +col_sep+, +quote_char+, and +row_sep+.
+    #
+    # Therefore for method +filter+ (and method +filter+ only),
+    # there are special options that allow these parsing and generating options
+    # to be specified separately:
+    #
+    # - Options +input_col_sep+ and +output_col_sep+
+    #   (and their aliases +in_col_sep+ and +out_col_sep+)
+    #   specify the column separators for parsing and generating.
+    # - Options +input_quote_char+ and +output_quote_char+
+    #   (and their aliases +in_quote_char+ and +out_quote_char+)
+    #   specify the quote characters for parsing and generting.
+    # - Options +input_row_sep+ and +output_row_sep+
+    #   (and their aliases +in_row_sep+ and +out_row_sep+)
+    #   specify the row separators for parsing and generating.
+    #
+    # Example options (for column separators):
+    #
+    #   CSV.filter                                    # Default for both parsing and generating.
+    #   CSV.filter(in_col_sep: ';')                   # ';' for parsing, default for generating.
+    #   CSV.filter(out_col_sep: '|')                  # Default for parsing, '|' for generating.
+    #   CSV.filter(in_col_sep: ';', out_col_sep: '|') # ';' for parsing, '|' for generating.
+    #
+    # Note that for a special option (e.g., +input_col_sep+)
+    # and its corresponding "regular" option (e.g., +col_sep+),
+    # the two are mutually overriding.
+    #
+    # Another example (possibly surprising):
+    #
+    #   CSV.filter(in_col_sep: ';', col_sep: '|') # '|' for both parsing(!) and generating.
+    #
     def filter(input=nil, output=nil, **options)
       # parse options for input, output, or both
-      in_options, out_options = Hash.new, {row_sep: $INPUT_RECORD_SEPARATOR}
+      in_options, out_options = Hash.new, {row_sep: InputRecordSeparator.value}
       options.each do |key, value|
-        case key.to_s
+        case key
         when /\Ain(?:put)?_(.+)\Z/
           in_options[$1.to_sym] = value
         when /\Aout(?:put)?_(.+)\Z/
@@ -1106,111 +1302,90 @@ class CSV
 
     #
     # :call-seq:
-    #   foreach(path, mode='r', **options) {|row| ... )
-    #   foreach(io, mode='r', **options {|row| ... )
-    #   foreach(path, mode='r', headers: ..., **options) {|row| ... )
-    #   foreach(io, mode='r', headers: ..., **options {|row| ... )
-    #   foreach(path, mode='r', **options) -> new_enumerator
-    #   foreach(io, mode='r', **options -> new_enumerator
-    #
-    # Calls the block with each row read from source +path+ or +io+.
-    #
-    # * Argument +path+, if given, must be the path to a file.
-    # :include: ../doc/csv/arguments/io.rdoc
-    # * Argument +mode+, if given, must be a \File mode
-    #   See {Open Mode}[IO.html#method-c-new-label-Open+Mode].
-    # * Arguments <tt>**options</tt> must be keyword options.
-    #   See {Options for Parsing}[#class-CSV-label-Options+for+Parsing].
-    # * This method optionally accepts an additional <tt>:encoding</tt> option
-    #   that you can use to specify the Encoding of the data read from +path+ or +io+.
-    #   You must provide this unless your data is in the encoding
-    #   given by <tt>Encoding::default_external</tt>.
-    #   Parsing will use this to determine how to parse the data.
-    #   You may provide a second Encoding to
-    #   have the data transcoded as it is read. For example,
-    #     encoding: 'UTF-32BE:UTF-8'
-    #   would read +UTF-32BE+ data from the file
-    #   but transcode it to +UTF-8+ before parsing.
+    #   foreach(path_or_io, mode='r', **options) {|row| ... )
+    #   foreach(path_or_io, mode='r', **options) -> new_enumerator
     #
-    # ====== Without Option +headers+
+    # Calls the block with each row read from source +path_or_io+.
     #
-    # Without option +headers+, returns each row as an \Array object.
+    # \Path input without headers:
     #
-    # These examples assume prior execution of:
     #   string = "foo,0\nbar,1\nbaz,2\n"
-    #   path = 't.csv'
-    #   File.write(path, string)
+    #   in_path = 't.csv'
+    #   File.write(in_path, string)
+    #   CSV.foreach(in_path) {|row| p row }
     #
-    # Read rows from a file at +path+:
-    #   CSV.foreach(path) {|row| p row }
     # Output:
-    #   ["foo", "0"]
-    #   ["bar", "1"]
-    #   ["baz", "2"]
     #
-    # Read rows from an \IO object:
-    #   File.open(path) do |file|
-    #     CSV.foreach(file) {|row| p row }
-    #   end
-    #
-    # Output:
     #   ["foo", "0"]
     #   ["bar", "1"]
     #   ["baz", "2"]
     #
-    # Returns a new \Enumerator if no block given:
-    #   CSV.foreach(path) # => #<Enumerator: CSV:foreach("t.csv", "r")>
-    #   CSV.foreach(File.open(path)) # => #<Enumerator: CSV:foreach(#<File:t.csv>, "r")>
+    # \Path input with headers:
+    #
+    #   string = "Name,Value\nfoo,0\nbar,1\nbaz,2\n"
+    #   in_path = 't.csv'
+    #   File.write(in_path, string)
+    #   CSV.foreach(in_path, headers: true) {|row| p row }
     #
-    # Issues a warning if an encoding is unsupported:
-    #   CSV.foreach(File.open(path), encoding: 'foo:bar') {|row| }
     # Output:
-    #   warning: Unsupported encoding foo ignored
-    #   warning: Unsupported encoding bar ignored
     #
-    # ====== With Option +headers+
+    #   <CSV::Row "Name":"foo" "Value":"0">
+    #   <CSV::Row "Name":"bar" "Value":"1">
+    #   <CSV::Row "Name":"baz" "Value":"2">
     #
-    # With {option +headers+}[#class-CSV-label-Option+headers],
-    # returns each row as a CSV::Row object.
+    # \IO stream input without headers:
     #
-    # These examples assume prior execution of:
-    #   string = "Name,Count\nfoo,0\nbar,1\nbaz,2\n"
+    #   string = "foo,0\nbar,1\nbaz,2\n"
     #   path = 't.csv'
     #   File.write(path, string)
-    #
-    # Read rows from a file at +path+:
-    #   CSV.foreach(path, headers: true) {|row| p row }
+    #   File.open('t.csv') do |in_io|
+    #     CSV.foreach(in_io) {|row| p row }
+    #   end
     #
     # Output:
-    #   #<CSV::Row "Name":"foo" "Count":"0">
-    #   #<CSV::Row "Name":"bar" "Count":"1">
-    #   #<CSV::Row "Name":"baz" "Count":"2">
     #
-    # Read rows from an \IO object:
-    #   File.open(path) do |file|
-    #     CSV.foreach(file, headers: true) {|row| p row }
+    #   ["foo", "0"]
+    #   ["bar", "1"]
+    #   ["baz", "2"]
+    #
+    # \IO stream input with headers:
+    #
+    #   string = "Name,Value\nfoo,0\nbar,1\nbaz,2\n"
+    #   path = 't.csv'
+    #   File.write(path, string)
+    #   File.open('t.csv') do |in_io|
+    #     CSV.foreach(in_io, headers: true) {|row| p row }
     #   end
     #
     # Output:
-    #   #<CSV::Row "Name":"foo" "Count":"0">
-    #   #<CSV::Row "Name":"bar" "Count":"1">
-    #   #<CSV::Row "Name":"baz" "Count":"2">
     #
-    # ---
-    #
-    # Raises an exception if +path+ is a \String, but not the path to a readable file:
-    #   # Raises Errno::ENOENT (No such file or directory @ rb_sysopen - nosuch.csv):
-    #   CSV.foreach('nosuch.csv') {|row| }
+    #   <CSV::Row "Name":"foo" "Value":"0">
+    #   <CSV::Row "Name":"bar" "Value":"1">
+    #   <CSV::Row "Name":"baz" "Value":"2">
     #
-    # Raises an exception if +io+ is an \IO object, but not open for reading:
-    #   io = File.open(path, 'w') {|row| }
-    #   # Raises TypeError (no implicit conversion of nil into String):
-    #   CSV.foreach(io) {|row| }
+    # With no block given, returns an \Enumerator:
     #
-    # Raises an exception if +mode+ is invalid:
-    #   # Raises ArgumentError (invalid access mode nosuch):
-    #   CSV.foreach(path, 'nosuch') {|row| }
+    #   string = "foo,0\nbar,1\nbaz,2\n"
+    #   path = 't.csv'
+    #   File.write(path, string)
+    #   CSV.foreach(path) # => #<Enumerator: CSV:foreach("t.csv", "r")>
     #
+    # Arguments:
+    # * Argument +path_or_io+ must be a file path or an \IO stream.
+    # * Argument +mode+, if given, must be a \File mode.
+    #   See {Access Modes}[https://docs.ruby-lang.org/en/master/File.html#class-File-label-Access+Modes].
+    # * Arguments <tt>**options</tt> must be keyword options.
+    #   See {Options for Parsing}[#class-CSV-label-Options+for+Parsing].
+    # * This method optionally accepts an additional <tt>:encoding</tt> option
+    #   that you can use to specify the Encoding of the data read from +path+ or +io+.
+    #   You must provide this unless your data is in the encoding
+    #   given by <tt>Encoding::default_external</tt>.
+    #   Parsing will use this to determine how to parse the data.
+    #   You may provide a second Encoding to
+    #   have the data transcoded as it is read. For example,
+    #     encoding: 'UTF-32BE:UTF-8'
+    #   would read +UTF-32BE+ data from the file
+    #   but transcode it to +UTF-8+ before parsing.
     def foreach(path, mode="r", **options, &block)
       return to_enum(__method__, path, mode, **options) unless block_given?
       open(path, mode, **options) do |csv|
@@ -1303,8 +1478,8 @@ class CSV
     # Argument +ary+ must be an \Array.
     #
     # Special options:
-    # * Option <tt>:row_sep</tt> defaults to <tt>$INPUT_RECORD_SEPARATOR</tt>
-    #   (<tt>$/</tt>).:
+    # * Option <tt>:row_sep</tt> defaults to <tt>"\n"> on Ruby 3.0 or later
+    #   and <tt>$INPUT_RECORD_SEPARATOR</tt> (<tt>$/</tt>) otherwise.:
     #     $INPUT_RECORD_SEPARATOR # => "\n"
     # * This method accepts an additional option, <tt>:encoding</tt>, which sets the base
     #   Encoding for the output. This method will try to guess your Encoding from
@@ -1326,7 +1501,7 @@ class CSV
     #   CSV.generate_line(:foo)
     #
     def generate_line(row, **options)
-      options = {row_sep: $INPUT_RECORD_SEPARATOR}.merge(options)
+      options = {row_sep: InputRecordSeparator.value}.merge(options)
       str = +""
       if options[:encoding]
         str.force_encoding(options[:encoding])
@@ -1348,24 +1523,62 @@ class CSV
       (new(str, **options) << row).string
     end
 
+    # :call-seq:
+    #   CSV.generate_lines(rows)
+    #   CSV.generate_lines(rows, **options)
+    #
+    # Returns the \String created by generating \CSV from
+    # using the specified +options+.
+    #
+    # Argument +rows+ must be an \Array of row. Row is \Array of \String or \CSV::Row.
+    #
+    # Special options:
+    # * Option <tt>:row_sep</tt> defaults to <tt>"\n"</tt> on Ruby 3.0 or later
+    #   and <tt>$INPUT_RECORD_SEPARATOR</tt> (<tt>$/</tt>) otherwise.:
+    #     $INPUT_RECORD_SEPARATOR # => "\n"
+    # * This method accepts an additional option, <tt>:encoding</tt>, which sets the base
+    #   Encoding for the output. This method will try to guess your Encoding from
+    #   the first non-+nil+ field in +row+, if possible, but you may need to use
+    #   this parameter as a backup plan.
+    #
+    # For other +options+,
+    # see {Options for Generating}[#class-CSV-label-Options+for+Generating].
+    #
+    # ---
+    #
+    # Returns the \String generated from an
+    #   CSV.generate_lines([['foo', '0'], ['bar', '1'], ['baz', '2']]) # => "foo,0\nbar,1\nbaz,2\n"
+    #
+    # ---
+    #
+    # Raises an exception
+    #   # Raises NoMethodError (undefined method `each' for :foo:Symbol)
+    #   CSV.generate_lines(:foo)
+    #
+    def generate_lines(rows, **options)
+      self.generate(**options) do |csv|
+        rows.each do |row|
+          csv << row
+        end
+      end
+    end
+
     #
     # :call-seq:
-    #   open(file_path, mode = "rb", **options ) -> new_csv
-    #   open(io, mode = "rb", **options ) -> new_csv
-    #   open(file_path, mode = "rb", **options ) { |csv| ... } -> object
-    #   open(io, mode = "rb", **options ) { |csv| ... } -> object
+    #   open(path_or_io, mode = "rb", **options ) -> new_csv
+    #   open(path_or_io, mode = "rb", **options ) { |csv| ... } -> object
     #
     # possible options elements:
-    #   hash form:
+    #   keyword form:
     #     :invalid => nil      # raise error on invalid byte sequence (default)
     #     :invalid => :replace # replace invalid byte sequence
     #     :undef => :replace   # replace undefined conversion
     #     :replace => string   # replacement string ("?" or "\uFFFD" if not specified)
     #
-    # * Argument +path+, if given, must be the path to a file.
+    # * Argument +path_or_io+, must be a file path or an \IO stream.
     # :include: ../doc/csv/arguments/io.rdoc
-    # * Argument +mode+, if given, must be a \File mode
-    #   See {Open Mode}[IO.html#method-c-new-label-Open+Mode].
+    # * Argument +mode+, if given, must be a \File mode.
+    #   See {Access Modes}[https://docs.ruby-lang.org/en/master/File.html#class-File-label-Access+Modes].
     # * Arguments <tt>**options</tt> must be keyword options.
     #   See {Options for Generating}[#class-CSV-label-Options+for+Generating].
     # * This method optionally accepts an additional <tt>:encoding</tt> option
@@ -1386,6 +1599,9 @@ class CSV
     #   path = 't.csv'
     #   File.write(path, string)
     #
+    #   string_io = StringIO.new
+    #   string_io << "foo,0\nbar,1\nbaz,2\n"
+    #
     # ---
     #
     # With no block given, returns a new \CSV object.
@@ -1398,6 +1614,9 @@ class CSV
     #   csv = CSV.open(File.open(path))
     #   csv # => #<CSV io_type:File io_path:"t.csv" encoding:UTF-8 lineno:0 col_sep:"," row_sep:"\n" quote_char:"\"">
     #
+    # Create a \CSV object using a \StringIO:
+    #   csv = CSV.open(string_io)
+    #   csv # => #<CSV io_type:StringIO encoding:UTF-8 lineno:0 col_sep:"," row_sep:"\n" quote_char:"\"">
     # ---
     #
     # With a block given, calls the block with the created \CSV object;
@@ -1415,27 +1634,46 @@ class CSV
     # Output:
     #   #<CSV io_type:File io_path:"t.csv" encoding:UTF-8 lineno:0 col_sep:"," row_sep:"\n" quote_char:"\"">
     #
+    # Using a \StringIO:
+    #   csv = CSV.open(string_io) {|csv| p csv}
+    #   csv # => #<CSV io_type:StringIO encoding:UTF-8 lineno:0 col_sep:"," row_sep:"\n" quote_char:"\"">
+    # Output:
+    #   #<CSV io_type:StringIO encoding:UTF-8 lineno:0 col_sep:"," row_sep:"\n" quote_char:"\"">
     # ---
     #
     # Raises an exception if the argument is not a \String object or \IO object:
     #   # Raises TypeError (no implicit conversion of Symbol into String)
     #   CSV.open(:foo)
-    def open(filename, mode="r", **options)
+    def open(filename_or_io, mode="r", **options)
       # wrap a File opened with the remaining +args+ with no newline
       # decorator
-      file_opts = {universal_newline: false}.merge(options)
+      file_opts = {}
+      may_enable_bom_detection_automatically(filename_or_io,
+                                             mode,
+                                             options,
+                                             file_opts)
+      file_opts.merge!(options)
+      unless file_opts.key?(:newline)
+        file_opts[:universal_newline] ||= false
+      end
       options.delete(:invalid)
       options.delete(:undef)
       options.delete(:replace)
+      options.delete_if {|k, _| /newline\z/.match?(k)}
 
-      begin
-        f = File.open(filename, mode, **file_opts)
-      rescue ArgumentError => e
-        raise unless /needs binmode/.match?(e.message) and mode == "r"
-        mode = "rb"
-        file_opts = {encoding: Encoding.default_external}.merge(file_opts)
-        retry
+      if filename_or_io.is_a?(StringIO)
+        f = create_stringio(filename_or_io.string, mode, **file_opts)
+      else
+        begin
+          f = File.open(filename_or_io, mode, **file_opts)
+        rescue ArgumentError => e
+          raise unless /needs binmode/.match?(e.message) and mode == "r"
+          mode = "rb"
+          file_opts = {encoding: Encoding.default_external}.merge(file_opts)
+          retry
+        end
       end
+
       begin
         csv = new(f, **options)
       rescue Exception
@@ -1567,6 +1805,23 @@ class CSV
     # Raises an exception if the argument is not a \String object or \IO object:
     #   # Raises NoMethodError (undefined method `close' for :foo:Symbol)
     #   CSV.parse(:foo)
+    #
+    # ---
+    #
+    # Please make sure if your text contains \BOM or not. CSV.parse will not remove
+    # \BOM automatically. You might want to remove \BOM before calling CSV.parse :
+    #   # remove BOM on calling File.open
+    #   File.open(path, encoding: 'bom|utf-8') do |file|
+    #     CSV.parse(file, headers: true) do |row|
+    #       # you can get value by column name because BOM is removed
+    #       p row['Name']
+    #     end
+    #   end
+    #
+    # Output:
+    #   # "foo"
+    #   # "bar"
+    #   # "baz"
     def parse(str, **options, &block)
       csv = new(str, **options)
 
@@ -1681,7 +1936,7 @@ class CSV
     #
     # Calls CSV.read with +source+, +options+, and certain default options:
     # - +headers+: +true+
-    # - +converbers+: +:numeric+
+    # - +converters+: +:numeric+
     # - +header_converters+: +:symbol+
     #
     # Returns a CSV::Table object.
@@ -1700,6 +1955,42 @@ class CSV
       options = default_options.merge(options)
       read(path, **options)
     end
+
+    ON_WINDOWS = /mingw|mswin/.match?(RUBY_PLATFORM)
+    private_constant :ON_WINDOWS
+
+    private
+    def may_enable_bom_detection_automatically(filename_or_io,
+                                               mode,
+                                               options,
+                                               file_opts)
+      if filename_or_io.is_a?(StringIO)
+        # Support to StringIO was dropped for Ruby 2.6 and earlier without BOM support:
+        # https://github.com/ruby/stringio/pull/47
+        return if RUBY_VERSION < "2.7"
+      else
+        # "bom|utf-8" may be buggy on Windows:
+        # https://bugs.ruby-lang.org/issues/20526
+        return if ON_WINDOWS
+      end
+      return unless Encoding.default_external == Encoding::UTF_8
+      return if options.key?(:encoding)
+      return if options.key?(:external_encoding)
+      return if mode.include?(":")
+      file_opts[:encoding] = "bom|utf-8"
+    end
+
+    if RUBY_VERSION < "2.7"
+      def create_stringio(str, mode, opts)
+        opts.delete_if {|k, _| k == :universal_newline or DEFAULT_OPTIONS.key?(k)}
+        raise ArgumentError, "Unsupported options parsing StringIO: #{opts.keys}" unless opts.empty?
+        StringIO.new(str, mode)
+      end
+    else
+      def create_stringio(str, mode, opts)
+        StringIO.new(str, mode, **opts)
+      end
+    end
   end
 
   # :call-seq:
@@ -1745,6 +2036,7 @@ class CSV
                  row_sep: :auto,
                  quote_char: '"',
                  field_size_limit: nil,
+                 max_field_size: nil,
                  converters: nil,
                  unconverted_fields: nil,
                  headers: false,
@@ -1760,16 +2052,27 @@ class CSV
                  encoding: nil,
                  nil_value: nil,
                  empty_value: "",
+                 strip: false,
                  quote_empty: true,
                  write_converters: nil,
                  write_nil_value: nil,
-                 write_empty_value: "",
-                 strip: false)
+                 write_empty_value: "")
     raise ArgumentError.new("Cannot parse nil as CSV") if data.nil?
 
     if data.is_a?(String)
+      if encoding
+        if encoding.is_a?(String)
+          data_external_encoding, data_internal_encoding = encoding.split(":", 2)
+          if data_internal_encoding
+            data = data.encode(data_internal_encoding, data_external_encoding)
+          else
+            data = data.dup.force_encoding(data_external_encoding)
+          end
+        else
+          data = data.dup.force_encoding(encoding)
+        end
+      end
       @io = StringIO.new(data)
-      @io.set_encoding(encoding || data.encoding)
     else
       @io = data
     end
@@ -1787,11 +2090,14 @@ class CSV
     @initial_header_converters = header_converters
     @initial_write_converters = write_converters
 
+    if max_field_size.nil? and field_size_limit
+      max_field_size = field_size_limit - 1
+    end
     @parser_options = {
       column_separator: col_sep,
       row_separator: row_sep,
       quote_character: quote_char,
-      field_size_limit: field_size_limit,
+      max_field_size: max_field_size,
       unconverted_fields: unconverted_fields,
       headers: headers,
       return_headers: return_headers,
@@ -1823,6 +2129,12 @@ class CSV
     writer if @writer_options[:write_headers]
   end
 
+  class TSV < CSV
+    def initialize(data, **options)
+      super(data, **({col_sep: "\t"}.merge(options)))
+    end
+  end
+
   # :call-seq:
   #   csv.col_sep -> string
   #
@@ -1859,10 +2171,24 @@ class CSV
   # Returns the limit for field size; used for parsing;
   # see {Option +field_size_limit+}[#class-CSV-label-Option+field_size_limit]:
   #   CSV.new('').field_size_limit # => nil
+  #
+  # Deprecated since 3.2.3. Use +max_field_size+ instead.
   def field_size_limit
     parser.field_size_limit
   end
 
+  # :call-seq:
+  #   csv.max_field_size -> integer or nil
+  #
+  # Returns the limit for field size; used for parsing;
+  # see {Option +max_field_size+}[#class-CSV-label-Option+max_field_size]:
+  #   CSV.new('').max_field_size # => nil
+  #
+  # Since 3.2.3.
+  def max_field_size
+    parser.max_field_size
+  end
+
   # :call-seq:
   #   csv.skip_lines -> regexp or nil
   #
@@ -1884,6 +2210,10 @@ class CSV
   #   csv.converters # => [:integer]
   #   csv.convert(proc {|x| x.to_s })
   #   csv.converters
+  #
+  # Notes that you need to call
+  # +Ractor.make_shareable(CSV::Converters)+ on the main Ractor to use
+  # this method.
   def converters
     parser_fields_converter.map do |converter|
       name = Converters.rassoc(converter)
@@ -1946,6 +2276,10 @@ class CSV
   # Returns an \Array containing header converters; used for parsing;
   # see {Header Converters}[#class-CSV-label-Header+Converters]:
   #   CSV.new('').header_converters # => []
+  #
+  # Notes that you need to call
+  # +Ractor.make_shareable(CSV::HeaderConverters)+ on the main Ractor
+  # to use this method.
   def header_converters
     header_fields_converter.map do |converter|
       name = HeaderConverters.rassoc(converter)
@@ -1985,7 +2319,7 @@ class CSV
   end
 
   # :call-seq:
-  #   csv.encoding -> endcoding
+  #   csv.encoding -> encoding
   #
   # Returns the encoding used for parsing and generating;
   # see {Character Encodings (M17n or Multilingualization)}[#class-CSV-label-Character+Encodings+-28M17n+or+Multilingualization-29]:
@@ -2353,7 +2687,13 @@ class CSV
   #     p row
   #   end
   def each(&block)
-    parser_enumerator.each(&block)
+    return to_enum(__method__) unless block_given?
+    begin
+      while true
+        yield(parser_enumerator.next)
+      end
+    rescue StopIteration
+    end
   end
 
   # :call-seq:
@@ -2586,7 +2926,7 @@ class CSV
 
   def build_parser_fields_converter
     specific_options = {
-      builtin_converters: Converters,
+      builtin_converters_name: :Converters,
     }
     options = @base_fields_converter_options.merge(specific_options)
     build_fields_converter(@initial_converters, options)
@@ -2598,7 +2938,7 @@ class CSV
 
   def build_header_fields_converter
     specific_options = {
-      builtin_converters: HeaderConverters,
+      builtin_converters_name: :HeaderConverters,
       accept_nil: true,
     }
     options = @base_fields_converter_options.merge(specific_options)
@@ -2661,8 +3001,15 @@ end
 #     c.read.any? { |a| a.include?("zombies") }
 #   } #=> false
 #
-def CSV(*args, &block)
-  CSV.instance(*args, &block)
+# CSV options may also be given.
+#
+#   io = StringIO.new
+#   CSV(io, col_sep: ";") { |csv| csv << ["a", "b", "c"] }
+#
+# This API is not Ractor-safe.
+#
+def CSV(*args, **options, &block)
+  CSV.instance(*args, **options, &block)
 end
 
 require_relative "csv/version"