strong_csv 0.8.6 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 642cc6ac08fe76d56643c190d645fc65f4c5d8a7dbe8a304ad38c265918ae40e
-  data.tar.gz: b83f3232be02019f69ee13774a2b0299bacb25810ed7b7b35bd5b2e68126332f
+  metadata.gz: dd735e06083068bf676047a6ac4f2a0441bc3b36cb3455e04ac7ae2c0206acb6
+  data.tar.gz: 77be5288dbd7854f82385367b62352c4ac98e41f108046c99fc68dadc8611890
 SHA512:
-  metadata.gz: a9d429bfef2e5c40ff3c1b1e9d37f299f2f0f66607b5fc513357a1d03b8b20a16721ead6b12c1a7e8533ad8b31d8c35fea777c870a70c6ee9cfd8b5a912646dc
-  data.tar.gz: 4d393b455dbcd89a8a6fd051f781c6776ed5ae5c802ab582898843457748fb37d10d0cdd94c7b430a89decd9532bc9e6267a02d402433e00ea15fcdf00833612
+  metadata.gz: 3d2bbc51b2006f8769ce112c3a4280f6ca0000e5503680716cd218eb06db1011805ab7ff6113ad9804f58e26af03921f9a1f188f1626a7e6a2715663df9fa082
+  data.tar.gz: 2e531d2ecbe4e9b035a882701a1eedd342ccf6bacea712964517d68b56f73ecf0654afb26229e6e1596bfb7d77805cf09c6c5d6d1e67ba337eecc94f57fedb22

data/README.md

@@ -108,6 +108,11 @@ strong_csv.parse(data, field_size_limit: 2048) do |row|
 end
 ```
 
+`pick` buffers all values of the specified column in memory before invoking the block,
+because the block receives the complete array. For very large CSV files, this can consume
+a large amount of memory; the streaming form of `StrongCSV#parse` does not remove this
+buffering for picked columns.
+
 You can also define types without CSV headers by specifying column numbers.
 
 > [!NOTE]
@@ -302,7 +307,19 @@ result[2].slice(:name, :description) # => {:name=>nil, :description=>"🤷"} (`n
 The value must be casted to Time. `time?` allows the value to be `nil` as an optional time value.
 They have the `:format` argument, which is used as the format
 of [`Time.strptime`](https://rubydoc.info/stdlib/time/Time.strptime);
-it means you can ensure the value must satisfy the time format. The default value of `:format` is `"%Y-%m-%d"`.
+it means you can ensure the value must satisfy the time format.
+The whole value must match the format; trailing characters are rejected.
+A value with a timezone directive is invalid when the zone cannot be
+resolved to an offset; numeric offsets must use a supported form such as
+`+05`, `+0530`, `+05:30`, or `+05:30:45`. Use `%z` for numeric offsets
+and `%Z` for named zones. Zone-bearing
+formats other than `%s` or `%Q` must include a year, and day-of-year
+and week-date directives are not supported with a timezone. Fractional
+epoch values with a zero whole-second part are invalid. Zone names
+follow Ruby's built-in abbreviation table, so prefer a numeric offset
+when the offset must be unambiguous. Values without a timezone directive
+are interpreted in the process timezone. The default value of `:format`
+is `"%Y-%m-%d"`.
 
 _Example_
 
@@ -379,6 +396,14 @@ result[1].slice(0, 1, 2, 3, 4) # => {0=>123, 1=>"test", 2=>2.5, 3=>"0", 4=>"xyz"
 result[2].slice(0, 1, 2, 3, 4) # => {0=>123, 1=>"Hey", 2=>2.5, 3=>10, 4=>"!"} ("Hey" is not equal to "test", and "!" does not match /[a-z]+/)
 ```
 
+> [!NOTE]
+> On Ruby 3.2 and later, Regexp literals are matched with a one-second timeout per match by
+> default. A match that exceeds the timeout is treated as a non-match. On Ruby 3.2 and later,
+> pass a Regexp with an explicit timeout to change the limit, e.g.
+> `let :id, Regexp.new("\\A[a-z]+\\z", timeout: 5.0)`. A per-Regexp timeout or the global
+> `Regexp.timeout` takes precedence over the default. Ruby 3.1 and earlier have no regexp
+> timeout, and the `timeout:` keyword must not be used there because it is not supported.
+
 ### Union
 
 There would be a case that it's alright if a value satisfies one of the types.

data/lib/strong_csv/let.rb

@@ -17,6 +17,7 @@ class StrongCSV
       @headers = false
       @pickers = {}
       @picked = {}
+      @picker_columns = {}
     end
 
     # @param name [String, Symbol, Integer]
@@ -39,6 +40,9 @@ class StrongCSV
     # This might be useful for the case where you want to receive IDs that are stored in a database,
     # and you want to make sure the IDs actually exist.
     #
+    # The block receives all values of the column at once, so they are buffered in memory
+    # for the entire CSV.
+    #
     # @example
     #  pick :user_id, as: :user_ids do |user_ids|
     #    User.where(id: user_ids).ids
@@ -51,9 +55,24 @@ class StrongCSV
       define_singleton_method(as) do
         @picked[as]
       end
-      @pickers[as] = lambda do |csv|
-        @picked[as] = block.call(csv.map { |row| row[column] })
+      @pickers[as] = lambda do |values|
+        @picked[as] = block.call(values)
       end
+      @picker_columns[as] = column
+    end
+
+    # It collects the values for each picker in a single pass over the CSV,
+    # then invokes the picker blocks with the collected values.
+    # The collected values are held in memory until the picker blocks return.
+    # The CSV stream is rewound before returning so the caller can continue
+    # reading from the beginning.
+    def pick_all(csv)
+      return if @pickers.empty?
+
+      buffers = Hash.new { |hash, key| hash[key] = [] }
+      csv.each { |row| @pickers.each_key { |as| buffers[as] << row[@picker_columns[as]] } }
+      csv.rewind
+      @pickers.each { |as, picker| picker.call(buffers[as]) }
     end
 
     # @param options [Hash] See `Types::Integer#initialize` for more details.

data/lib/strong_csv/types/literal.rb

@@ -5,6 +5,45 @@ class StrongCSV
     # Literal is a module to collect all the literal types for CSV parsing.
     # This module is intended for refining built-in classes and provide `#cast` like `Base` class.
     module Literal
+      # The default time limit, in seconds, for matching a Regexp literal on Ruby 3.2 and later.
+      REGEXP_TIMEOUT = 1.0
+      private_constant :REGEXP_TIMEOUT
+
+      class << self
+        # Returns a Regexp that enforces the default match timeout.
+        #
+        # The original Regexp is returned when the running Ruby has no regexp
+        # timeout support, or when a per-Regexp or global timeout is already set.
+        #
+        # @param regexp [Regexp]
+        # @return [Regexp]
+        def regexp_with_timeout(regexp)
+          return regexp unless defined?(::Regexp::TimeoutError)
+          return regexp if regexp.timeout || ::Regexp.timeout
+
+          ::Regexp.new(regexp.source, regexp.options, timeout: REGEXP_TIMEOUT)
+        end
+
+        # Matches the value against the Regexp with the default timeout.
+        #
+        # @param regexp [Regexp]
+        # @param value [Object]
+        # @return [Integer, nil, :timeout] The match position, nil, or :timeout when the match exceeded the limit
+        def regexp_match(regexp, value)
+          matcher = regexp_with_timeout(regexp)
+
+          if defined?(::Regexp::TimeoutError)
+            begin
+              matcher =~ value
+            rescue ::Regexp::TimeoutError
+              :timeout
+            end
+          else
+            matcher =~ value
+          end
+        end
+      end
+
       refine ::Integer do
         # @param value [Object] Value to be casted to Integer
         # @return [ValueResult]
@@ -79,10 +118,13 @@ class StrongCSV
         def cast(value)
           return ValueResult.new(original_value: value, error_messages: ["`#{value.inspect}` can't be casted to String"]) if value.nil?
 
-          if self =~ value
-            ValueResult.new(value: value, original_value: value)
-          else
+          case Literal.regexp_match(self, value)
+          when :timeout
+            ValueResult.new(original_value: value, error_messages: ["`#{inspect}` matching timed out"])
+          when nil
             ValueResult.new(original_value: value, error_messages: ["`#{value.inspect}` did not match `#{inspect}`"])
+          else
+            ValueResult.new(value: value, original_value: value)
           end
         end
       end

data/lib/strong_csv/types/time.rb

@@ -4,23 +4,109 @@ class StrongCSV
   module Types
     # Time type
     class Time < Base
+      # Matches the numeric offset forms accepted by this type: +05, +0530,
+      # +05:30, +05:30:45, and the same forms with a GMT/UTC prefix.
+      OFFSET_TOKEN = /\A(?:gmt|utc)?[+-](?:[01]\d|2[0-3])(?::[0-5]\d(?::[0-5]\d)?|[0-5]\d(?:[0-5]\d)?)?\z/i.freeze
+      private_constant :OFFSET_TOKEN
+
       # @param format [String] The format of #strptime
       def initialize(format: "%Y-%m-%d")
         raise ArgumentError, "`format` must be a String" unless format.is_a?(::String)
 
         super()
         @format = format
+        @zone_name_allowed = format.include?("%Z")
       end
 
       # @param value [Object] Value to be casted to Time
       # @return [ValueResult]
       def cast(value)
-        return ValueResult.new(original_value: value, error_messages: ["`#{value.inspect}` can't be casted to Time with the format `#{@format.inspect}`"]) if value.nil?
+        return failure(value) if value.nil?
+
+        string = value.to_s
+        # Time.strptime stops at the end of the format and ignores the rest,
+        # so check the unconsumed suffix reported by Date._strptime first.
+        parsed = ::Date._strptime(string, @format)
+        return failure(value) unless parsed_valid?(parsed)
+
+        time = parsed.key?(:zone) ? build_zoned_time(parsed) : ::Time.strptime(string, @format)
+        return failure(value) if time.nil?
+
+        ValueResult.new(value: time, original_value: value)
+      rescue ArgumentError, RangeError
+        failure(value)
+      end
+
+      private
 
-        ValueResult.new(value: ::Time.strptime(value.to_s, @format), original_value: value)
-      rescue ArgumentError
+      def failure(value)
         ValueResult.new(original_value: value, error_messages: ["`#{value.inspect}` can't be casted to Time with the format `#{@format.inspect}`"])
       end
+
+      def parsed_valid?(parsed)
+        return false if parsed.nil? || parsed[:leftover]
+        return false if ambiguous_epoch?(parsed)
+
+        !invalid_zone?(parsed)
+      end
+
+      # Date._strptime loses the sign of a fractional epoch whose whole-second
+      # part is zero, so such values cannot be casted reliably.
+      def ambiguous_epoch?(parsed)
+        parsed.key?(:seconds) && parsed[:seconds].zero? && parsed.fetch(:sec_fraction, 0).positive?
+      end
+
+      # Date._strptime accepts loose zone tokens and may leave :offset nil,
+      # in which case Time.strptime silently uses the local timezone.
+      def invalid_zone?(parsed)
+        zone = parsed[:zone]
+        return false if zone.nil?
+        return true if parsed[:offset].nil?
+        return true unless valid_zone_token?(zone)
+
+        false
+      end
+
+      def valid_zone_token?(zone)
+        return OFFSET_TOKEN.match?(zone) if zone.match?(/\d/)
+        return true if zone == "Z"
+
+        @zone_name_allowed
+      end
+
+      # Build the Time from the parsed fields so the result does not depend on
+      # the host timezone or its DST rules. Zone-bearing formats other than epoch
+      # directives must include a year; day-of-year and week-date directives are
+      # not supported with a zone.
+      def build_zoned_time(parsed)
+        return nil if unsupported_date_fields?(parsed)
+
+        offset = parsed[:offset]
+        return build_zoned_epoch(parsed, offset) if parsed.key?(:seconds)
+        return nil unless parsed.key?(:year)
+
+        time = ::Time.new(parsed[:year], parsed.fetch(:mon, 1), parsed.fetch(:mday, 1), 0, 0, 0, offset) + seconds_of_day(parsed)
+        utc_time(time, offset)
+      end
+
+      def build_zoned_epoch(parsed, offset)
+        seconds = parsed[:seconds]
+        fraction = parsed.fetch(:sec_fraction, 0)
+        seconds += seconds.negative? ? -fraction : fraction
+        utc_time(::Time.at(seconds).getlocal(offset), offset)
+      end
+
+      def seconds_of_day(parsed)
+        (parsed.fetch(:hour, 0) * 3600) + (parsed.fetch(:min, 0) * 60) + parsed.fetch(:sec, 0) + parsed.fetch(:sec_fraction, 0)
+      end
+
+      def utc_time(time, offset)
+        offset.zero? ? time.utc : time
+      end
+
+      def unsupported_date_fields?(parsed)
+        parsed.key?(:yday) || parsed.key?(:cwyear) || parsed.key?(:cweek) || parsed.key?(:cwday) || parsed.key?(:wnum0) || parsed.key?(:wnum1)
+      end
     end
   end
 end

data/lib/strong_csv/types/union.rb

@@ -16,11 +16,15 @@ class StrongCSV
       # @param value [Object] Value to be casted to Integer
       # @return [ValueResult]
       def cast(value)
-        results = @types.map { |type| type.cast(value) }
-        results.find(&:success?) || results.reduce do |memo, result|
-          memo.error_messages.concat(result.error_messages).uniq!
-          memo
+        error_messages = nil
+        @types.each do |type|
+          result = type.cast(value)
+          return result if result.success?
+
+          error_messages = (error_messages || []).concat(result.error_messages).uniq
         end
+
+        ValueResult.new(original_value: value, error_messages: error_messages)
       end
     end
   end

data/lib/strong_csv/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class StrongCSV
-  VERSION = "0.8.6"
+  VERSION = "0.9.0"
 end

data/lib/strong_csv.rb

@@ -52,10 +52,7 @@ class StrongCSV
     options = options.merge(headers: @let.headers, header_converters: :symbol)
     csv = CSV.new(csv, **options)
 
-    @let.pickers.each_value do |picker|
-      picker.call(csv)
-      csv.rewind
-    end
+    @let.pick_all(csv)
 
     if block_given?
       csv.each do |row|

data/sig/strong_csv.rbs

@@ -64,6 +64,12 @@ class StrongCSV
     end
 
     module Literal
+      REGEXP_TIMEOUT: ::Float
+
+      def self.regexp_with_timeout: (::Regexp) -> ::Regexp
+
+      def self.regexp_match: (::Regexp, untyped) -> (::Integer | nil | :timeout)
+
       def cast: (untyped value) -> ValueResult
         | (untyped value) -> ValueResult
         | (untyped value) -> ValueResult
@@ -106,6 +112,7 @@ class StrongCSV
 
   class Let
     @picked: Hash[untyped, untyped]
+    @picker_columns: Hash[untyped, column]
 
     attr_reader types: Array[TypeWrapper]
     attr_reader headers: bool
@@ -117,6 +124,8 @@ class StrongCSV
 
     def pick: (column, as: ::Symbol) { (Array[::String]) -> untyped } -> void
 
+    def pick_all: (untyped) -> void
+
     def integer: (**untyped) -> Types::Integer
 
     def integer?: (**untyped) -> Types::Optional

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: strong_csv
 version: !ruby/object:Gem::Version
-  version: 0.8.6
+  version: 0.9.0
 platform: ruby
 authors:
 - Yutaka Kamei