reading 0.8.0 → 0.9.1

data/lib/reading/parsing/attributes/experiences/history_transformer.rb

@@ -1,5 +1,16 @@
-require_relative "spans_validator"
-
+require_relative 'spans_validator'
+
+# TODO Refactor! This entire file has become 🤢🤮 with the accumulation of new
+# features in the History column.
+#
+# Goals of the refactor:
+#   - if possible, avoid daily_spans; build spans with date ranges directly.
+#   - validate spans at every step; that way the origin of bugs will be easier
+#     to find, e.g. for the bug fixed in 6310639, spans became invalid in
+#     #fix_open_ranges! and led to an error elsewhere that didn't give a trace
+#     back to the origin.
+#   - to facilitate the points above, create a class ExperienceBuilder to
+#     contain much of the logic that is currently in this file.
 module Reading
   module Parsing
     module Attributes
@@ -16,13 +27,14 @@ module Reading
           # many days, for example.
           AVERAGE_DAYS_IN_A_MONTH = 30.437r
 
-          private attr_reader :parsed_row, :config
+          private attr_reader :parsed_row, :head_index, :next_open_range_id
 
           # @param parsed_row [Hash] a parsed row (the intermediate hash).
-          # @param config [Hash] an entire config
-          def initialize(parsed_row, config)
+          # @param head_index [Integer] current item's position in the Head column.
+          def initialize(parsed_row, head_index)
             @parsed_row = parsed_row
-            @config = config
+            @head_index = head_index
+            @next_open_range_id = 0
           end
 
           # Extracts experiences from the parsed row.
@@ -37,8 +49,7 @@ module Reading
               }
             }
 
-            # Raises an error if experiences overlap or are out of order.
-            Experiences::SpansValidator.validate(experiences, config, history_column: true)
+            Experiences::SpansValidator.validate(experiences, history_column: true)
 
             experiences
           end
@@ -48,7 +59,7 @@ module Reading
           # A shortcut to the span template.
           # @return [Hash]
           def span_template
-            @span_template ||= config.deep_fetch(:item, :template, :experiences, 0, :spans).first
+            @span_template ||= Config.hash.deep_fetch(:item, :template, :experiences, 0, :spans).first
           end
 
           # The :spans sub-attribute for the given History column entries.
@@ -62,16 +73,21 @@ module Reading
               month: nil,
               day: nil,
               after_single_date: false,
-              open_range: false,
+              open_range_id: nil,
               planned: false,
               amount: nil,
+              repetitions: nil,
+              frequency: nil,
               last_start_year: nil,
               last_start_month: nil,
             }
 
+            # Dates after "not" entries.
+            except_dates = []
+
             entries.each do |entry|
               if entry[:except_dates]
-                reject_exception_dates!(entry, daily_spans, active)
+                except_dates += reject_exception_dates!(entry, daily_spans, active)
                 next
               end
 
@@ -80,10 +96,14 @@ module Reading
 
             spans = merge_daily_spans(daily_spans)
 
-            fix_open_ranges!(spans)
+            spans = fix_open_ranges(spans, except_dates)
 
             relativize_amounts_from_progress!(spans)
 
+            remove_last_end_date_of_today_if_open_range!(spans)
+
+            remove_temporary_keys!(spans)
+
             spans
           end
 
@@ -93,6 +113,7 @@ module Reading
           #   date-and-name combination.
           # @param active [Hash] variables that persist across entries, such as
           #   amount and implied date.
+          # @return [Array<Date>] the rejected dates.
           def reject_exception_dates!(entry, daily_spans, active)
             except_active = {
               year: active[:last_start_year],
@@ -123,6 +144,8 @@ module Reading
             daily_spans.reject! do |(date, _name), _span|
               except_dates.include?(date)
             end
+
+            except_dates
           end
 
           # Expands the given entry into one span per day, then adds them to daily_spans.
@@ -146,7 +169,7 @@ module Reading
             active[:month] = start_month if start_month
             active[:last_start_month] = active[:month]
             if start_day
-              active[:open_range] = false
+              active[:open_range_id] = nil
               active[:day] = start_day
             end
 
@@ -154,8 +177,14 @@ module Reading
               raise InvalidHistoryError, "Missing or incomplete first date"
             end
 
-            duplicate_open_range = !start_day && active[:open_range]
-            date_range = date_range(entry, active, duplicate_open_range:)
+            if entry[:planned] || (active[:planned] && !start_day)
+              active[:planned] = true
+            elsif active[:planned] && start_day
+              active[:planned] = false
+            end
+
+            duplicate_open_range_id = active[:open_range_id] if !start_day
+            date_range = date_range(entry, active, duplicate_open_range: !!duplicate_open_range_id)
 
             # A startless date range (i.e. with an implied start date) appearing
             # immediately after a single date has its start date bumped forward
@@ -165,20 +194,26 @@ module Reading
             end
             active[:after_single_date] = !date_range
 
-            amount =
-              Attributes::Shared.length(entry, key_name: :amount, ignore_repetitions: true) ||
-              Attributes::Shared.length(parsed_row[:length], episodic: true)
-            active[:amount] = amount if amount
+            variant_index = (entry[:variant_index] || 1).to_i - 1
+            format = parsed_row[:sources]&.dig(variant_index)&.dig(:format) ||
+              parsed_row[:head][head_index][:format]
 
             progress = Attributes::Shared.progress(entry)
 
+            amount_from_entry =
+              Attributes::Shared.length(entry, format:, key_name: :amount, ignore_repetitions: true)
+            amount_from_length =
+              Attributes::Shared.length(parsed_row[:length], format:, episodic: progress.nil? || parsed_row.dig(:length, :repetitions).nil?)
+            amount = amount_from_entry || amount_from_length
+            active[:amount] = amount if amount
+
             # If the entry has no amount and the item has no episodic length,
             # then use progress as amount instead. The typical scenario for this
             # is when tracking fixed-length items such as books. See
             # https://github.com/fpsvogel/reading/blob/main/doc/csv-format.md#history-pages-and-stopping-points-books
             if !amount && progress
               if progress.is_a? Float
-                total_length = Attributes::Shared.length(parsed_row[:length])
+                total_length = Attributes::Shared.length(parsed_row[:length], format:)
                 amount = total_length * progress
               else
                 amount = progress
@@ -186,44 +221,56 @@ module Reading
               amount_from_progress = true
             end
 
-            repetitions = entry[:repetitions]&.to_i || 1
+            repetitions = entry[:repetitions]&.to_i
             frequency = entry[:frequency]
 
+            # If the entry has no amount or progress, default to the previous
+            # repetitions and frequency.
+            unless amount_from_entry || progress || repetitions
+              repetitions = active[:repetitions]
+              frequency = active[:frequency]
+            end
+
+            active[:repetitions] = repetitions if repetitions
+            active[:frequency] = frequency if frequency
+
             amounts_by_date = distribute_amount_across_date_range(
               date_range || Date.new(active[:year], active[:month], active[:day]),
               amount || active[:amount],
-              repetitions,
+              repetitions || 1,
               frequency,
             )
 
-            in_open_range = active[:open_range] || duplicate_open_range
+            open_range_id = active[:open_range_id] || duplicate_open_range_id
 
             daily_spans_from_entry = amounts_by_date.map { |date, daily_amount|
               span_without_dates = {
                 dates: nil,
                 amount: daily_amount || span_template[:amount],
-                progress: (progress unless amount_from_progress) || span_template[:progress],
+                progress: (progress unless amount_from_progress) ||
+                  (0.0 if entry[:planned] || active[:planned]) ||
+                  span_template[:progress],
                 name: entry[:name] || span_template[:name],
                 favorite?: !!entry[:favorite] || span_template[:favorite?],
                 # Temporary keys (not in the final item data) for marking
                 # spans to ...
                 # ... be distributed evenly across an open date range.
-                in_open_range: in_open_range,
+                open_range_id:,
                 # ... have their amounts adjusted to be relative to previous progress.
-                amount_from_progress: amount_from_progress,
+                amount_from_progress?: amount_from_progress,
+                amount_from_frequency?: !!frequency,
+                implied_date_range?: !date_range && !!frequency,
               }
 
-              if entry[:planned] || (active[:planned] && !start_day)
+              if entry[:planned] || active[:planned]
                 date = nil
-                active[:planned] = true
               end
 
               key = [date, span_without_dates[:name]]
 
-              # When any entry in an open range lacks a name, add a random
-              # number to the key so that it does not overwrite a different
-              # entry in the open range that also lacks a name.
-              if in_open_range && !entry[:name]
+              # For entries in an open range, add a random number to the key to
+              # avoid overwriting entries with the same name, or lacking a name.
+              if open_range_id
                 key << rand
               end
 
@@ -276,7 +323,7 @@ module Reading
             return nil unless entry[:range] || duplicate_open_range
 
             if entry[:end_day]
-              active[:open_range] = false
+              active[:open_range_id] = nil
 
               end_year = entry[:end_year]&.to_i
               end_month = entry[:end_month]&.to_i
@@ -290,11 +337,17 @@ module Reading
               date_range = Date.new(active[:year], active[:month], active[:day])..
                 Date.new(end_year || active[:year], end_month || active[:month], end_day)
 
-              active[:day] = end_day + 1
-              active[:month] = end_month if end_month
-              active[:year] = end_year if end_year
+              date_after_end = date_range.end.next_day
+
+              active[:day] = date_after_end.day
+              active[:month] = date_after_end.month if end_month
+              active[:year] = date_after_end.year if end_year
             else # either starting or continuing (duplicating) an open range
-              active[:open_range] ||= true
+              unless active[:open_range_id]
+                active[:open_range_id] = next_open_range_id
+                @next_open_range_id += 1
+              end
+
               date_range = Date.new(active[:year], active[:month], active[:day])..Date.today
             end
 
@@ -307,7 +360,7 @@ module Reading
           # @param amount [Float, Integer, Item::TimeLength] amount in
           #   pages or time.
           # @param repetitions [Integer] e.g. "x4" in a History entry.
-          # @param frequency [Integer] e.g. "/week" in a History entry.
+          # @param frequency [String] e.g. "/week" in a History entry.
           # @return [Hash{Date => Float, Integer, Item::TimeLength}]
           def distribute_amount_across_date_range(date_or_range, amount, repetitions, frequency)
             unless amount
@@ -357,33 +410,64 @@ module Reading
           # Set each open date range's last end date (wherever it's today, i.e.
           # it wasn't defined) to the day before the next entry's start date.
           # At the same time, distribute each open range's spans evenly.
-          # Lastly, remove the :in_open_range key from spans.
+          # Lastly, remove the :open_range_id key from spans.
           # @param spans [Array<Hash>] spans after being merged from daily_spans.
+          # @param except_dates [Date] dates after "not" entries which were
+          #   rejected from spans.
           # @return [Array<Hash>]
-          def fix_open_ranges!(spans)
+          def fix_open_ranges(spans, except_dates)
             chunked_by_open_range = spans.chunk_while { |a, b|
               a[:dates] && b[:dates] && # in case of planned entry
-              a[:dates].begin == b[:dates].begin &&
-                a[:in_open_range] == b[:in_open_range]
+                a[:open_range_id] == b[:open_range_id]
             }
 
-            next_chunk_start_date = nil
+            next_start_date = nil
             chunked_by_open_range
-              .reverse_each { |chunk|
-                unless chunk.first[:in_open_range] && chunk.any? { _1[:dates].end == Date.today }
+              .to_a.reverse.map { |chunk|
+                unless chunk.first[:open_range_id]
                   # safe nav. in case of planned entry
-                  next_chunk_start_date = chunk.first[:dates]&.begin
-                  next
+                  next_start_date = chunk.first[:dates]&.begin
+                  next chunk
+                end
+
+                # Filter out spans that begin after the next chunk's start date.
+                if next_start_date
+                  chunk.reject! do |span|
+                    span[:dates].begin >= next_start_date
+                  end
+                end
+
+                # For the remaining spans (which begin before the next chunk's
+                # start date), bound each end date to that date.
+                chunk.reverse_each do |span|
+                  if !next_start_date
+                    next_start_date = span[:dates].begin
+                    next
+                  end
+
+                  if span[:dates].end >= next_start_date
+                    new_dates = span[:dates].begin..next_start_date.prev_day
+
+                    if span[:amount_from_frequency?]
+                      new_to_old_dates_ratio = new_dates.count / span[:dates].count.to_f
+                      span[:amount] = (span[:amount] * new_to_old_dates_ratio).to_i_if_whole
+                    end
+
+                    span[:dates] = new_dates
+                  end
+
+                  next_start_date = span[:dates].begin if span[:dates].begin < next_start_date
                 end
 
-                # Set last end date.
-                if chunk.last[:dates].end == Date.today && next_chunk_start_date
-                  chunk.last[:dates] = chunk.last[:dates].begin..next_chunk_start_date.prev_day
+                if !next_start_date || chunk.first[:dates].begin < next_start_date
+                  next_start_date = chunk.first[:dates].begin
                 end
-                next_chunk_start_date = chunk.first[:dates].begin
+
+                next chunk if chunk.map { |span| span[:open_range_id] }.uniq.count == 1 &&
+                  chunk.map { |span| span[:dates].begin }.uniq.count > 1
 
                 # Distribute spans across the open date range.
-                total_amount = chunk.sum { |c| c[:amount] }
+                total_amount = chunk.sum { |span| span[:amount] }
                 dates = chunk.last[:dates]
                 amount_per_day = total_amount / dates.count.to_f
 
@@ -391,6 +475,7 @@ module Reading
                 last_end_date = chunk.last[:dates].end
 
                 span = nil
+                chunk_dup = chunk.dup
                 amount_acc = 0
                 span_needing_end = nil
                 dates.each do |date|
@@ -400,8 +485,8 @@ module Reading
                   end
 
                   while amount_acc < amount_per_day
-                    break if chunk.empty?
-                    span = chunk.shift
+                    break if chunk_dup.empty?
+                    span = chunk_dup.shift
                     amount_acc += span[:amount]
 
                     if amount_acc < amount_per_day
@@ -418,11 +503,11 @@ module Reading
                 end
 
                 span[:dates] = span[:dates].begin..last_end_date
-              }
 
-            spans.each do |span|
-              span.delete(:in_open_range)
-            end
+                chunk
+              }
+              .reverse
+              .flatten
           end
 
           # Changes amounts taken from progress, from absolute to relative,
@@ -434,15 +519,42 @@ module Reading
           def relativize_amounts_from_progress!(spans)
             amount_acc = 0
             spans.each do |span|
-              if span[:amount_from_progress]
+              if span[:amount_from_progress?]
                 span[:amount] -= amount_acc
               end
 
               amount_acc += span[:amount]
             end
+          end
+
+          # Removes the end date from the last span if it's today, and if it was
+          # written as an open range.
+          # @param spans [Array<Hash>] spans after being merged from daily_spans.
+          # @return [Array<Hash>]
+          def remove_last_end_date_of_today_if_open_range!(spans)
+            if spans.last[:dates] &&
+              spans.last[:dates].end == Date.today &&
+              (spans.last[:open_range_id] || spans.last[:implied_date_range?])
+
+              spans.last[:dates] = spans.last[:dates].begin..
+            end
+          end
+
+          # Removes all keys that shouldn't be in the final item data.
+          # @param spans [Array<Hash>] spans after being merged from daily_spans.
+          # @return [Array<Hash>]
+          def remove_temporary_keys!(spans)
+            temporary_keys = %i[
+              open_range_id
+              amount_from_progress?
+              amount_from_frequency?
+              implied_date_range?
+            ]
 
             spans.each do |span|
-              span.delete(:amount_from_progress)
+              temporary_keys.each do |key|
+                span.delete(key)
+              end
             end
           end
         end

data/lib/reading/parsing/attributes/experiences/spans_validator.rb

@@ -11,24 +11,23 @@ module Reading
             # Checks the dates in the given experiences hash, and raises an error
             # at the first invalid date found.
             # @param experiences [Array<Hash>] experience hashes.
-            # @param config [Hash] an entire config.
             # @param history_column [Boolean] whether this validation is for
             #   experiences from the History column.
             # @raise [InvalidDateError] if any date is invalid.
-            def validate(experiences, config, history_column: false)
-              if both_date_columns?(config)
+            def validate(experiences, history_column: false)
+              if both_date_columns? && !history_column
                 validate_number_of_start_dates_and_end_dates(experiences)
               end
 
-              if start_dates_column?(config) || history_column
+              if start_dates_column? || history_column
                 validate_start_dates_are_in_order(experiences)
               end
 
-              if end_dates_column?(config) || history_column
+              if end_dates_column? || history_column
                 validate_end_dates_are_in_order(experiences)
               end
 
-              if both_date_columns?(config) || history_column
+              if both_date_columns? || history_column
                 validate_experiences_of_same_variant_do_not_overlap(experiences)
               end
 
@@ -39,20 +38,20 @@ module Reading
 
             # Whether the Start Dates column is enabled.
             # @return [Boolean]
-            def start_dates_column?(config)
-              config.fetch(:enabled_columns).include?(:start_dates)
+            def start_dates_column?
+              Config.hash.fetch(:enabled_columns).include?(:start_dates)
             end
 
             # Whether the End Dates column is enabled.
             # @return [Boolean]
-            def end_dates_column?(config)
-              config.fetch(:enabled_columns).include?(:end_dates)
+            def end_dates_column?
+              Config.hash.fetch(:enabled_columns).include?(:end_dates)
             end
 
             # Whether both the Start Dates and End Dates columns are enabled.
             # @return [Boolean]
-            def both_date_columns?(config)
-              start_dates_column?(config) && end_dates_column?(config)
+            def both_date_columns?
+              start_dates_column? && end_dates_column?
             end
 
             # Raises an error if there are more end dates than start dates, or
@@ -60,7 +59,7 @@ module Reading
             # @raise [InvalidDateError]
             def validate_number_of_start_dates_and_end_dates(experiences)
               _both_dates, not_both_dates = experiences
-                .filter { |exp| exp[:spans].first&.dig(:dates) }
+                .select { |exp| exp[:spans].first&.dig(:dates) }
                 .map { |exp| [exp[:spans].first[:dates].begin, exp[:spans].last[:dates].end] }
                 .partition { |start_date, end_date| start_date && end_date }
 
@@ -76,7 +75,7 @@ module Reading
             # @raise [InvalidDateError]
             def validate_start_dates_are_in_order(experiences)
               experiences
-                .filter { |exp| exp[:spans].first&.dig(:dates) }
+                .select { |exp| exp[:spans].first&.dig(:dates) }
                 .map { |exp| exp[:spans].first[:dates].begin }
                 .each_cons(2) do |a, b|
                   if (a.nil? && b.nil?) || (a && b && a > b )
@@ -89,8 +88,8 @@ module Reading
             # @raise [InvalidDateError]
             def validate_end_dates_are_in_order(experiences)
               experiences
-                .filter { |exp| exp[:spans].first&.dig(:dates) }
-                .map { |exp| exp[:spans].last[:dates].end }
+                .select { |exp| exp[:spans].first&.dig(:dates) }
+                .map { |exp| exp[:spans].last[:dates]&.end }
                 .each_cons(2) do |a, b|
                   if (a.nil? && b.nil?) || (a && b && a > b )
                     raise InvalidDateError, "End dates are not in order"
@@ -104,7 +103,7 @@ module Reading
               experiences
                 .group_by { |exp| exp[:variant_index] }
                 .each do |_variant_index, exps|
-                  exps.filter { |exp| exp[:spans].any? }.each_cons(2) do |a, b|
+                  exps.select { |exp| exp[:spans].any? }.each_cons(2) do |a, b|
                     a_metaspan = a[:spans].first[:dates].begin..a[:spans].last[:dates].end
                     b_metaspan = b[:spans].first[:dates].begin..b[:spans].last[:dates].end
                     if a_metaspan.cover?(b_metaspan.begin || a_metaspan.begin || a_metaspan.end) ||
@@ -116,11 +115,11 @@ module Reading
             end
 
             # Raises an error if the spans within an experience are out of order
-            # or if the spans overlap.
+            # or if the spans overlap. Spans with nil dates are not considered.
             # @raise [InvalidDateError]
             def validate_spans_are_in_order_and_not_overlapping(experiences)
               experiences
-                .filter { |exp| exp[:spans].first&.dig(:dates) }
+                .select { |exp| exp[:spans].first&.dig(:dates) }
                 .each do |exp|
                   exp[:spans]
                     .map { |span| span[:dates] }
@@ -132,7 +131,7 @@ module Reading
                       end
                     end
                     .each_cons(2) do |a, b|
-                      if a.begin > b.begin || a.end > b.end
+                      if a.begin > b.begin || (a.end || Date.today) > (b.end || Date.today)
                         raise InvalidDateError, "Dates are not in order"
                       end
                       if a.cover?(b.begin + 1)

data/lib/reading/parsing/attributes/experiences.rb

@@ -1,6 +1,6 @@
-require "date"
-require_relative "experiences/history_transformer"
-require_relative "experiences/dates_and_head_transformer"
+require 'date'
+require_relative 'experiences/history_transformer'
+require_relative 'experiences/dates_and_head_transformer'
 
 module Reading
   module Parsing
@@ -16,10 +16,10 @@ module Reading
         #   Config#default_config[:item][:template][:experiences]
         def transform_from_parsed(parsed_row, head_index)
           if !parsed_row[:history].blank?
-            return HistoryTransformer.new(parsed_row, config).transform
+            return HistoryTransformer.new(parsed_row, head_index).transform
           end
 
-          DatesAndHeadTransformer.new(parsed_row, head_index, config).transform
+          DatesAndHeadTransformer.new(parsed_row, head_index).transform
         end
       end
     end

data/lib/reading/parsing/attributes/shared.rb

@@ -1,23 +1,31 @@
 module Reading
   module Parsing
     module Attributes
-      # Shared
+      # Sub-attributes that are shared across multiple attributes.
       module Shared
+        using Util::HashArrayDeepFetch
+        using Util::NumericToIIfWhole
+
         # Extracts the :progress sub-attribute (percent, pages, or time) from
         # the given hash.
         # @param hash [Hash] any parsed hash that contains progress.
+        # @param no_end_date [Boolean] for start and end dates (as opposed to
+        #   the History column), whether an end date is present.
         # @return [Float, Integer, Item::TimeLength]
-        def self.progress(hash)
+        def self.progress(hash, no_end_date: nil)
           hash[:progress_percent]&.to_f&./(100) ||
             hash[:progress_pages]&.to_i ||
-            hash[:progress_time]&.then { Item::TimeLength.parse _1 } ||
+            hash[:progress_time]&.then { Item::TimeLength.parse(_1) } ||
             (0 if hash[:progress_dnf]) ||
             (1.0 if hash[:progress_done]) ||
+            (0.0 if no_end_date) ||
             nil
         end
 
         # Extracts the :length sub-attribute (pages or time) from the given hash.
         # @param hash [Hash] any parsed hash that contains length.
+        # @param format [Symbol] the item format, which affects length in cases
+        #   where Config.hash[:speed][:format] is customized.
         # @param key_name [Symbol] the first part of the keys to be checked.
         # @param episodic [Boolean] whether to look for episodic (not total) length.
         #   If false, returns nil if hash contains :each. If true, returns a
@@ -30,15 +38,15 @@ module Reading
         #   This is useful for the History column, where that 1 hour can be used
         #   as the default amount.
         # @return [Float, Integer, Item::TimeLength]
-        def self.length(hash, key_name: :length, episodic: false, ignore_repetitions: false)
+        def self.length(hash, format:, key_name: :length, episodic: false, ignore_repetitions: false)
           return nil unless hash
 
           length = hash[:"#{key_name}_pages"]&.to_i ||
-            hash[:"#{key_name}_time"]&.then { Item::TimeLength.parse _1 }
+            hash[:"#{key_name}_time"]&.then { Item::TimeLength.parse(_1) }
 
           return nil unless length
 
-          if hash[:each]
+          if hash[:each] && !hash[:repetitions]
             # Length is calculated based on History column in this case.
             if episodic
               return length
@@ -54,7 +62,9 @@ module Reading
             return nil if episodic && !hash[:each]
           end
 
-          length
+          speed = Config.hash.deep_fetch(:speed, :format)[format] || 1.0
+
+          (length / speed).to_i_if_whole
         end
       end
     end