reading 0.6.0 → 0.7.0

data/lib/reading/parsing/parser.rb

@@ -0,0 +1,292 @@
+require_relative "rows/regular"
+require_relative "rows/compact_planned"
+require_relative "rows/comment"
+
+module Reading
+  module Parsing
+    #
+    # Parses a string containing a row of a CSV reading log, into a hash
+    # mirroring the structure of the row. This hash is an intermediate form and
+    # not the final item data. It's the raw material for Parsing::Transformer to
+    # generate the final item data.
+    #
+    # Below is an example intermediate hash parsed from this row, which has a Rating
+    # column, then a Head column containing an author, title, series, and extra info:
+    #
+    # 3|📕Thomas More - Utopia -- trans. Robert Adams -- ed. George Logan -- in Cambridge History of Political Thought
+    #
+    # {
+    #   rating: { number: "1" },
+    #   head: [{
+    #     author: "Thomas More",
+    #     title: "Utopia",
+    #     series_names: ["Cambridge History of Political Thought"],
+    #     series_volumes: [nil],
+    #     extra_info: ["trans. Robert Adams", "ed. George Logan"],
+    #     format: :print,
+    #   }]
+    # }
+    #
+    # The hash's top-level keys are column names. The nested keys come from
+    # regex capture group names in each column (for this example, see ::regexes
+    # in rating.rb and head.rb in parsing/rows/regular_columns).
+    #
+    # All the rest is just details of how the parts of a column are joined:
+    #
+    # - The :head value is an array because Head.split_by_format? is
+    #   true (because a Head column can potentially contain multiple items).
+    #   That's also where { format: :print } comes from.
+    #
+    # - The :series_names and :series_volumes values are arrays because these
+    #   keys are in Head.flatten_into_arrays, which causes the column's segments
+    #   (separated by " -- ") to be merged into one hash.
+    #
+    class Parser
+      using Util::HashArrayDeepFetch
+      using Util::StringRemove
+
+      attr_reader :config
+
+      # @param config [Hash] an entire config.
+      def initialize(config)
+        @config = config
+      end
+
+      # Parses a row string into a hash that mirrors the structure of the row.
+      # @param string [String] a string containing a row of a CSV reading log.
+      # @return [Hash]
+      def parse_row_to_intermediate_hash(string)
+        columns = extract_columns(string)
+
+        if config.fetch(:skip_compact_planned) && columns.has_key?(Rows::CompactPlanned::Head)
+          return {}
+        end
+
+        columns.map { |column, column_string|
+          parse_column(column, column_string)
+        }.to_h
+      end
+
+      private
+
+      # Splits the row string by column and pairs them in a hash with column
+      # classes, which contain the information necessary to parse each column.
+      # @param string [String] a string containing a row of a CSV reading log.
+      # @return [Hash{Class => String}] a hash whose keys are classes inheriting
+      #   Parsing::Rows::Column.
+      def extract_columns(string)
+        clean_string = string.dup.force_encoding(Encoding::UTF_8)
+        column_strings = clean_string.split(config.fetch(:column_separator))
+
+        row_types = [Rows::Regular, Rows::CompactPlanned, Rows::Comment]
+        column_classes = row_types
+          .find { |row_type| row_type.match?(string, config) }
+          .column_classes
+          .filter { |column_class|
+            config.fetch(:enabled_columns).include?(column_class.to_sym)
+          }
+
+        if !column_classes.count.zero? && column_strings.count > column_classes.count
+          raise TooManyColumnsError, "Too many columns"
+        end
+
+        column_classes
+          .zip(column_strings)
+          .reject { |_class, string| string.nil? }
+          .to_h
+      end
+
+      # Parses a column into an array of two elements (a key for the column name
+      # and a value of its contents).
+      # @param column_class [Class] a class inheriting Parsing::Rows::Column.
+      # @param column_string [String] a string containing a column from a row.
+      # @return [Array(Symbol, Hash), Array(Symbol, Array)]
+      def parse_column(column_class, column_string)
+        # Multiple format emojis are possible in some columns:
+        # - Head column, for multiple items.
+        # - Sources column, for multiple variants of an item.
+        # - Compact planned head column, for multiple items.
+        # This is the default case below the two guard clauses. It's more complex
+        # because there's possibly a string before the first format, and there's
+        # an extra level of nesting in the returned array.
+
+        # Simplest case: if the column is never split by format, return the
+        # column name and the parsed segment(s), which is either a Hash (if the
+        # column can't have multiple segments or if its segments are flattened)
+        # or an Array (if there are multiple segments and they're not flattened).
+        if !column_class.split_by_format?
+          parsed_column = parse_segments(column_class, column_string)
+          return [column_class.to_sym, parsed_column]
+        end
+
+        # Also simple: if the column *can* be split by format but in this row
+        # it doesn't contain any format emojis, return the same as above but
+        # with an extra level of nesting (except when the parsed result is nil).
+        if column_class.split_by_format? &&
+            !column_string.match?(config.deep_fetch(:regex, :formats))
+
+          parsed_column = parse_segments(column_class, column_string)
+          # Wrap a non-empty value in an array so that e.g. a head without
+          # emojis is still an array. This way the extra level of nesting can
+          # be consistently expected for columns that *can* be split by format.
+          parsed_column_nonempty_nested = [parsed_column.presence].compact
+          return [column_class.to_sym, parsed_column_nonempty_nested]
+        end
+
+        # The rest is the complex case: if the column *can and is* split by format.
+
+        # Each format plus the string after it.
+        format_strings = column_string.split(config.deep_fetch(:regex, :formats_split))
+
+        # If there's a string before the first format, e.g. "DNF" in Head column.
+        unless format_strings.first.match?(config.deep_fetch(:regex, :formats))
+          before_formats = parse_segment(column_class, format_strings.shift, before_formats: true)
+        end
+
+        # Parse each format-plus-string into an array of segments.
+        heads = format_strings.map { |string|
+          format_emoji = string[config.deep_fetch(:regex, :formats)]
+          string.remove!(format_emoji)
+          format = config.fetch(:formats).key(format_emoji)
+
+          parse_segments(column_class, string)
+            .merge(format: format)
+        }
+
+        # Combine values of conflicting keys so that in a compact planned
+        # Head column, sources from before_formats are not ignored.
+        if before_formats
+          heads.each do |head|
+            head.merge!(before_formats) do |k, old_v, new_v|
+              (new_v + old_v).uniq
+            end
+          end
+        end
+
+        [column_class.to_sym, heads]
+      end
+
+      # Parses a string of segments, e.g. "Utopia -- trans. Robert Adams -- ed. George Logan"
+      # @param column_class [Class] a class inheriting Parsing::Rows::Column.
+      # @param string [String] a string containing segments, which is either an
+      #   entire column or (for columns that are split by format emoji) a string
+      #   following a format emoji.
+      # @return [Array<Hash>, Hash] either an array of parsed segments (hashes),
+      #   or a single hash if the column can't be split by segment or if the
+      #   segments are flattened into one hash.
+      def parse_segments(column_class, string)
+        return {} if string.blank?
+
+        # If the column can't be split by segment, parse as a single segment.
+        if !column_class.split_by_segment?
+          return parse_segment(column_class, string)
+        end
+
+        # Add an extra level of nesting if the column can have segment groups,
+        # as in "2021/1/28..2/1 x4 -- ..2/3 x5 ---- 11/1 -- 11/2"
+        if column_class.split_by_segment_group?
+          segments = string
+            .split(column_class.segment_group_separator)
+            .map { |segment_group|
+              segment_group
+                .split(column_class.segment_separator)
+                .map.with_index { |segment, i|
+                  parse_segment(column_class, segment, i)
+                }
+            }
+        else
+          segments = string
+            .split(column_class.segment_separator)
+            .map.with_index { |segment, i|
+              parse_segment(column_class, segment, i)
+            }
+        end
+
+        if column_class.flatten_into_arrays.any?
+          segments = segments.reduce { |merged, segment|
+            merged.merge!(segment) { |_k, old_v, new_v|
+              # old_v is already an array by this point, since its key should be
+              # in Column.flatten_into_arrays
+              old_v + new_v
+            }
+          }
+        end
+
+        segments
+      end
+
+      # Parses a segment using a regular expression from the column class.
+      # @param column_class [Class] a class inheriting Parsing::Rows::Column.
+      # @param segment [String] a segment, e.g. "Bram Stoker - Dracula".
+      # @param segment_index [Integer] the position of the segment when it's in
+      #   part of a series of segments; this can change which regular expressions
+      #   are applicable to it.
+      # @param before_formats [Boolean] whether to use the before-formats regexes.
+      # @return [Hash{Symbol => Object}] the parsed segment, whose values are Strings
+      #   unless changed via column_class.tweaks or column_class.flatten_into_arrays.
+      #   Example: { author: "Bram Stoker", title: "Dracula"}
+      def parse_segment(column_class, segment, segment_index = 0, before_formats: false)
+        if before_formats
+          regexes = column_class.regexes_before_formats
+        else
+          regexes = column_class.regexes(segment_index)
+        end
+
+        parsed_segment = nil
+        regexes.each do |regex|
+          parsed_segment = parse_segment_with_regex(segment, regex)
+          break if parsed_segment
+        end
+
+        if parsed_segment.nil?
+          raise ParsingError, "Could not parse \"#{segment}\" in " \
+            "the #{column_class.column_name} column"
+        end
+
+        tweak_and_arrayify_parsed_segment(parsed_segment, column_class)
+      end
+
+      # Parses a segment using the given regular expression.
+      # @param segment [String] a segment, e.g. "Bram Stoker - Dracula".
+      # @param regex [Regexp] the regular expression with which to parse the segment.
+      # @return [Hash{Symbol => String}] e.g. { author: "Bram Stoker", title: "Dracula"}
+      def parse_segment_with_regex(segment, regex)
+        segment
+          .tr(config.fetch(:ignored_characters), "")
+          .strip
+          .match(regex)
+          &.named_captures
+          &.compact
+          &.transform_keys(&:to_sym)
+          &.transform_values(&:strip)
+          &.transform_values(&:presence)
+      end
+
+      # Modify the values of the parsed segment according to column_class.tweaks,
+      # and wrap them in an array according to column_class.flatten_into_arrays.
+      # @param parsed_segment [Hash] e.g. { author: "Bram Stoker", title: "Dracula"}
+      # @return [Hash{Symbol => Object}]
+      def tweak_and_arrayify_parsed_segment(parsed_segment, column_class)
+        column_class.tweaks.each do |key, tweak|
+          if parsed_segment.has_key?(key)
+            parsed_segment[key] = tweak.call(parsed_segment[key])
+          end
+        end
+
+        # Ensure that values of keys in column_class.flatten_into_arrays are arrays.
+        column_class.flatten_into_arrays.each do |key|
+          if parsed_segment.has_key?(key)
+            val = parsed_segment[key]
+            # Not using Array(val) because that results in an empty array when
+            # val is nil, and the nil must be preserved for series name and
+            # volume arrays to line up with an equal number of elements (because
+            # the volume may be nil).
+            parsed_segment[key] = [val] if !val.is_a?(Array)
+          end
+        end
+
+        parsed_segment
+      end
+    end
+  end
+end

data/lib/reading/parsing/rows/column.rb

@@ -0,0 +1,131 @@
+module Reading
+  module Parsing
+    module Rows
+      # The base class for all the columns in parsing/rows/compact_planned_columns
+      # and parsing/rows/regular_columns.
+      class Column
+        # The class name changed into a string, e.g. StartDates => "Start Dates"
+        # @return [String]
+        def self.column_name
+          class_name = name.split("::").last
+          class_name.gsub(/(.)([A-Z])/,'\1 \2')
+        end
+
+        # The class name changed into a symbol, e.g. StartDates => :start_dates
+        # @return [Symbol]
+        def self.to_sym
+          class_name = name.split("::").last
+          class_name
+            .gsub(/(.)([A-Z])/,'\1_\2')
+            .downcase
+            .to_sym
+        end
+
+        # Whether the column can contain "chunks" each set off by a format emoji.
+        # For example, the Head column of a compact planned row typically
+        # contains a list of multiple items. (The two others are the Sources
+        # column, for multiple variants of an item; and the regular Head column,
+        # for multiple items.)
+        # @return [Boolean]
+        def self.split_by_format?
+          false
+        end
+
+        # Whether the column can contain multiple segments, e.g. "Cosmos -- 2013 paperback"
+        # @return [Boolean]
+        def self.split_by_segment?
+          !!segment_separator
+        end
+
+        # The regular expression used to split segments (e.g. /\s*--\s*/),
+        # or nil if the column should not be split by segment.
+        # @return [Regexp, nil]
+        def self.segment_separator
+          nil
+        end
+
+        # Whether the column can contain multiple segment groups, e.g.
+        # "2021/1/28..2/1 x4 -- ..2/3 x5 ---- 11/1 -- 11/2"
+        # @return [Boolean]
+        def self.split_by_segment_group?
+          !!segment_group_separator
+        end
+
+        # The regular expression used to split segment groups (e.g. /\s*----\s*/),
+        # or nil if the column should not be split by segment group.
+        # @return [Regexp, nil]
+        def self.segment_group_separator
+          nil
+        end
+
+        # Adjustments that are made to captured values at the end of parsing
+        # the column. For example, if ::regexes includes a capture group named
+        # "sources" and it needs to be split by commas:
+        # { sources: -> { _1.split(/\s*,\s*/) } }
+        # @return [Hash{Symbol => Proc}]
+        def self.tweaks
+          {}
+        end
+
+        # Keys in the parsed output hash that should be converted to an array, even
+        # if only one value was in the input, as in { ... extra_info: ["ed. Jane Doe"] }
+        # @return [Array<Symbol>]
+        def self.flatten_into_arrays
+          []
+        end
+
+        # The regular expressions used to parse the column (except the part of
+        # the column before the first format emoji, which is in
+        # ::regexes_before_formats below). An array because sometimes it's
+        # simpler to try several smaller regular expressions in series, and
+        # because a regular expression might be applicable only for segments in
+        # a certain position. See parsing/rows/regular_columns/head.rb for an example.
+        # @param segment_index [Integer] the position of the current segment.
+        # @return [Array<Regexp>]
+        def self.regexes(segment_index)
+          []
+        end
+
+        # The regular expressions used to parse the part of the column before
+        # the first format emoji.
+        # @return [Array<Regexp>]
+        def self.regexes_before_formats
+          []
+        end
+
+        # Regular expressions that are shared across more than one column,
+        # placed here just to be DRY.
+        SHARED_REGEXES = {
+          progress: %r{
+            (DNF\s+)?(?<progress_percent>\d\d?)%
+            |
+            (DNF\s+)?p?(?<progress_pages>\d+)p?
+            |
+            (DNF\s+)?(?<progress_time>\d+:\d\d)
+            |
+            # just DNF
+            (?<progress_dnf>DNF)
+          }x,
+          series_and_extra_info: [
+            # just series
+            %r{\A
+              in\s(?<series_names>.+)
+              # empty volume so that names and volumes have equal sizes when turned into arrays
+              (?<series_volumes>)
+            \z}x,
+            # series and volume
+            %r{\A
+              (?<series_names>.+?)
+              ,?\s*
+              \#(?<series_volumes>\d+)
+            \z}x,
+            # extra info
+            %r{\A
+              (?<extra_info>.+)
+            \z}x,
+          ],
+        }.freeze
+      end
+    end
+  end
+end

data/lib/reading/parsing/rows/comment.rb

@@ -0,0 +1,26 @@
+module Reading
+  module Parsing
+    module Rows
+      # A row that is a comment.
+      module Comment
+        using Util::HashArrayDeepFetch
+
+        # No columns; comments are parsed as if the row were blank.
+        # @return [Array]
+        def self.column_classes
+          []
+        end
+
+        # Starts with a comment character and does not include any format emojis.
+        # (Commented rows that DO include format emojis are matched as compact
+        # planned rows.)
+        # @param row_string [String]
+        # @param config [Hash]
+        # @return [Boolean]
+        def self.match?(row_string, config)
+          row_string.lstrip.start_with?(config.fetch(:comment_character))
+        end
+      end
+    end
+  end
+end

data/lib/reading/parsing/rows/compact_planned.rb

@@ -0,0 +1,30 @@
+require_relative "column"
+require_relative "compact_planned_columns/head"
+require_relative "regular_columns/sources"
+
+module Reading
+  module Parsing
+    module Rows
+      # A row that contains compact planned items.
+      module CompactPlanned
+        using Util::HashArrayDeepFetch
+
+        # The columns that are possible in this type of row.
+        # @return [Array<Class>]
+        def self.column_classes
+          [CompactPlanned::Head, Regular::Sources]
+        end
+
+        # Starts with a comment character and includes one or more format emojis.
+        # @param row_string [String]
+        # @param config [Hash]
+        # @return [Boolean]
+        def self.match?(row_string, config)
+          row_string.lstrip.start_with?(config.fetch(:comment_character)) &&
+            row_string.match?(config.deep_fetch(:regex, :formats)) &&
+            row_string.count(config.fetch(:column_separator)) <= column_classes.count - 1
+        end
+      end
+    end
+  end
+end

data/lib/reading/parsing/rows/compact_planned_columns/head.rb

@@ -0,0 +1,60 @@
+module Reading
+  module Parsing
+    module Rows
+      module CompactPlanned
+        # See https://github.com/fpsvogel/reading/blob/main/doc/csv-format.md#compact-planned-items
+        # and the sections following.
+        class Head < Column
+          def self.split_by_format?
+            true
+          end
+
+          def self.regexes_before_formats
+            [
+              %r{\A
+                \\ # comment character
+                \s*
+                (
+                  (?<genres>[^a-z]+)?
+                  \s*
+                  (?<sources>@.+)?
+                  \s*:
+                )?
+              \z}x,
+            ]
+          end
+
+          def self.segment_separator
+            /\s*--\s*/
+          end
+
+          def self.flatten_into_arrays
+            %i[extra_info series_names series_volumes]
+          end
+
+          def self.tweaks
+            {
+              genres: -> { _1.downcase.split(/\s*,\s*/) },
+              sources: -> { _1.split(/\s*@/).map(&:presence).compact }
+            }
+          end
+
+          def self.regexes(segment_index)
+            [
+              # author, title, sources
+              (%r{\A
+                (
+                  (?<author>[^@]+?)
+                  \s+-\s+
+                )?
+                (?<title>[^@]+)
+                (?<sources>@.+)?
+              \z}x if  segment_index.zero?),
+              *Column::SHARED_REGEXES[:series_and_extra_info],
+            ].compact
+          end
+        end
+      end
+    end
+  end
+end

data/lib/reading/parsing/rows/regular.rb

@@ -0,0 +1,33 @@
+require_relative "column"
+require_relative "regular_columns/rating"
+require_relative "regular_columns/head"
+require_relative "regular_columns/sources"
+require_relative "regular_columns/start_dates"
+require_relative "regular_columns/end_dates"
+require_relative "regular_columns/genres"
+require_relative "regular_columns/length"
+require_relative "regular_columns/notes"
+require_relative "regular_columns/history"
+
+module Reading
+  module Parsing
+    module Rows
+      # A normal row of (usually) one item.
+      module Regular
+        # The columns that are possible in this type of row.
+        # @return [Array<Class>]
+        def self.column_classes
+          [Rating, Head, Sources, StartDates, EndDates, Genres, Length, Notes, History]
+        end
+
+        # Does not start with a comment character.
+        # @param row_string [String]
+        # @param config [Hash]
+        # @return [Boolean]
+        def self.match?(row_string, config)
+          !row_string.lstrip.start_with?(config.fetch(:comment_character))
+        end
+      end
+    end
+  end
+end

data/lib/reading/parsing/rows/regular_columns/end_dates.rb

@@ -0,0 +1,20 @@
+module Reading
+  module Parsing
+    module Rows
+      module Regular
+        # See https://github.com/fpsvogel/reading/blob/main/doc/csv-format.md#start-dates-and-end-dates-columns
+        class EndDates < Column
+          def self.segment_separator
+            /,\s*/
+          end
+
+          def self.regexes(segment_index)
+            [%r{\A
+              (?<date>\d{4}/\d\d?/\d\d?)
+            \z}x]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/reading/parsing/rows/regular_columns/genres.rb

@@ -0,0 +1,20 @@
+module Reading
+  module Parsing
+    module Rows
+      module Regular
+        # See https://github.com/fpsvogel/reading/blob/main/doc/csv-format.md#genres-column
+        class Genres < Column
+          def self.segment_separator
+            /,\s*/
+          end
+
+          def self.regexes(segment_index)
+            [%r{\A
+              (?<genre>.+)
+            \z}x]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/reading/parsing/rows/regular_columns/head.rb

@@ -0,0 +1,45 @@
+module Reading
+  module Parsing
+    module Rows
+      module Regular
+        # See https://github.com/fpsvogel/reading/blob/main/doc/csv-format.md#head-column-title
+        # and https://github.com/fpsvogel/reading/blob/main/doc/csv-format.md#head-column-dnf
+        # and the sections following.
+        class Head < Column
+          def self.split_by_format?
+            true
+          end
+
+          def self.regexes_before_formats
+            [
+              /\A#{Column::SHARED_REGEXES[:progress]}\z/,
+              /.+/,
+            ]
+          end
+
+          def self.segment_separator
+            /\s*--\s*/
+          end
+
+          def self.flatten_into_arrays
+            %i[extra_info series_names series_volumes]
+          end
+
+          def self.regexes(segment_index)
+            [
+              # author and title
+              (%r{\A
+                (
+                  (?<author>.+?)
+                  \s+-\s+
+                )?
+                (?<title>.+)
+              \z}x if segment_index.zero?),
+              *Column::SHARED_REGEXES[:series_and_extra_info],
+            ].compact
+          end
+        end
+      end
+    end
+  end
+end