jekyll 4.2.1 → 4.3.0

data/lib/jekyll/reader.rb

@@ -1,192 +1,209 @@
-# frozen_string_literal: true
-
-module Jekyll
-  class Reader
-    attr_reader :site
-
-    def initialize(site)
-      @site = site
-    end
-
-    # Read Site data from disk and load it into internal data structures.
-    #
-    # Returns nothing.
-    def read
-      @site.layouts = LayoutReader.new(site).read
-      read_directories
-      read_included_excludes
-      sort_files!
-      @site.data = DataReader.new(site).read(site.config["data_dir"])
-      CollectionReader.new(site).read
-      ThemeAssetsReader.new(site).read
-    end
-
-    # Sorts posts, pages, and static files.
-    def sort_files!
-      site.collections.each_value { |c| c.docs.sort! }
-      site.pages.sort_by!(&:name)
-      site.static_files.sort_by!(&:relative_path)
-    end
-
-    # Recursively traverse directories to find pages and static files
-    # that will become part of the site according to the rules in
-    # filter_entries.
-    #
-    # dir - The String relative path of the directory to read. Default: ''.
-    #
-    # Returns nothing.
-    def read_directories(dir = "")
-      base = site.in_source_dir(dir)
-
-      return unless File.directory?(base)
-
-      dot_dirs = []
-      dot_pages = []
-      dot_static_files = []
-
-      dot = Dir.chdir(base) { filter_entries(Dir.entries("."), base) }
-      dot.each do |entry|
-        file_path = @site.in_source_dir(base, entry)
-        if File.directory?(file_path)
-          dot_dirs << entry
-        elsif Utils.has_yaml_header?(file_path)
-          dot_pages << entry
-        else
-          dot_static_files << entry
-        end
-      end
-
-      retrieve_posts(dir)
-      retrieve_dirs(base, dir, dot_dirs)
-      retrieve_pages(dir, dot_pages)
-      retrieve_static_files(dir, dot_static_files)
-    end
-
-    # Retrieves all the posts(posts/drafts) from the given directory
-    # and add them to the site and sort them.
-    #
-    # dir - The String representing the directory to retrieve the posts from.
-    #
-    # Returns nothing.
-    def retrieve_posts(dir)
-      return if outside_configured_directory?(dir)
-
-      site.posts.docs.concat(post_reader.read_posts(dir))
-      site.posts.docs.concat(post_reader.read_drafts(dir)) if site.show_drafts
-    end
-
-    # Recursively traverse directories with the read_directories function.
-    #
-    # base - The String representing the site's base directory.
-    # dir - The String representing the directory to traverse down.
-    # dot_dirs - The Array of subdirectories in the dir.
-    #
-    # Returns nothing.
-    def retrieve_dirs(_base, dir, dot_dirs)
-      dot_dirs.each do |file|
-        dir_path = site.in_source_dir(dir, file)
-        rel_path = PathManager.join(dir, file)
-        @site.reader.read_directories(rel_path) unless @site.dest.chomp("/") == dir_path
-      end
-    end
-
-    # Retrieve all the pages from the current directory,
-    # add them to the site and sort them.
-    #
-    # dir - The String representing the directory retrieve the pages from.
-    # dot_pages - The Array of pages in the dir.
-    #
-    # Returns nothing.
-    def retrieve_pages(dir, dot_pages)
-      site.pages.concat(PageReader.new(site, dir).read(dot_pages))
-    end
-
-    # Retrieve all the static files from the current directory,
-    # add them to the site and sort them.
-    #
-    # dir - The directory retrieve the static files from.
-    # dot_static_files - The static files in the dir.
-    #
-    # Returns nothing.
-    def retrieve_static_files(dir, dot_static_files)
-      site.static_files.concat(StaticFileReader.new(site, dir).read(dot_static_files))
-    end
-
-    # Filter out any files/directories that are hidden or backup files (start
-    # with "." or "#" or end with "~"), or contain site content (start with "_"),
-    # or are excluded in the site configuration, unless they are web server
-    # files such as '.htaccess'.
-    #
-    # entries - The Array of String file/directory entries to filter.
-    # base_directory - The string representing the optional base directory.
-    #
-    # Returns the Array of filtered entries.
-    def filter_entries(entries, base_directory = nil)
-      EntryFilter.new(site, base_directory).filter(entries)
-    end
-
-    # Read the entries from a particular directory for processing
-    #
-    # dir - The String representing the relative path of the directory to read.
-    # subfolder - The String representing the directory to read.
-    #
-    # Returns the list of entries to process
-    def get_entries(dir, subfolder)
-      base = site.in_source_dir(dir, subfolder)
-      return [] unless File.exist?(base)
-
-      entries = Dir.chdir(base) { filter_entries(Dir["**/*"], base) }
-      entries.delete_if { |e| File.directory?(site.in_source_dir(base, e)) }
-    end
-
-    private
-
-    # Internal
-    #
-    # Determine if the directory is supposed to contain posts and drafts.
-    # If the user has defined a custom collections_dir, then attempt to read
-    # posts and drafts only from within that directory.
-    #
-    # Returns true if a custom collections_dir has been set but current directory lies
-    # outside that directory.
-    def outside_configured_directory?(dir)
-      collections_dir = site.config["collections_dir"]
-      !collections_dir.empty? && !dir.start_with?("/#{collections_dir}")
-    end
-
-    # Create a single PostReader instance to retrieve drafts and posts from all valid
-    # directories in current site.
-    def post_reader
-      @post_reader ||= PostReader.new(site)
-    end
-
-    def read_included_excludes
-      entry_filter = EntryFilter.new(site)
-
-      site.include.each do |entry|
-        entry_path = site.in_source_dir(entry)
-        next if File.directory?(entry_path)
-        next if entry_filter.symlink?(entry_path)
-
-        read_included_file(entry_path) if File.file?(entry_path)
-      end
-    end
-
-    def read_included_file(entry_path)
-      if Utils.has_yaml_header?(entry_path)
-        conditionally_generate_entry(entry_path, site.pages, PageReader)
-      else
-        conditionally_generate_entry(entry_path, site.static_files, StaticFileReader)
-      end
-    end
-
-    def conditionally_generate_entry(entry_path, container, reader)
-      return if container.find { |item| site.in_source_dir(item.relative_path) == entry_path }
-
-      dir, files = File.split(entry_path)
-      dir.sub!(site.source, "")
-      files = Array(files)
-      container.concat(reader.new(site, dir).read(files))
-    end
-  end
-end
+# frozen_string_literal: true
+
+module Jekyll
+  class Reader
+    attr_reader :site
+
+    def initialize(site)
+      @site = site
+    end
+
+    # Read Site data from disk and load it into internal data structures.
+    #
+    # Returns nothing.
+    def read
+      @site.layouts = LayoutReader.new(site).read
+      read_directories
+      read_included_excludes
+      sort_files!
+      CollectionReader.new(site).read
+      ThemeAssetsReader.new(site).read
+      read_data
+    end
+
+    # Read and merge the data files.
+    # If a theme is specified and it contains data, it will be read.
+    # Site data will overwrite theme data with the same key using the
+    # semantics of Utils.deep_merge_hashes.
+    #
+    # Returns nothing.
+    def read_data
+      @site.data = DataReader.new(site).read(site.config["data_dir"])
+      return unless site.theme&.data_path
+
+      theme_data = DataReader.new(
+        site,
+        :in_source_dir => site.method(:in_theme_dir)
+      ).read(site.theme.data_path)
+      @site.data = Jekyll::Utils.deep_merge_hashes(theme_data, @site.data)
+    end
+
+    # Sorts posts, pages, and static files.
+    def sort_files!
+      site.collections.each_value { |c| c.docs.sort! }
+      site.pages.sort_by!(&:name)
+      site.static_files.sort_by!(&:relative_path)
+    end
+
+    # Recursively traverse directories to find pages and static files
+    # that will become part of the site according to the rules in
+    # filter_entries.
+    #
+    # dir - The String relative path of the directory to read. Default: ''.
+    #
+    # Returns nothing.
+    def read_directories(dir = "")
+      base = site.in_source_dir(dir)
+
+      return unless File.directory?(base)
+
+      dot_dirs = []
+      dot_pages = []
+      dot_static_files = []
+
+      dot = Dir.chdir(base) { filter_entries(Dir.entries("."), base) }
+      dot.each do |entry|
+        file_path = @site.in_source_dir(base, entry)
+        if File.directory?(file_path)
+          dot_dirs << entry
+        elsif Utils.has_yaml_header?(file_path)
+          dot_pages << entry
+        else
+          dot_static_files << entry
+        end
+      end
+
+      retrieve_posts(dir)
+      retrieve_dirs(base, dir, dot_dirs)
+      retrieve_pages(dir, dot_pages)
+      retrieve_static_files(dir, dot_static_files)
+    end
+
+    # Retrieves all the posts(posts/drafts) from the given directory
+    # and add them to the site and sort them.
+    #
+    # dir - The String representing the directory to retrieve the posts from.
+    #
+    # Returns nothing.
+    def retrieve_posts(dir)
+      return if outside_configured_directory?(dir)
+
+      site.posts.docs.concat(post_reader.read_posts(dir))
+      site.posts.docs.concat(post_reader.read_drafts(dir)) if site.show_drafts
+    end
+
+    # Recursively traverse directories with the read_directories function.
+    #
+    # base - The String representing the site's base directory.
+    # dir - The String representing the directory to traverse down.
+    # dot_dirs - The Array of subdirectories in the dir.
+    #
+    # Returns nothing.
+    def retrieve_dirs(_base, dir, dot_dirs)
+      dot_dirs.each do |file|
+        dir_path = site.in_source_dir(dir, file)
+        rel_path = PathManager.join(dir, file)
+        @site.reader.read_directories(rel_path) unless @site.dest.chomp("/") == dir_path
+      end
+    end
+
+    # Retrieve all the pages from the current directory,
+    # add them to the site and sort them.
+    #
+    # dir - The String representing the directory retrieve the pages from.
+    # dot_pages - The Array of pages in the dir.
+    #
+    # Returns nothing.
+    def retrieve_pages(dir, dot_pages)
+      site.pages.concat(PageReader.new(site, dir).read(dot_pages))
+    end
+
+    # Retrieve all the static files from the current directory,
+    # add them to the site and sort them.
+    #
+    # dir - The directory retrieve the static files from.
+    # dot_static_files - The static files in the dir.
+    #
+    # Returns nothing.
+    def retrieve_static_files(dir, dot_static_files)
+      site.static_files.concat(StaticFileReader.new(site, dir).read(dot_static_files))
+    end
+
+    # Filter out any files/directories that are hidden or backup files (start
+    # with "." or "#" or end with "~"), or contain site content (start with "_"),
+    # or are excluded in the site configuration, unless they are web server
+    # files such as '.htaccess'.
+    #
+    # entries - The Array of String file/directory entries to filter.
+    # base_directory - The string representing the optional base directory.
+    #
+    # Returns the Array of filtered entries.
+    def filter_entries(entries, base_directory = nil)
+      EntryFilter.new(site, base_directory).filter(entries)
+    end
+
+    # Read the entries from a particular directory for processing
+    #
+    # dir - The String representing the relative path of the directory to read.
+    # subfolder - The String representing the directory to read.
+    #
+    # Returns the list of entries to process
+    def get_entries(dir, subfolder)
+      base = site.in_source_dir(dir, subfolder)
+      return [] unless File.exist?(base)
+
+      entries = Dir.chdir(base) { filter_entries(Dir["**/*"], base) }
+      entries.delete_if { |e| File.directory?(site.in_source_dir(base, e)) }
+    end
+
+    private
+
+    # Internal
+    #
+    # Determine if the directory is supposed to contain posts and drafts.
+    # If the user has defined a custom collections_dir, then attempt to read
+    # posts and drafts only from within that directory.
+    #
+    # Returns true if a custom collections_dir has been set but current directory lies
+    # outside that directory.
+    def outside_configured_directory?(dir)
+      collections_dir = site.config["collections_dir"]
+      !collections_dir.empty? && !dir.start_with?("/#{collections_dir}")
+    end
+
+    # Create a single PostReader instance to retrieve drafts and posts from all valid
+    # directories in current site.
+    def post_reader
+      @post_reader ||= PostReader.new(site)
+    end
+
+    def read_included_excludes
+      entry_filter = EntryFilter.new(site)
+
+      site.include.each do |entry|
+        entry_path = site.in_source_dir(entry)
+        next if File.directory?(entry_path)
+        next if entry_filter.symlink?(entry_path)
+
+        read_included_file(entry_path) if File.file?(entry_path)
+      end
+    end
+
+    def read_included_file(entry_path)
+      if Utils.has_yaml_header?(entry_path)
+        conditionally_generate_entry(entry_path, site.pages, PageReader)
+      else
+        conditionally_generate_entry(entry_path, site.static_files, StaticFileReader)
+      end
+    end
+
+    def conditionally_generate_entry(entry_path, container, reader)
+      return if container.find { |item| site.in_source_dir(item.relative_path) == entry_path }
+
+      dir, files = File.split(entry_path)
+      dir.sub!(site.source, "")
+      files = Array(files)
+      container.concat(reader.new(site, dir).read(files))
+    end
+  end
+end

data/lib/jekyll/readers/collection_reader.rb

@@ -1,23 +1,23 @@
-# frozen_string_literal: true
-
-module Jekyll
-  class CollectionReader
-    SPECIAL_COLLECTIONS = %w(posts data).freeze
-
-    attr_reader :site, :content
-
-    def initialize(site)
-      @site = site
-      @content = {}
-    end
-
-    # Read in all collections specified in the configuration
-    #
-    # Returns nothing.
-    def read
-      site.collections.each_value do |collection|
-        collection.read unless SPECIAL_COLLECTIONS.include?(collection.label)
-      end
-    end
-  end
-end
+# frozen_string_literal: true
+
+module Jekyll
+  class CollectionReader
+    SPECIAL_COLLECTIONS = %w(posts data).freeze
+
+    attr_reader :site, :content
+
+    def initialize(site)
+      @site = site
+      @content = {}
+    end
+
+    # Read in all collections specified in the configuration
+    #
+    # Returns nothing.
+    def read
+      site.collections.each_value do |collection|
+        collection.read unless SPECIAL_COLLECTIONS.include?(collection.label)
+      end
+    end
+  end
+end

data/lib/jekyll/readers/data_reader.rb

@@ -1,79 +1,116 @@
-# frozen_string_literal: true
-
-module Jekyll
-  class DataReader
-    attr_reader :site, :content
-
-    def initialize(site)
-      @site = site
-      @content = {}
-      @entry_filter = EntryFilter.new(site)
-      @source_dir = site.in_source_dir("/")
-    end
-
-    # Read all the files in <dir> and adds them to @content
-    #
-    # dir - The String relative path of the directory to read.
-    #
-    # Returns @content, a Hash of the .yaml, .yml,
-    # .json, and .csv files in the base directory
-    def read(dir)
-      base = site.in_source_dir(dir)
-      read_data_to(base, @content)
-      @content
-    end
-
-    # Read and parse all .yaml, .yml, .json, .csv and .tsv
-    # files under <dir> and add them to the <data> variable.
-    #
-    # dir - The string absolute path of the directory to read.
-    # data - The variable to which data will be added.
-    #
-    # Returns nothing
-    def read_data_to(dir, data)
-      return unless File.directory?(dir) && !@entry_filter.symlink?(dir)
-
-      entries = Dir.chdir(dir) do
-        Dir["*.{yaml,yml,json,csv,tsv}"] + Dir["*"].select { |fn| File.directory?(fn) }
-      end
-
-      entries.each do |entry|
-        path = @site.in_source_dir(dir, entry)
-        next if @entry_filter.symlink?(path)
-
-        if File.directory?(path)
-          read_data_to(path, data[sanitize_filename(entry)] = {})
-        else
-          key = sanitize_filename(File.basename(entry, ".*"))
-          data[key] = read_data_file(path)
-        end
-      end
-    end
-
-    # Determines how to read a data file.
-    #
-    # Returns the contents of the data file.
-    def read_data_file(path)
-      Jekyll.logger.debug "Reading:", path.sub(@source_dir, "")
-
-      case File.extname(path).downcase
-      when ".csv"
-        CSV.read(path,
-                 :headers  => true,
-                 :encoding => site.config["encoding"]).map(&:to_hash)
-      when ".tsv"
-        CSV.read(path,
-                 :col_sep  => "\t",
-                 :headers  => true,
-                 :encoding => site.config["encoding"]).map(&:to_hash)
-      else
-        SafeYAML.load_file(path)
-      end
-    end
-
-    def sanitize_filename(name)
-      name.gsub(%r![^\w\s-]+|(?<=^|\b\s)\s+(?=$|\s?\b)!, "")
-        .gsub(%r!\s+!, "_")
-    end
-  end
-end
+# frozen_string_literal: true
+
+module Jekyll
+  class DataReader
+    attr_reader :site, :content
+
+    def initialize(site, in_source_dir: nil)
+      @site = site
+      @content = DataHash.new
+      @entry_filter = EntryFilter.new(site)
+      @in_source_dir = in_source_dir || @site.method(:in_source_dir)
+      @source_dir = @in_source_dir.call("/")
+    end
+
+    # Read all the files in <dir> and adds them to @content
+    #
+    # dir - The String relative path of the directory to read.
+    #
+    # Returns @content, a Hash of the .yaml, .yml,
+    # .json, and .csv files in the base directory
+    def read(dir)
+      base = @in_source_dir.call(dir)
+      read_data_to(base, @content)
+      @content
+    end
+
+    # rubocop:disable Metrics/AbcSize
+
+    # Read and parse all .yaml, .yml, .json, .csv and .tsv
+    # files under <dir> and add them to the <data> variable.
+    #
+    # dir - The string absolute path of the directory to read.
+    # data - The variable to which data will be added.
+    #
+    # Returns nothing
+    def read_data_to(dir, data)
+      return unless File.directory?(dir) && !@entry_filter.symlink?(dir)
+
+      entries = Dir.chdir(dir) do
+        Dir["*.{yaml,yml,json,csv,tsv}"] + Dir["*"].select { |fn| File.directory?(fn) }
+      end
+
+      entries.each do |entry|
+        path = @in_source_dir.call(dir, entry)
+        next if @entry_filter.symlink?(path)
+
+        if File.directory?(path)
+          read_data_to(path, data[sanitize_filename(entry)] = DataHash.new)
+        else
+          key = sanitize_filename(File.basename(entry, ".*"))
+          data[key] = DataEntry.new(site, path, read_data_file(path))
+        end
+      end
+    end
+    # rubocop:enable Metrics/AbcSize
+
+    # Determines how to read a data file.
+    #
+    # Returns the contents of the data file.
+    def read_data_file(path)
+      Jekyll.logger.debug "Reading:", path.sub(@source_dir, "")
+
+      case File.extname(path).downcase
+      when ".csv"
+        CSV.read(path, **csv_config).map { |row| convert_row(row) }
+      when ".tsv"
+        CSV.read(path, **tsv_config).map { |row| convert_row(row) }
+      else
+        SafeYAML.load_file(path)
+      end
+    end
+
+    def sanitize_filename(name)
+      name.gsub(%r![^\w\s-]+|(?<=^|\b\s)\s+(?=$|\s?\b)!, "")
+        .gsub(%r!\s+!, "_")
+    end
+
+    private
+
+    # @return [Hash]
+    def csv_config
+      @csv_config ||= read_config("csv_reader")
+    end
+
+    # @return [Hash]
+    def tsv_config
+      @tsv_config ||= read_config("tsv_reader", { :col_sep => "\t" })
+    end
+
+    # @param config_key [String]
+    # @param overrides [Hash]
+    # @return [Hash]
+    # @see https://ruby-doc.org/stdlib-2.5.0/libdoc/csv/rdoc/CSV.html#Converters
+    def read_config(config_key, overrides = {})
+      reader_config = config[config_key] || {}
+
+      defaults = {
+        :converters => reader_config.fetch("csv_converters", []).map(&:to_sym),
+        :headers    => reader_config.fetch("headers", true),
+        :encoding   => reader_config.fetch("encoding", config["encoding"]),
+      }
+
+      defaults.merge(overrides)
+    end
+
+    def config
+      @config ||= site.config
+    end
+
+    # @param row [Array, CSV::Row]
+    # @return [Array, Hash]
+    def convert_row(row)
+      row.instance_of?(CSV::Row) ? row.to_hash : row
+    end
+  end
+end