wax_tasks 0.3.2 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d88624cfe6327eb5449dda2246712718f4bdfb2fea155e12e1593b6a5e10af08
-  data.tar.gz: d6aa6846dedcbaa295d4ae3f909c1f22fa68c55db1524b7aea1e7a958ef241ad
+  metadata.gz: de0380de7a7fbba0a355c3983945d2517460f0ed7e0848e518896140ea3a1992
+  data.tar.gz: 609510c447fa45acd2cd638141fe8f81180a27e9b1e43d6fae714e2b052e1add
 SHA512:
-  metadata.gz: b3c8dd46e8631bbff1bcdb2792255882ac37cb1afd4b8ed9e193e24e0625c934eeda807d56417832df1fbfbdc9dec435df02df5058c3907794761bf704734fcc
-  data.tar.gz: 954bbf0a5a9561ad4b74bf751de5e45e6cc5ec7de21a5fd82720789483470c29f1efc161fa62af1e69c2b4b8479956f82a4acd73277d3efd42bf36cfb0a2ef97
+  metadata.gz: 513a791a3d0a622af9f0cd386d5ad4d95c76a7c2feae1cd18126f2cdd83df605f86d81884a42e95a8ff152ab784898457b18c151a5c5db2ab71ea13b34c84a6b
+  data.tar.gz: 158e96f76794b2fbafc26c6881cca091b5621fe7cdf4c0700353af4d89dfd1fe43ab3754820332e516fb8bfa04101c76a5f96e95d6438eb9ff68932d1b84f808

data/Gemfile

@@ -1,8 +1,11 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 gemspec
 
 # dev/test utilities
+gem 'bundle-audit', require: false
 gem 'diane', require: false
 gem 'rubocop', require: false
-gem 'simplecov', require: false
+gem 'simplecov', '0.17.1', require: false
 gem 'yard', require: false

data/lib/tasks/clobber.rake

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'wax_tasks'
+
+namespace :wax do
+  desc 'destroy wax-generated collection files, including pages, derivatives, and search index(es)'
+  task :clobber do
+    args = ARGV.drop(1).each { |a| task a.to_sym }
+    args.reject! { |a| a.start_with? '-' }
+
+    raise WaxTasks::Error::MissingArguments, Rainbow("You must specify a collection after 'wax:clobber'").magenta if args.empty?
+
+    site = WaxTasks::Site.new
+    args.each { |a| site.clobber a }
+  end
+end

data/lib/tasks/derivatives_iiif.rake

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'wax_tasks'
+
+namespace :wax do
+  namespace :derivatives do
+    desc 'generate iiif derivatives from local image files'
+    task :iiif do
+      args = ARGV.drop(1).each { |a| task a.to_sym }
+      args.reject! { |a| a.start_with? '-' }
+
+      raise WaxTasks::Error::MissingArguments, Rainbow("You must specify a collection after 'wax:derivatives:iiif'").magenta if args.empty?
+
+      site = WaxTasks::Site.new
+      args.each { |a| site.generate_derivatives(a, 'iiif') }
+    end
+  end
+
+  # alias wax:iiif to wax:derivatives:iiif for backwards compatibility
+  task :iiif do
+    t = Rake::Task['wax:derivatives:iiif']
+    desc t.full_comment if t.full_comment
+    arguments = ARGV.drop(1).each { |a| task a.to_sym }
+    t.invoke(*arguments)
+  end
+end

data/lib/tasks/derivatives_simple.rake

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'wax_tasks'
+
+namespace :wax do
+  namespace :derivatives do
+    desc 'generate iiif derivatives from local image files'
+    task :simple do
+      args = ARGV.drop(1).each { |a| task a.to_sym }
+      args.reject! { |a| a.start_with? '-' }
+      raise WaxTasks::Error::MissingArguments, Rainbow("You must specify a collection after 'wax:derivatives:simple'").magenta if args.empty?
+
+      site = WaxTasks::Site.new
+      args.each { |a| site.generate_derivatives(a, 'simple') }
+    end
+  end
+end

data/lib/tasks/pages.rake

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'wax_tasks'
+
+namespace :wax do
+  desc 'generate collection md pages from yaml or csv data source'
+  task :pages do
+    args = ARGV.drop(1).each { |a| task a.to_sym }
+    args.reject! { |a| a.start_with? '-' }
+    raise WaxTasks::Error::MissingArguments, Rainbow('You must specify a collection after wax:pages').magenta if args.empty?
+
+    site = WaxTasks::Site.new
+    args.each { |a| site.generate_pages a }
+  end
+
+  # alias :pagemaster to wax:pages for backwards compatibility
+  task :pagemaster do
+    t = Rake::Task['wax:pages']
+    desc t.full_comment if t.full_comment
+    args = ARGV.drop(1).each { |a| task a.to_sym }
+    t.invoke(*args)
+  end
+end

data/lib/tasks/search.rake

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'wax_tasks'
+
+namespace :wax do
+  desc 'build lunr search index (with default UI if UI=true)'
+  task :search do
+    args = ARGV.drop(1).each { |a| task a.to_sym }
+    args.reject! { |a| a.start_with? '-' }
+    raise WaxTasks::Error::MissingArguments, Rainbow('You must specify a collection after wax:search').magenta if args.empty?
+
+    site = WaxTasks::Site.new
+    args.each { |a| site.generate_static_search a }
+  end
+
+  # alias lunr to search for backwards compatibility
+  task :lunr do
+    t = Rake::Task['wax:search']
+    desc t.full_comment if t.full_comment
+    args = ARGV.drop(1).each { |a| task a.to_sym }
+    t.invoke(*args)
+  end
+end

data/lib/wax_tasks.rb

@@ -1,37 +1,36 @@
-require_relative 'wax_tasks/branch'
+# frozen_string_literal: true
+
+# rubygems
+require 'rubygems'
+
+# stdlib
+require 'csv'
+require 'fileutils'
+require 'json'
+
+# 3rd party
+require 'rainbow'
+require 'safe_yaml'
+
+# relative
+require_relative 'wax_tasks/asset'
 require_relative 'wax_tasks/collection'
+require_relative 'wax_tasks/config'
 require_relative 'wax_tasks/error'
-require_relative 'wax_tasks/iiif_collection'
-require_relative 'wax_tasks/local_branch'
-require_relative 'wax_tasks/lunr_collection'
-require_relative 'wax_tasks/lunr_index'
-require_relative 'wax_tasks/pagemaster_collection'
-require_relative 'wax_tasks/task_runner'
-require_relative 'wax_tasks/travis_branch'
+require_relative 'wax_tasks/index'
+require_relative 'wax_tasks/item'
+require_relative 'wax_tasks/record'
+require_relative 'wax_tasks/site'
 require_relative 'wax_tasks/utils'
 
-# The WaxTasks module powers the Rake tasks in `./tasks`, including:
-#
-# wax:pagemaster  :: generate collection md pages from csv, json, or yaml file
-# wax:lunr        :: build lunr search index (with default UI if UI=true)
-# wax:iiif        :: generate iiif derivatives from local image files
-# wax:jspackage   :: write a simple package.json for monitoring js dependencies
-# wax:push        :: push compiled Jekyll site to git branch
-# wax:test        :: run htmlproofer, rspec if .rspec file exists
 #
-# Tasks are run by a WaxTasks::TaskRunner object which is resposible
-# for reading in site config from `_config.yml`
 module WaxTasks
-  # ----------
-  # CONSTANTS
-  # ----------
-
-  # @return [String] The path to load Jekyll site config
-  DEFAULT_CONFIG  = '_config.yml'.freeze
-  # @return [String] The path to write WaxTasks::LunrIndex
-  LUNR_INDEX_PATH = 'js/lunr-index.json'.freeze
-  # @return [String] The path to write default LunrUI
-  LUNR_UI_PATH    = 'js/lunr-ui.js'.freeze
-  # @return [String] The path to the compiled Jekyll site
-  SITE_DIR        = './_site'.freeze
+  DEFAULT_CONFIG_FILE = './_config.yml'
+  #
+  #
+  def self.config_from_file(file = nil)
+    Utils.validate_yaml(file || DEFAULT_CONFIG_FILE)
+  rescue StandardError => e
+    raise WaxTasks::Error::InvalidConfig, "Cannot open config file '#{file}'.\n #{e}"
+  end
 end

data/lib/wax_tasks/asset.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+#
+module WaxTasks
+  Derivative = Struct.new(:path, :label, :img)
+
+  #
+  class Asset
+    attr_reader :id, :path
+
+    DEFAULT_VARIANTS = { 'thumbnail' => 250, 'full' => 1140 }.freeze
+
+    def initialize(path, pid, variants)
+      @path     = path
+      @pid      = pid
+      @id       = asset_id
+      @variants = DEFAULT_VARIANTS.merge variants
+    end
+
+    #
+    #
+    def asset_id
+      id = File.basename @path, '.*'
+      id.prepend "#{@pid}_" unless id == @pid
+      id
+    end
+
+    #
+    #
+    def simple_derivatives
+      @variants.map do |label, width|
+        img = MiniMagick::Image.open @path
+        if width > img.width
+          warn Rainbow("Tried to create derivative #{width}px wide, but asset #{@id} for item #{@pid} only has a width of #{img.width}px.").yellow
+        else
+          img.resize width
+        end
+
+        img.format 'jpg'
+        Derivative.new("#{@id}/#{label}.jpg", label, img)
+      end
+    end
+
+    #
+    #
+    def to_iiif_image_record(is_only, index, base_opts)
+      opts = base_opts.clone
+
+      opts[:is_primary]    = index.zero?
+      opts[:section_label] = "Page #{index + 1}" unless is_only
+      opts[:path]          = @path
+      opts[:manifest_id]   = @pid
+      opts[:id]            = @id
+      opts[:variants]      = @variants
+
+      WaxIiif::ImageRecord.new(opts)
+    end
+  end
+end

data/lib/wax_tasks/collection.rb

@@ -1,65 +1,53 @@
+# frozen_string_literal: true
+
+require_relative 'collection/images'
+require_relative 'collection/metadata'
+
 module WaxTasks
-  # Parent class representing a Jekyll collection
-  # that cannot be created directly. Only child classes
-  # (IiifCollection, LunrCollection, PagemasterCollection)
-  # can be initialized.
   #
-  # @attr config    [Hash]    the collection config within site config
-  # @attr name      [String]  the name of the collection in site:collections
-  # @attr page_dir  [String]  the directory path for generated collection pages
-  # @attr site      [Hash]    the site config
   class Collection
-    attr_reader :name, :page_dir
-    private_class_method :new
+    attr_reader :name, :config, :ext, :search_fields,
+                :page_source, :metadata_source, :imagedata_source,
+                :iiif_derivative_source, :simple_derivative_source
 
-    # This method ensures child classes can be instantiated though
-    # Collection.new cannot be.
-    def self.inherited(*)
-      public_class_method :new
-    end
+    include Collection::Metadata
+    include Collection::Images
 
-    # Creates a new collection with name @name given site config @site
+    IMAGE_DERIVATIVE_DIRECTORY = 'img/derivatives'
+
+    #
     #
-    # @param name     [String]  the name of the collection in site:collections
-    # @param site     [Hash]    the site config
-    def initialize(name, site)
-      @name     = name
-      @site     = site
-      @config   = collection_config
-      @page_dir = Utils.make_path(@site[:source_dir],
-                                  @site[:collections_dir],
-                                  "_#{@name}")
+    def initialize(name, config, source, collections_dir, ext)
+      @name                     = name
+      @config                   = config
+      @page_extension           = ext
+      @site_source              = source
+      @page_source              = Utils.safe_join source, collections_dir, "_#{@name}"
+      @metadata_source          = Utils.safe_join source, '_data', config.dig('metadata', 'source')
+      @imagedata_source         = Utils.safe_join source, '_data', config.dig('images', 'source')
+      @iiif_derivative_source   = Utils.safe_join source, IMAGE_DERIVATIVE_DIRECTORY, 'iiif'
+      @simple_derivative_source = Utils.safe_join source, IMAGE_DERIVATIVE_DIRECTORY, 'simple'
+      @search_fields            = %w[pid label thumbnail permalink collection]
+      @image_variants           = @config.dig('images', 'variants') || {}
     end
 
-    # Finds the collection config within the site config
     #
-    # @return [Hash] the config for the collection
-    def collection_config
-      @site[:collections].fetch(@name)
-    rescue StandardError => e
-      raise Error::InvalidCollection, "Cannot load collection config for #{@name}.\n#{e}"
+    #
+    def clobber_pages
+      return unless Dir.exist? @page_source
+      puts Rainbow("Removing pages from #{@page_source}").cyan
+      FileUtils.remove_dir @page_source, true
     end
 
-    # Ingests the collection source data as an Array of Hashes
     #
-    # @param source [String] the path to the CSV, JSON, or YAML source file
-    # @return [Array] the collection data
-    def ingest_file(source)
-      raise Error::MissingSource, "Cannot find #{source}" unless File.exist? source
-
-      data = case File.extname(source)
-             when '.csv'
-               WaxTasks::Utils.validate_csv(source)
-             when '.json'
-               WaxTasks::Utils.validate_json(source)
-             when /\.ya?ml/
-               WaxTasks::Utils.validate_yaml(source)
-             else
-               raise Error::InvalidSource, "Can't load #{File.extname(source)} files. Culprit: #{source}"
-             end
-
-      WaxTasks::Utils.assert_pids(data)
-      WaxTasks::Utils.assert_unique(data)
+    #
+    def clobber_derivatives
+      [@iiif_derivative_source, @simple_derivative_source].each do |dir|
+        if Dir.exist? dir
+          puts Rainbow("Removing derivatives from #{dir}").cyan
+          FileUtils.remove_dir dir, true
+        end
+      end
     end
   end
 end

data/lib/wax_tasks/collection/images.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require 'mini_magick'
+require 'progress_bar'
+require 'wax_iiif'
+
+#
+module WaxTasks
+  #
+  class Collection
+    #
+    module Images
+      #
+      #
+      def items_from_imagedata
+        raise Error::MissingSource, "Cannot find image data source '#{@imagedata_source}'" unless Dir.exist? @imagedata_source
+
+        pre_process_pdfs
+        records = records_from_metadata
+        Dir.glob(Utils.safe_join(@imagedata_source, '*')).map do |path|
+          item = WaxTasks::Item.new(path, @image_variants)
+          if item.valid?
+            item.record      = records.find { |r| r.pid == item.pid }
+            item.iiif_config = @config.dig 'images', 'iiif'
+            warn Rainbow("\nCould not find record in #{@metadata_source} for image item #{path}.\n").orange if item.record.nil?
+            item
+          else
+            puts Rainbow("Skipping #{path} because type #{item.type} is not an accepted format").yellow unless item.type == '.pdf'
+          end
+        end.compact
+      end
+
+      #
+      #
+      def pre_process_pdfs
+        Dir.glob(Utils.safe_join(@imagedata_source, '*.pdf')).each do |path|
+          target_dir = path.gsub '.pdf', ''
+          next unless Dir.glob("#{target_dir}/*").empty?
+
+          puts Rainbow("\nPreprocessing #{path} into image files. This may take a minute.\n").cyan
+
+          opts = { output_dir: File.dirname(target_dir) }
+          WaxIiif::Utilities::PdfSplitter.split(path, opts)
+        end
+      end
+
+      #
+      #
+      def write_simple_derivatives
+        puts Rainbow("Generating simple image derivatives for collection '#{@name}'\nThis might take awhile.").cyan
+
+        bar = ProgressBar.new(items_from_imagedata.length)
+        bar.write
+        items_from_imagedata.map do |item|
+          item.simple_derivatives.each do |d|
+            path = "#{@simple_derivative_source}/#{d.path}"
+            FileUtils.mkdir_p File.dirname(path)
+            next if File.exist? path
+
+            d.img.write path
+            item.record.set d.label, path if item.record?
+          end
+          bar.increment!
+          bar.write
+          item
+        end.flat_map(&:record).compact
+      end
+
+      #
+      #
+      def iiif_builder(dir)
+        build_opts = {
+          base_url: "{{ '/' | absolute_url }}#{dir}",
+          output_dir: dir,
+          collection_label: @name,
+          variants: @image_variants.dup.tap { |h| h.delete 'full' }
+        }
+        WaxIiif::Builder.new build_opts
+      end
+
+      #
+      #
+      def add_font_matter_to_json_files(dir)
+        Dir.glob("#{dir}/**/*.json").each do |f|
+          Utils.add_yaml_front_matter_to_file f
+        end
+      end
+
+      #
+      #
+      def add_iiif_results_to_records(records, manifests)
+        records.map do |record|
+          next nil if record.nil?
+
+          manifest = manifests.find { |m| m.base_id == record.pid }
+          next record if manifest.nil?
+
+          json = JSON.parse manifest.to_json
+          @image_variants.each do |k, _v|
+            value = json.dig k
+            record.set k, "/#{Utils.content_clean(value)}" unless value.nil?
+          end
+
+          record.set 'manifest', "/#{Utils.content_clean(manifest.id)}"
+          record
+        end.compact
+      end
+
+      #
+      #
+      def write_iiif_derivatives
+        items     = items_from_imagedata
+        iiif_data = items.map(&:iiif_image_records).flatten
+        builder   = iiif_builder @iiif_derivative_source
+
+        builder.load iiif_data
+
+        puts Rainbow("Generating IIIF derivatives for collection '#{@name}'\nThis might take awhile.").cyan
+        builder.process_data
+        records = items.map(&:record).compact
+
+        add_font_matter_to_json_files @iiif_derivative_source
+        add_iiif_results_to_records records, builder.manifests
+      end
+    end
+  end
+end