wax_tasks 1.0.0.pre.beta → 1.1.2

data/lib/tasks/clobber.rake

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'wax_tasks'
+
+namespace :wax do
+  desc 'destroy wax-generated collection files, including pages, derivatives, and search index(es)'
+  task :clobber do
+    args = ARGV.drop(1).each { |a| task a.to_sym }
+    args.reject! { |a| a.start_with? '-' }
+
+    raise WaxTasks::Error::MissingArguments, Rainbow("You must specify a collection after 'wax:clobber'").magenta if args.empty?
+
+    site = WaxTasks::Site.new
+    args.each { |a| site.clobber a }
+  end
+end

data/lib/tasks/derivatives_iiif.rake

@@ -1,13 +1,18 @@
+# frozen_string_literal: true
+
 require 'wax_tasks'
 
 namespace :wax do
   namespace :derivatives do
     desc 'generate iiif derivatives from local image files'
     task :iiif do
-      arguments = ARGV.drop(1).each { |a| task a.to_sym }
-      raise WaxTasks::Error::MissingArguments, "You must specify a collection after 'wax:derivatives:iiif'" if arguments.empty?
-      task_runner = WaxTasks::TaskRunner.new
-      task_runner.derivatives_iiif(arguments)
+      args = ARGV.drop(1).each { |a| task a.to_sym }
+      args.reject! { |a| a.start_with? '-' }
+
+      raise WaxTasks::Error::MissingArguments, Rainbow("You must specify a collection after 'wax:derivatives:iiif'").magenta if args.empty?
+
+      site = WaxTasks::Site.new
+      args.each { |a| site.generate_derivatives(a, 'iiif') }
     end
   end
 

data/lib/tasks/derivatives_simple.rake

@@ -1,13 +1,17 @@
+# frozen_string_literal: true
+
 require 'wax_tasks'
 
 namespace :wax do
   namespace :derivatives do
     desc 'generate iiif derivatives from local image files'
     task :simple do
-      arguments = ARGV.drop(1).each { |a| task a.to_sym }
-      raise WaxTasks::Error::MissingArguments, "You must specify a collection after 'wax:derivatives:simple'" if arguments.empty?
-      task_runner = WaxTasks::TaskRunner.new
-      task_runner.derivatives_simple(arguments)
+      args = ARGV.drop(1).each { |a| task a.to_sym }
+      args.reject! { |a| a.start_with? '-' }
+      raise WaxTasks::Error::MissingArguments, Rainbow("You must specify a collection after 'wax:derivatives:simple'").magenta if args.empty?
+
+      site = WaxTasks::Site.new
+      args.each { |a| site.generate_derivatives(a, 'simple') }
     end
   end
 end

data/lib/tasks/pages.rake

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'wax_tasks'
+
+namespace :wax do
+  desc 'generate collection md pages from yaml or csv data source'
+  task :pages do
+    args = ARGV.drop(1).each { |a| task a.to_sym }
+    args.reject! { |a| a.start_with? '-' }
+    raise WaxTasks::Error::MissingArguments, Rainbow('You must specify a collection after wax:pages').magenta if args.empty?
+
+    site = WaxTasks::Site.new
+    args.each { |a| site.generate_pages a }
+  end
+
+  # alias :pagemaster to wax:pages for backwards compatibility
+  task :pagemaster do
+    t = Rake::Task['wax:pages']
+    desc t.full_comment if t.full_comment
+    args = ARGV.drop(1).each { |a| task a.to_sym }
+    t.invoke(*args)
+  end
+end

data/lib/tasks/search.rake

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'wax_tasks'
+
+namespace :wax do
+  desc 'build lunr search index (with default UI if UI=true)'
+  task :search do
+    args = ARGV.drop(1).each { |a| task a.to_sym }
+    args.reject! { |a| a.start_with? '-' }
+    raise WaxTasks::Error::MissingArguments, Rainbow('You must specify a collection after wax:search').magenta if args.empty?
+
+    site = WaxTasks::Site.new
+    args.each { |a| site.generate_static_search a }
+  end
+
+  # alias lunr to search for backwards compatibility
+  task :lunr do
+    t = Rake::Task['wax:search']
+    desc t.full_comment if t.full_comment
+    args = ARGV.drop(1).each { |a| task a.to_sym }
+    t.invoke(*args)
+  end
+end

data/lib/wax_tasks.rb

@@ -1,43 +1,37 @@
-require_relative 'wax_tasks/branch'
+# frozen_string_literal: true
+
+# rubygems
+require 'rubygems'
+
+# stdlib
+require 'csv'
+require 'fileutils'
+require 'json'
+
+# 3rd party
+require 'rainbow'
+require 'safe_yaml'
+
+# relative
+require_relative 'wax_tasks/asset'
 require_relative 'wax_tasks/collection'
+require_relative 'wax_tasks/config'
 require_relative 'wax_tasks/error'
-require_relative 'wax_tasks/image_collection'
-require_relative 'wax_tasks/local_branch'
-require_relative 'wax_tasks/lunr/index'
-require_relative 'wax_tasks/pagemaster_collection'
-require_relative 'wax_tasks/task_runner'
-require_relative 'wax_tasks/travis_branch'
+require_relative 'wax_tasks/index'
+require_relative 'wax_tasks/item'
+require_relative 'wax_tasks/record'
+require_relative 'wax_tasks/site'
 require_relative 'wax_tasks/utils'
+require_relative 'wax_tasks/version'
 
-# The WaxTasks module powers the Rake tasks in `./tasks`, including:
-#
-# wax:pagemaster          :: generate collection md pages from csv, json, or yaml file
-# wax:lunr                :: build lunr search index (with default UI if UI=true)
-# wax:derivatives:simple  :: generate simple image derivatives from local image files
-# wax:derivatves:iiif     :: generate iiif derivatives from local image files
-# wax:jspackage           :: write a simple package.json for monitoring js dependencies
-# wax:push                :: push compiled Jekyll site to git branch
-# wax:test                :: run htmlproofer, rspec if .rspec file exists
 #
-# Tasks are run by a WaxTasks::TaskRunner object which is resposible
-# for reading in site config from `_config.yml`
 module WaxTasks
-  # ----------
-  # CONSTANTS
-  # ----------
-
-  # @return [String] The path to load Jekyll site config
-  DEFAULT_CONFIG          = '_config.yml'.freeze
-
-  # @return [String] The path to write default LunrUI
-  LUNR_UI_PATH            = 'js/lunr-ui.js'.freeze
-
-  # @return [String] The path to the compiled Jekyll site
-  SITE_DIR                = '_site'.freeze
-
-  # @return [String] Default image variant/derivative widths to generate
-  DEFAULT_IMAGE_VARIANTS  = { thumbnail: 250, full: 1140 }.freeze
-
-  # @return [String] The path where image derivatives should be generated
-  DEFAULT_DERIVATIVE_DIR  = 'img/derivatives'.freeze
+  DEFAULT_CONFIG_FILE = './_config.yml'
+  #
+  #
+  def self.config_from_file(file = nil)
+    Utils.validate_yaml(file || DEFAULT_CONFIG_FILE)
+  rescue StandardError => e
+    raise WaxTasks::Error::InvalidConfig, "Cannot open config file '#{file}'.\n #{e}"
+  end
 end

data/lib/wax_tasks/asset.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+#
+module WaxTasks
+  Derivative = Struct.new(:path, :label, :img)
+
+  #
+  class Asset
+    attr_reader :id, :path
+
+    def initialize(path, pid, variants)
+      @path     = path
+      @pid      = pid
+      @id       = asset_id
+      @variants = variants
+    end
+
+    #
+    #
+    def asset_id
+      id = File.basename @path, '.*'
+      id.prepend "#{@pid}_" unless id == @pid
+      id
+    end
+
+    #
+    #
+    def simple_derivatives
+      @variants.map do |label, width|
+        img = MiniMagick::Image.open @path
+        if width > img.width
+          warn Rainbow("Tried to create derivative #{width}px wide, but asset #{@id} for item #{@pid} only has a width of #{img.width}px.").yellow
+        else
+          img.resize width
+        end
+
+        img.format 'jpg'
+        Derivative.new("#{@id}/#{label}.jpg", label, img)
+      end
+    end
+
+    #
+    #
+    def to_iiif_image_record(is_only, index, base_opts)
+      opts = base_opts.clone
+
+      opts[:is_primary]    = index.zero?
+      opts[:section_label] = "Page #{index + 1}" unless is_only
+      opts[:path]          = @path
+      opts[:manifest_id]   = @pid
+      opts[:id]            = @id
+      opts[:variants]      = @variants
+
+      WaxIiif::ImageRecord.new(opts)
+    end
+  end
+end

data/lib/wax_tasks/collection.rb

@@ -1,91 +1,60 @@
+# frozen_string_literal: true
+
+require_relative 'collection/images'
+require_relative 'collection/metadata'
+
 module WaxTasks
-  # Parent class representing a Jekyll collection
-  # that cannot be created directly. Only child classes
-  # (IiifCollection, LunrCollection, PagemasterCollection)
-  # can be initialized.
+  #
   class Collection
-    attr_accessor :name, :site
-    private_class_method :new
+    attr_reader :name, :config, :ext, :search_fields,
+                :page_source, :metadata_source, :imagedata_source,
+                :iiif_derivative_source, :simple_derivative_source
 
-    # This method ensures child classes can be instantiated though
-    # Collection.new cannot be.
-    def self.inherited(*)
-      public_class_method :new
-    end
+    include Collection::Metadata
+    include Collection::Images
 
-    # Creates a new collection with name @name given site configuration @site
-    #
-    # @param  name      [String]  name of the collection in site:collections
-    # @param  site      [Hash]    site config
-    def initialize(name, site)
-      @name     = name
-      @site     = site
-      @config   = self.config
-    end
+    IMAGE_DERIVATIVE_DIRECTORY = 'img/derivatives'
+    DEFAULT_VARIANTS = { 'thumbnail' => 250, 'fullwidth' => 1140 }.freeze
 
-    # Finds the collection config within the site config
     #
-    # @return [Hash] the config for the collection
-    def config
-      @site[:collections].fetch(@name)
-    rescue StandardError => e
-      raise Error::InvalidCollection, "Cannot load collection config for #{@name}.\n#{e}"
-    end
-
-    # Returns the target directory for generated collection pages
     #
-    # @return [String] path
-    def page_dir
-      WaxTasks::Utils.root_path(@site[:source_dir], @site[:collections_dir], "_#{@name}")
+    def initialize(name, config, source, collections_dir, ext)
+      @name                     = name
+      @config                   = config
+      @page_extension           = ext
+      @site_source              = source
+      @page_source              = Utils.safe_join source, collections_dir, "_#{@name}"
+      @metadata_source          = Utils.safe_join source, '_data', config.dig('metadata', 'source')
+      @imagedata_source         = Utils.safe_join source, '_data', config.dig('images', 'source')
+      @iiif_derivative_source   = Utils.safe_join source, IMAGE_DERIVATIVE_DIRECTORY, 'iiif'
+      @simple_derivative_source = Utils.safe_join source, IMAGE_DERIVATIVE_DIRECTORY, 'simple'
+      @search_fields            = %w[pid label thumbnail permalink collection]
+      @image_variants           = image_variants
     end
 
-    # Constructs the path to the data source file
     #
-    # @return [String] the path to the data source file
-    def metadata_source_path
-      source = @config.dig('metadata', 'source')
-      raise WaxTasks::Error::MissingSource, "Missing collection source in _config.yml for #{@name}" if source.nil?
-      WaxTasks::Utils.root_path(@site[:source_dir], '_data', source)
+    #
+    def image_variants
+      vars = @config.dig('images', 'variants') || {}
+      DEFAULT_VARIANTS.merge vars
     end
 
-    # Ingests the collection source data as an Array of Hashes
     #
-    # @param source [String] the path to the CSV, JSON, or YAML source file
-    # @return [Array] the collection data
-    def ingest_file(source)
-      raise Error::MissingSource, "Cannot find #{source}" unless File.exist? source
-
-      data = case File.extname(source)
-             when '.csv'
-               WaxTasks::Utils.validate_csv(source)
-             when '.json'
-               WaxTasks::Utils.validate_json(source)
-             when /\.ya?ml/
-               WaxTasks::Utils.validate_yaml(source)
-             else
-               raise Error::InvalidSource, "Can't load #{File.extname(source)} files. Culprit: #{source}"
-             end
-
-      WaxTasks::Utils.assert_pids(data)
-      WaxTasks::Utils.assert_unique(data)
+    #
+    def clobber_pages
+      return unless Dir.exist? @page_source
+      puts Rainbow("Removing pages from #{@page_source}").cyan
+      FileUtils.remove_dir @page_source, true
     end
 
-    # @return [Nil]
-    def overwrite_metadata
-      src = self.metadata_source_path
-      puts "Writing image derivative info #{src}.".cyan
-      case File.extname(src)
-      when '.csv'
-        keys = @metadata.map(&:keys).inject(&:|)
-        csv_string = keys.to_csv
-        @metadata.each { |h| csv_string += h.values_at(*keys).to_csv }
-        File.open(src, 'w') { |f| f.write(csv_string) }
-      when '.json'
-        File.open(src, 'w') { |f| f.write(JSON.pretty_generate(@metadata)) }
-      when /\.ya?ml/
-        File.open(src, 'w') { |f| f.write(@metadata.to_yaml) }
-      else
-        raise Error::InvalidSource
+    #
+    #
+    def clobber_derivatives
+      [@iiif_derivative_source, @simple_derivative_source].each do |dir|
+        if Dir.exist? dir
+          puts Rainbow("Removing derivatives from #{dir}").cyan
+          FileUtils.remove_dir dir, true
+        end
       end
     end
   end

data/lib/wax_tasks/collection/images.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require 'mini_magick'
+require 'progress_bar'
+require 'wax_iiif'
+
+#
+module WaxTasks
+  #
+  class Collection
+    #
+    module Images
+      #
+      #
+      def items_from_imagedata
+        raise Error::MissingSource, "Cannot find image data source '#{@imagedata_source}'" unless Dir.exist? @imagedata_source
+
+        pre_process_pdfs
+        records = records_from_metadata
+        Dir.glob(Utils.safe_join(@imagedata_source, '*')).map do |path|
+          item = WaxTasks::Item.new(path, @image_variants)
+          next if item.type == '.pdf'
+          next puts Rainbow("Skipping #{path} because type #{item.type} is not an accepted format").yellow unless item.valid?
+
+          item.record      = records.find { |r| r.pid == item.pid }
+          item.iiif_config = @config.dig 'images', 'iiif'
+          warn Rainbow("\nCould not find record in #{@metadata_source} for image item #{path}.\n").orange if item.record.nil?
+          item
+        end.compact
+      end
+
+      #
+      #
+      def pre_process_pdfs
+        Dir.glob(Utils.safe_join(@imagedata_source, '*.pdf')).each do |path|
+          target_dir = path.gsub '.pdf', ''
+          next unless Dir.glob("#{target_dir}/*").empty?
+
+          puts Rainbow("\nPreprocessing #{path} into image files. This may take a minute.\n").cyan
+
+          opts = { output_dir: File.dirname(target_dir) }
+          WaxIiif::Utilities::PdfSplitter.split(path, opts)
+        end
+      end
+
+      #
+      #
+      def write_simple_derivatives
+        puts Rainbow("Generating simple image derivatives for collection '#{@name}'\nThis might take awhile.").cyan
+
+        bar = ProgressBar.new(items_from_imagedata.length)
+        bar.write
+        items_from_imagedata.map do |item|
+          item.simple_derivatives.each do |d|
+            path = "#{@simple_derivative_source}/#{d.path}"
+            FileUtils.mkdir_p File.dirname(path)
+            next if File.exist? path
+
+            d.img.write path
+            item.record.set d.label, path if item.record?
+          end
+          bar.increment!
+          bar.write
+          item
+        end.flat_map(&:record).compact
+      end
+
+      #
+      #
+      def iiif_builder(dir)
+        build_opts = {
+          base_url: "{{ '/' | absolute_url }}#{dir}",
+          output_dir: dir,
+          collection_label: @name,
+          variants: @image_variants.dup.tap { |h| h.delete 'full' }
+        }
+        WaxIiif::Builder.new build_opts
+      end
+
+      #
+      #
+      def add_font_matter_to_json_files(dir)
+        Dir.glob("#{dir}/**/*.json").each do |f|
+          Utils.add_yaml_front_matter_to_file f
+        end
+      end
+
+      #
+      #
+      def add_iiif_results_to_records(records, manifests)
+        records.map do |record|
+          next nil if record.nil?
+
+          manifest = manifests.find { |m| m.base_id == record.pid }
+          next record if manifest.nil?
+
+          json = JSON.parse manifest.to_json
+          @image_variants.each do |k, _v|
+            value = json.fetch k, ''
+            record.set k, "/#{Utils.content_clean(value)}" unless value.empty?
+          end
+
+          record.set 'manifest', "/#{Utils.content_clean(manifest.id)}"
+          record
+        end.compact
+      end
+
+      #
+      #
+      def write_iiif_derivatives
+        items     = items_from_imagedata
+        iiif_data = items.map(&:iiif_image_records).flatten
+        builder   = iiif_builder @iiif_derivative_source
+
+        builder.load iiif_data
+
+        puts Rainbow("Generating IIIF derivatives for collection '#{@name}'\nThis might take awhile.").cyan
+        builder.process_data
+        records = items.map(&:record).compact
+
+        add_font_matter_to_json_files @iiif_derivative_source
+        add_iiif_results_to_records records, builder.manifests
+      end
+    end
+  end
+end