wax_tasks 1.1.4 → 1.1.5

data/lib/tasks/derivatives_simple.rake

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'wax_tasks'
+
+namespace :wax do
+  namespace :derivatives do
+    desc 'generate simple derivatives from local image files'
+    task :simple do
+      args = ARGV.drop(1).each { |a| task a.to_sym }
+      args.reject! { |a| a.start_with? '-' }
+      raise WaxTasks::Error::MissingArguments, Rainbow("You must specify a collection after 'wax:derivatives:simple'").magenta if args.empty?
+
+      site = WaxTasks::Site.new
+      args.each { |a| site.generate_derivatives(a, 'simple') }
+    end
+  end
+end

data/lib/tasks/pages.rake

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'wax_tasks'
+
+namespace :wax do
+  desc 'generate collection md pages from yaml or csv data source'
+  task :pages do
+    args = ARGV.drop(1).each { |a| task a.to_sym }
+    args.reject! { |a| a.start_with? '-' }
+    raise WaxTasks::Error::MissingArguments, Rainbow('You must specify a collection after wax:pages').magenta if args.empty?
+
+    site = WaxTasks::Site.new
+    args.each { |a| site.generate_pages a }
+  end
+
+  # alias :pagemaster to wax:pages for backwards compatibility
+  task :pagemaster do
+    t = Rake::Task['wax:pages']
+    desc t.full_comment if t.full_comment
+    args = ARGV.drop(1).each { |a| task a.to_sym }
+    t.invoke(*args)
+  end
+end

data/lib/tasks/search.rake

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'wax_tasks'
+
+namespace :wax do
+  desc 'build lunr search index (with default UI if UI=true)'
+  task :search do
+    args = ARGV.drop(1).each { |a| task a.to_sym }
+    args.reject! { |a| a.start_with? '-' }
+    raise WaxTasks::Error::MissingArguments, Rainbow('You must specify a collection after wax:search').magenta if args.empty?
+
+    site = WaxTasks::Site.new
+    args.each { |a| site.generate_static_search a }
+  end
+
+  # alias lunr to search for backwards compatibility
+  task :lunr do
+    t = Rake::Task['wax:search']
+    desc t.full_comment if t.full_comment
+    args = ARGV.drop(1).each { |a| task a.to_sym }
+    t.invoke(*args)
+  end
+end

data/lib/wax_tasks.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# rubygems
+require 'rubygems'
+
+# stdlib
+require 'csv'
+require 'fileutils'
+require 'json'
+
+# 3rd party
+require 'rainbow'
+require 'safe_yaml'
+
+# relative
+require_relative 'wax_tasks/asset'
+require_relative 'wax_tasks/collection'
+require_relative 'wax_tasks/config'
+require_relative 'wax_tasks/error'
+require_relative 'wax_tasks/index'
+require_relative 'wax_tasks/item'
+require_relative 'wax_tasks/record'
+require_relative 'wax_tasks/site'
+require_relative 'wax_tasks/utils'
+require_relative 'wax_tasks/version'
+
+#
+module WaxTasks
+  DEFAULT_CONFIG_FILE = './_config.yml'
+  #
+  #
+  def self.config_from_file(file = nil)
+    Utils.validate_yaml(file || DEFAULT_CONFIG_FILE)
+  rescue StandardError => e
+    raise WaxTasks::Error::InvalidConfig, "Cannot open config file '#{file}'.\n #{e}"
+  end
+end

data/lib/wax_tasks/asset.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+#
+module WaxTasks
+  Derivative = Struct.new(:path, :label, :img)
+
+  #
+  class Asset
+    attr_reader :id, :path
+
+    def initialize(path, pid, variants)
+      @path     = path
+      @pid      = pid
+      @id       = asset_id
+      @variants = variants
+    end
+
+    #
+    #
+    def asset_id
+      id = File.basename @path, '.*'
+      id.prepend "#{@pid}_" unless id == @pid
+      id
+    end
+
+    #
+    #
+    def simple_derivatives
+      @variants.map do |label, width|
+        img = MiniMagick::Image.open @path
+        if width > img.width
+          warn Rainbow("Tried to create derivative #{width}px wide, but asset #{@id} for item #{@pid} only has a width of #{img.width}px.").yellow
+        else
+          img.resize width
+        end
+
+        img.format 'jpg'
+        Derivative.new("#{@id}/#{label}.jpg", label, img)
+      end
+    end
+
+    #
+    #
+    def to_iiif_image_record(is_only, index, base_opts)
+      opts = base_opts.clone
+
+      opts[:is_primary]    = index.zero?
+      opts[:section_label] = "Page #{index + 1}" unless is_only
+      opts[:path]          = @path
+      opts[:manifest_id]   = @pid
+      opts[:id]            = @id
+      opts[:variants]      = @variants
+
+      WaxIiif::ImageRecord.new(opts)
+    end
+  end
+end

data/lib/wax_tasks/collection.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require_relative 'collection/images'
+require_relative 'collection/metadata'
+
+module WaxTasks
+  #
+  class Collection
+    attr_reader :name, :config, :ext, :search_fields,
+                :page_source, :metadata_source, :imagedata_source,
+                :iiif_derivative_source, :simple_derivative_source
+
+    include Collection::Metadata
+    include Collection::Images
+
+    IMAGE_DERIVATIVE_DIRECTORY = 'img/derivatives'
+    DEFAULT_VARIANTS = { 'thumbnail' => 250, 'fullwidth' => 1140 }.freeze
+
+    #
+    #
+    def initialize(name, config, source, collections_dir, ext)
+      @name                     = name
+      @config                   = config
+      @page_extension           = ext
+      @site_source              = source
+      @page_source              = Utils.safe_join source, collections_dir, "_#{@name}"
+      @metadata_source          = Utils.safe_join source, '_data', config.dig('metadata', 'source')
+      @imagedata_source         = Utils.safe_join source, '_data', config.dig('images', 'source')
+      @iiif_derivative_source   = Utils.safe_join source, IMAGE_DERIVATIVE_DIRECTORY, 'iiif'
+      @simple_derivative_source = Utils.safe_join source, IMAGE_DERIVATIVE_DIRECTORY, 'simple'
+      @search_fields            = %w[pid label thumbnail permalink collection]
+      @image_variants           = image_variants
+    end
+
+    #
+    #
+    def image_variants
+      vars = @config.dig('images', 'variants') || {}
+      DEFAULT_VARIANTS.merge vars
+    end
+
+    #
+    #
+    def clobber_pages
+      return unless Dir.exist? @page_source
+      puts Rainbow("Removing pages from #{@page_source}").cyan
+      FileUtils.remove_dir @page_source, true
+    end
+
+    #
+    #
+    def clobber_derivatives
+      [@iiif_derivative_source, @simple_derivative_source].each do |dir|
+        if Dir.exist? dir
+          puts Rainbow("Removing derivatives from #{dir}").cyan
+          FileUtils.remove_dir dir, true
+        end
+      end
+    end
+  end
+end

data/lib/wax_tasks/collection/images.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require 'mini_magick'
+require 'progress_bar'
+require 'wax_iiif'
+
+#
+module WaxTasks
+  #
+  class Collection
+    #
+    module Images
+      #
+      #
+      def items_from_imagedata
+        raise Error::MissingSource, "Cannot find image data source '#{@imagedata_source}'" unless Dir.exist? @imagedata_source
+
+        pre_process_pdfs
+        records = records_from_metadata
+        Dir.glob(Utils.safe_join(@imagedata_source, '*')).map do |path|
+          item = WaxTasks::Item.new(path, @image_variants)
+          next if item.type == '.pdf'
+          next puts Rainbow("Skipping #{path} because type #{item.type} is not an accepted format").yellow unless item.valid?
+
+          item.record      = records.find { |r| r.pid == item.pid }
+          item.iiif_config = @config.dig 'images', 'iiif'
+          warn Rainbow("\nCould not find record in #{@metadata_source} for image item #{path}.\n").orange if item.record.nil?
+          item
+        end.compact
+      end
+
+      #
+      #
+      def pre_process_pdfs
+        Dir.glob(Utils.safe_join(@imagedata_source, '*.pdf')).each do |path|
+          target_dir = path.gsub '.pdf', ''
+          next unless Dir.glob("#{target_dir}/*").empty?
+
+          puts Rainbow("\nPreprocessing #{path} into image files. This may take a minute.\n").cyan
+
+          opts = { output_dir: File.dirname(target_dir) }
+          WaxIiif::Utilities::PdfSplitter.split(path, opts)
+        end
+      end
+
+      #
+      #
+      def write_simple_derivatives
+        puts Rainbow("Generating simple image derivatives for collection '#{@name}'\nThis might take awhile.").cyan
+
+        bar = ProgressBar.new(items_from_imagedata.length)
+        bar.write
+        items_from_imagedata.map do |item|
+          item.simple_derivatives.each do |d|
+            path = "#{@simple_derivative_source}/#{d.path}"
+            FileUtils.mkdir_p File.dirname(path)
+            next if File.exist? path
+
+            d.img.write path
+            item.record.set d.label, "/#{path}" if item.record?
+          end
+          bar.increment!
+          bar.write
+          item
+        end.flat_map(&:record).compact
+      end
+
+      #
+      #
+      def iiif_builder(dir)
+        build_opts = {
+          base_url: "{{ '/' | absolute_url }}#{dir}",
+          output_dir: dir,
+          collection_label: @name,
+          variants: @image_variants.dup.tap { |h| h.delete 'full' }
+        }
+        WaxIiif::Builder.new build_opts
+      end
+
+      #
+      #
+      def add_font_matter_to_json_files(dir)
+        Dir.glob("#{dir}/**/*.json").each do |f|
+          Utils.add_yaml_front_matter_to_file f
+        end
+      end
+
+      #
+      #
+      def add_iiif_results_to_records(records, manifests)
+        records.map do |record|
+          next nil if record.nil?
+
+          manifest = manifests.find { |m| m.base_id == record.pid }
+          next record if manifest.nil?
+
+          json = JSON.parse manifest.to_json
+          @image_variants.each do |k, _v|
+            value = json.fetch k, ''
+            record.set k, "/#{Utils.content_clean(value)}" unless value.empty?
+          end
+
+          record.set 'manifest', "/#{Utils.content_clean(manifest.id)}"
+          record
+        end.compact
+      end
+
+      #
+      #
+      def write_iiif_derivatives
+        items     = items_from_imagedata
+        iiif_data = items.map(&:iiif_image_records).flatten
+        builder   = iiif_builder @iiif_derivative_source
+
+        builder.load iiif_data
+
+        puts Rainbow("Generating IIIF derivatives for collection '#{@name}'\nThis might take awhile.").cyan
+        builder.process_data
+        records = items.map(&:record).compact
+
+        add_font_matter_to_json_files @iiif_derivative_source
+        add_iiif_results_to_records records, builder.manifests
+      end
+    end
+  end
+end

data/lib/wax_tasks/collection/metadata.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+#
+module WaxTasks
+  #
+  class Collection
+    #
+    module Metadata
+      #
+      #
+      def search_fields=(fields)
+        @search_fields.concat(fields).flatten.compact.uniq
+      end
+
+      #
+      #
+      def records_from_pages
+        paths = Dir.glob("#{@page_source}/*.{md, markdown}")
+        warn Rainbow("There are no pages in #{@page_source} to index.").orange if paths.empty?
+
+        paths.map do |path|
+          begin
+            content = WaxTasks::Utils.content_clean File.read(path)
+            Record.new(SafeYAML.load_file(path)).tap do |r|
+              r.set 'content', content
+              r.set 'permalink', "/#{@name}/#{r.pid}#{@ext}" unless r.permalink?
+            end
+          rescue StandardError => e
+            raise Error::PageLoad, "Cannot load page #{path}\n#{e}"
+          end
+        end
+      end
+
+      #
+      #
+      def records_from_metadata
+        raise Error::MissingSource, "Cannot find metadata source '#{@metadata_source}'" unless File.exist? @metadata_source
+
+        metadata = Utils.ingest @metadata_source
+        metadata.each_with_index.map do |meta, i|
+          Record.new(meta).tap do |r|
+            r.set 'order', Utils.padded_int(i, metadata.length) unless r.order?
+            r.set 'layout', @config['layout'] if @config.key? 'layout'
+            r.set 'collection', @name
+          end
+        end
+      end
+
+      #
+      #
+      def update_metadata(update)
+        records = consolidate_records records_from_metadata, update
+        reformatted = case File.extname @metadata_source
+                      when '.csv'
+                        csv_string records
+                      when '.json'
+                        json_string records
+                      when /\.ya?ml/
+                        yaml_string records
+                      end
+        File.open(@metadata_source, 'w') { |f| f.puts reformatted }
+      end
+
+      #
+      #
+      def consolidate_records(original, new)
+        lost_record_pids = original.map(&:pid) - new.map(&:pid)
+        lost_record_pids.each do |pid|
+          new << original.find { |r| r.pid == pid }
+        end
+        new.sort_by(&:order)
+      end
+
+      #
+      #
+      def csv_string(records)
+        keys = records.flat_map(&:keys).uniq
+        CSV.generate do |csv|
+          csv << keys
+          records.each do |r|
+            csv << keys.map { |k| r.hash.fetch(k, '') }
+          end
+        end
+      end
+
+      #
+      #
+      def json_string(records)
+        hashes = records.map(&:hash)
+        JSON.pretty_generate hashes
+      end
+
+      #
+      #
+      def yaml_string(records)
+        hashes = records.map(&:hash)
+        hashes.to_yaml
+      end
+    end
+  end
+end