wax_tasks 1.0.0.pre.beta → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6df6c21281405044f2958da14b8eda608a4a265337f684c1371612817468767a
-  data.tar.gz: f1a7248a2b83fa1cf34cfd15580baecdefdd91601c7328b987d9ac247316e154
+  metadata.gz: 1f09ee093e87d8111dc4f0ee522cb33a6f3298fdd82a8f80c2c48d4f2c24baa5
+  data.tar.gz: 12bb3dc4a77e92ebbec52f8b9900a8e6d2349592fc5fdff70a875111e11cc76e
 SHA512:
-  metadata.gz: 2a2c5464d13a57208f0c79bbf44a4355b6fad5efd3d2e511ad30ce7a0b4075321868a906a70b680b76ced742d5394fb758a746aff42f0601ea9137ad774dd448
-  data.tar.gz: 87e84935bcda6fcc92bab5a5fecc7b8fec29d19e3348b6b1f0404dd3b6e47595b778a1b7a25562e0ce7b5c187c700be7c9c20a728bc7adbf98284bba26e7b691
+  metadata.gz: 53bee380df63833cf919d3fa2b890617c7dea35bc350dd31fd1cbfd331032a17edf479565b841168f1db65a0ddae29fa14ab32870677ef9a1b3f33d9a0d5e27a
+  data.tar.gz: dcd8227c7274a19473a62ea9f5507e813ca70985f1fca21cb1d9955b0e6d300225165b81028fa8aeae07c186da84f8b2c6abb21cd5273cf18a1cd1c01084185f

data/Gemfile

@@ -1,8 +1,9 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 gemspec
 
 # dev/test utilities
-gem 'diane', require: false
-gem 'rubocop', '0.59.0', require: false
+gem 'rubocop', require: false
 gem 'simplecov', require: false
 gem 'yard', require: false

data/lib/tasks/derivatives_iiif.rake

@@ -1,13 +1,16 @@
+# frozen_string_literal: true
+
 require 'wax_tasks'
 
 namespace :wax do
   namespace :derivatives do
     desc 'generate iiif derivatives from local image files'
     task :iiif do
-      arguments = ARGV.drop(1).each { |a| task a.to_sym }
-      raise WaxTasks::Error::MissingArguments, "You must specify a collection after 'wax:derivatives:iiif'" if arguments.empty?
-      task_runner = WaxTasks::TaskRunner.new
-      task_runner.derivatives_iiif(arguments)
+      args = ARGV.drop(1).each { |a| task a.to_sym }
+      raise WaxTasks::Error::MissingArguments, Rainbow("You must specify a collection after 'wax:derivatives:iiif'").magenta if args.empty?
+
+      site = WaxTasks::Site.new
+      args.each { |a| site.generate_derivatives(a, 'iiif') }
     end
   end
 

data/lib/tasks/derivatives_simple.rake

@@ -1,13 +1,16 @@
+# frozen_string_literal: true
+
 require 'wax_tasks'
 
 namespace :wax do
   namespace :derivatives do
     desc 'generate iiif derivatives from local image files'
     task :simple do
-      arguments = ARGV.drop(1).each { |a| task a.to_sym }
-      raise WaxTasks::Error::MissingArguments, "You must specify a collection after 'wax:derivatives:simple'" if arguments.empty?
-      task_runner = WaxTasks::TaskRunner.new
-      task_runner.derivatives_simple(arguments)
+      args = ARGV.drop(1).each { |a| task a.to_sym }
+      raise WaxTasks::Error::MissingArguments, Rainbow("You must specify a collection after 'wax:derivatives:simple'").magenta if args.empty?
+
+      site = WaxTasks::Site.new
+      args.each { |a| site.generate_derivatives(a, 'simple') }
     end
   end
 end

data/lib/tasks/pages.rake

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'wax_tasks'
+
+namespace :wax do
+  desc 'generate collection md pages from yaml or csv data source'
+  task :pages do
+    args = ARGV.drop(1).each { |a| task a.to_sym }
+    raise WaxTasks::Error::MissingArguments, Rainbow('You must specify a collection after wax:pages').magenta if args.empty?
+
+    site = WaxTasks::Site.new
+    args.each { |a| site.generate_pages a }
+  end
+
+  # alias :pagemaster to wax:pages for backwards compatibility
+  task :pagemaster do
+    t = Rake::Task['wax:pages']
+    desc t.full_comment if t.full_comment
+    args = ARGV.drop(1).each { |a| task a.to_sym }
+    t.invoke(*args)
+  end
+end

data/lib/tasks/search.rake

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'wax_tasks'
+
+namespace :wax do
+  desc 'build lunr search index (with default UI if UI=true)'
+  task :search do
+    args = ARGV.drop(1).each { |a| task a.to_sym }
+    raise WaxTasks::Error::MissingArguments, Rainbow('You must specify a collection after wax:search').magenta if args.empty?
+
+    site = WaxTasks::Site.new
+    args.each { |a| site.generate_static_search a }
+  end
+
+  # alias lunr to search for backwards compatibility
+  task :lunr do
+    t = Rake::Task['wax:search']
+    desc t.full_comment if t.full_comment
+    args = ARGV.drop(1).each { |a| task a.to_sym }
+    t.invoke(*args)
+  end
+end

data/lib/wax_tasks.rb

@@ -1,43 +1,36 @@
-require_relative 'wax_tasks/branch'
+# frozen_string_literal: true
+
+# rubygems
+require 'rubygems'
+
+# stdlib
+require 'csv'
+require 'fileutils'
+require 'json'
+
+# 3rd party
+require 'rainbow'
+require 'safe_yaml'
+
+# relative
+require_relative 'wax_tasks/asset'
 require_relative 'wax_tasks/collection'
+require_relative 'wax_tasks/config'
 require_relative 'wax_tasks/error'
-require_relative 'wax_tasks/image_collection'
-require_relative 'wax_tasks/local_branch'
-require_relative 'wax_tasks/lunr/index'
-require_relative 'wax_tasks/pagemaster_collection'
-require_relative 'wax_tasks/task_runner'
-require_relative 'wax_tasks/travis_branch'
+require_relative 'wax_tasks/index'
+require_relative 'wax_tasks/item'
+require_relative 'wax_tasks/record'
+require_relative 'wax_tasks/site'
 require_relative 'wax_tasks/utils'
 
-# The WaxTasks module powers the Rake tasks in `./tasks`, including:
-#
-# wax:pagemaster          :: generate collection md pages from csv, json, or yaml file
-# wax:lunr                :: build lunr search index (with default UI if UI=true)
-# wax:derivatives:simple  :: generate simple image derivatives from local image files
-# wax:derivatves:iiif     :: generate iiif derivatives from local image files
-# wax:jspackage           :: write a simple package.json for monitoring js dependencies
-# wax:push                :: push compiled Jekyll site to git branch
-# wax:test                :: run htmlproofer, rspec if .rspec file exists
 #
-# Tasks are run by a WaxTasks::TaskRunner object which is resposible
-# for reading in site config from `_config.yml`
 module WaxTasks
-  # ----------
-  # CONSTANTS
-  # ----------
-
-  # @return [String] The path to load Jekyll site config
-  DEFAULT_CONFIG          = '_config.yml'.freeze
-
-  # @return [String] The path to write default LunrUI
-  LUNR_UI_PATH            = 'js/lunr-ui.js'.freeze
-
-  # @return [String] The path to the compiled Jekyll site
-  SITE_DIR                = '_site'.freeze
-
-  # @return [String] Default image variant/derivative widths to generate
-  DEFAULT_IMAGE_VARIANTS  = { thumbnail: 250, full: 1140 }.freeze
-
-  # @return [String] The path where image derivatives should be generated
-  DEFAULT_DERIVATIVE_DIR  = 'img/derivatives'.freeze
+  DEFAULT_CONFIG_FILE = './_config.yml'
+  #
+  #
+  def self.config_from_file(file = nil)
+    Utils.validate_yaml(file || DEFAULT_CONFIG_FILE)
+  rescue StandardError => e
+    raise WaxTasks::Error::InvalidConfig, "Cannot open config file '#{file}'.\n #{e}"
+  end
 end

data/lib/wax_tasks/asset.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+#
+module WaxTasks
+  Derivative = Struct.new(:path, :label, :img)
+  attr_reader :id, :path
+
+  #
+  class Asset
+    def initialize(path, pid, variants)
+      @path     = path
+      @pid      = pid
+      @id       = asset_id
+      @variants = variants
+    end
+
+    #
+    #
+    def asset_id
+      id = File.basename(@path, '.*')
+      id.prepend "#{@pid}_" unless id == @pid
+      id
+    end
+
+    #
+    #
+    def simple_derivatives
+      @variants.map do |label, width|
+        img = MiniMagick::Image.open(@path)
+        raise WaxTasks::Error::InvalidConfig, "Requested variant width '#{width}' is larger than original image width." if width > img.width
+
+        img.resize width
+        img.format 'jpg'
+        Derivative.new("#{@id}/#{label}.jpg", label, img)
+      end
+    end
+
+    #
+    #
+    def to_iiif_image_record(is_only, index, base_opts)
+      opts = base_opts.clone
+      opts[:is_primary]    = index.zero?
+      opts[:section_label] = "Page #{index + 1}" unless is_only
+      opts[:path]          = @path
+      opts[:manifest_id]   = @pid
+      opts[:id]            = @id
+
+      WaxIiif::ImageRecord.new(opts)
+    end
+  end
+end

data/lib/wax_tasks/collection.rb

@@ -1,92 +1,29 @@
-module WaxTasks
-  # Parent class representing a Jekyll collection
-  # that cannot be created directly. Only child classes
-  # (IiifCollection, LunrCollection, PagemasterCollection)
-  # can be initialized.
-  class Collection
-    attr_accessor :name, :site
-    private_class_method :new
+# frozen_string_literal: true
 
-    # This method ensures child classes can be instantiated though
-    # Collection.new cannot be.
-    def self.inherited(*)
-      public_class_method :new
-    end
+require_relative 'collection/images'
+require_relative 'collection/metadata'
 
-    # Creates a new collection with name @name given site configuration @site
-    #
-    # @param  name      [String]  name of the collection in site:collections
-    # @param  site      [Hash]    site config
-    def initialize(name, site)
-      @name     = name
-      @site     = site
-      @config   = self.config
-    end
-
-    # Finds the collection config within the site config
-    #
-    # @return [Hash] the config for the collection
-    def config
-      @site[:collections].fetch(@name)
-    rescue StandardError => e
-      raise Error::InvalidCollection, "Cannot load collection config for #{@name}.\n#{e}"
-    end
+module WaxTasks
+  #
+  class Collection
+    attr_reader :name, :config, :ext, :search_fields,
+                :page_source, :metadata_source, :imagedata_source
 
-    # Returns the target directory for generated collection pages
-    #
-    # @return [String] path
-    def page_dir
-      WaxTasks::Utils.root_path(@site[:source_dir], @site[:collections_dir], "_#{@name}")
-    end
+    include Collection::Metadata
+    include Collection::Images
 
-    # Constructs the path to the data source file
     #
-    # @return [String] the path to the data source file
-    def metadata_source_path
-      source = @config.dig('metadata', 'source')
-      raise WaxTasks::Error::MissingSource, "Missing collection source in _config.yml for #{@name}" if source.nil?
-      WaxTasks::Utils.root_path(@site[:source_dir], '_data', source)
-    end
-
-    # Ingests the collection source data as an Array of Hashes
     #
-    # @param source [String] the path to the CSV, JSON, or YAML source file
-    # @return [Array] the collection data
-    def ingest_file(source)
-      raise Error::MissingSource, "Cannot find #{source}" unless File.exist? source
-
-      data = case File.extname(source)
-             when '.csv'
-               WaxTasks::Utils.validate_csv(source)
-             when '.json'
-               WaxTasks::Utils.validate_json(source)
-             when /\.ya?ml/
-               WaxTasks::Utils.validate_yaml(source)
-             else
-               raise Error::InvalidSource, "Can't load #{File.extname(source)} files. Culprit: #{source}"
-             end
-
-      WaxTasks::Utils.assert_pids(data)
-      WaxTasks::Utils.assert_unique(data)
-    end
-
-    # @return [Nil]
-    def overwrite_metadata
-      src = self.metadata_source_path
-      puts "Writing image derivative info #{src}.".cyan
-      case File.extname(src)
-      when '.csv'
-        keys = @metadata.map(&:keys).inject(&:|)
-        csv_string = keys.to_csv
-        @metadata.each { |h| csv_string += h.values_at(*keys).to_csv }
-        File.open(src, 'w') { |f| f.write(csv_string) }
-      when '.json'
-        File.open(src, 'w') { |f| f.write(JSON.pretty_generate(@metadata)) }
-      when /\.ya?ml/
-        File.open(src, 'w') { |f| f.write(@metadata.to_yaml) }
-      else
-        raise Error::InvalidSource
-      end
+    def initialize(name, config, source, collections_dir, ext)
+      @name             = name
+      @config           = config
+      @page_extension   = ext
+      @site_source      = source
+      @page_source      = Utils.safe_join source, collections_dir, "_#{name}"
+      @metadata_source  = Utils.safe_join source, '_data', config.dig('metadata', 'source')
+      @imagedata_source = Utils.safe_join source, '_data', config.dig('images', 'source')
+      @search_fields    = %w[pid label thumbnail permalink collection]
+      @image_variants   = image_variants
     end
   end
 end

data/lib/wax_tasks/collection/images.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require 'mini_magick'
+require 'progress_bar'
+require 'progress_bar/core_ext/enumerable_with_progress'
+require 'wax_iiif'
+
+#
+module WaxTasks
+  #
+  class Collection
+    #
+    module Images
+      #
+      #
+      def image_variants
+        default_variants = { 'thumbnail' => 250, 'full' => 1140 }
+        custom_variants  = @config.dig('images', 'variants') || {}
+        default_variants.merge custom_variants
+      end
+
+      #
+      #
+      def items_from_imagedata
+        raise Error::MissingSource, "Cannot find image data source '#{@imagedata_source}'" unless Dir.exist? @imagedata_source
+
+        pre_process_pdfs
+        records = records_from_metadata
+        Dir.glob(Utils.safe_join(@imagedata_source, '*')).map do |path|
+          item = WaxTasks::Item.new(path, @image_variants)
+          next unless item.valid?
+
+          item.record      = records.find { |r| r.pid == item.pid }
+          item.iiif_config = @config.dig 'images', 'iiif'
+          warn Rainbow("\nWarning:\nCould not find record in #{@metadata_source} for image item #{path}.\n").orange if item.record.nil?
+          item
+        end.compact
+      end
+
+      #
+      #
+      def pre_process_pdfs
+        Dir.glob(Utils.safe_join(@imagedata_source, '*.pdf')).each do |path|
+          target_dir = path.gsub '.pdf', ''
+          next unless Dir.glob("#{target_dir}/*").empty?
+
+          puts Rainbow("\nPreprocessing #{path} into image files. This may take a minute.\n").cyan
+          opts = { output_dir: File.dirname(target_dir) }
+          WaxIiif::Utilities::PdfSplitter.split(path, opts)
+        end
+      end
+
+      #
+      #
+      def write_simple_derivatives(dir)
+        puts Rainbow("Generating simple image derivatives for collection '#{@name}'\nThis might take awhile.").cyan
+        items_from_imagedata.each_with_progress.map do |item|
+          item.simple_derivatives.each do |d|
+            path = "#{dir}/#{d.path}"
+            FileUtils.mkdir_p File.dirname(path)
+            next if File.exist? path
+
+            d.img.write path
+            item.record.set d.label, path if item.record?
+          end
+          item
+        end.flat_map(&:record)
+      end
+
+      #
+      #
+      def iiif_builder(dir)
+        build_opts = {
+          base_url: "{{ '/' | absolute_url }}#{dir}",
+          output_dir: dir,
+          # variants: @image_variants,
+          collection_label: @name
+        }
+        WaxIiif::Builder.new(build_opts)
+      end
+
+      #
+      #
+      def add_font_matter_to_json_files(dir)
+        Dir.glob("#{dir}/**/*.json").each do |f|
+          Utils.add_yaml_front_matter_to_file f
+        end
+      end
+
+      #
+      #
+      def add_iiif_results_to_records(records, manifests)
+        records.map do |record|
+          next nil if record.nil?
+
+          manifest = manifests.find { |m| m.base_id == record.pid }
+          next record if manifest.nil?
+
+          json = JSON.parse manifest.to_json
+          @image_variants.each do |k, _v|
+            value = json.dig k
+            record.set k, "/#{Utils.content_clean(value)}" unless value.nil?
+          end
+
+          record.set 'manifest', "/#{Utils.content_clean(manifest.id)}"
+          record
+        end.compact
+      end
+
+      #
+      #
+      def write_iiif_derivatives(dir)
+        items     = items_from_imagedata
+        iiif_data = items.map(&:iiif_image_records).flatten
+        builder   = iiif_builder(dir)
+
+        builder.load iiif_data
+
+        puts Rainbow("Generating IIIF derivatives for collection '#{@name}'\nThis might take awhile.").cyan
+        builder.process_data
+
+        add_font_matter_to_json_files dir
+        add_iiif_results_to_records items.map(&:record), builder.manifests
+      end
+    end
+  end
+end