wax_tasks 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dcf7d468f41464dd970e43b0f4a22b857b1e412872d9fb1a3467ab63d8482b80
-  data.tar.gz: 3d8b8640ec00cbf86ed1c06b39935745abf24001ee858d9e9a7dec2ee6d075f9
+  metadata.gz: a05cb7d5dd4a2218f7c20546a7bb40762ff50c1382457ab5dc8573f5a8131eb9
+  data.tar.gz: d80600e1d481d7aa91c78fc38fb214aa234f86859ff1ae8660310f17586b5ed0
 SHA512:
-  metadata.gz: f2315b406a0d970572b617571e140a29e82a09125dfbc5a700059f21da63b4b9f5e5e9bb4195dcdcdcb9d904d33f7319605c9b8c581a70fc1fb233eb41635e03
-  data.tar.gz: 1742b039101f3f9b7962dbe1685d0900f1da91a5bc50b263c4987bffae0ae5c6532e0ba61b473e90b9c72f687c8a529fa1bf0625df980e51bb3ba975e261b054
+  metadata.gz: 36879c674be8917117b4219e485c987386370c8d8c89d6d8fdc8bc0149a6d05265ede31b4f4720a1aa48d9cca5db1d2e10a8511663f0aaab9ee34a48bbb96edd
+  data.tar.gz: 8b9ba90bd8a415fcacc969bd787862d8d37e1b7f222341bd23a9c9b08c33cd5eb9fd641f31d5a02b0df99707009ffb57969d945d462698df10c45bde58d0e758

data/Gemfile

@@ -1,6 +1,8 @@
 source 'https://rubygems.org'
 gemspec
 
+# dev/test utilities
 gem 'diane', require: false
 gem 'rubocop', require: false
 gem 'simplecov', require: false
+gem 'yard', require: false

data/lib/{wax/tasks → tasks}/iiif.rake

@@ -1,9 +1,11 @@
 require 'wax_tasks'
 
 namespace :wax do
+  desc 'generate iiif derivatives from local image files'
   task :iiif do
     ARGS = ARGV.drop(1).each { |a| task a.to_sym }
     abort "You must specify a collection after 'wax:iiif'" if ARGS.empty?
-    ARGS.each { |name| IiifCollection.new(name).process }
+    task_runner = WaxTasks::TaskRunner.new
+    task_runner.iiif(ARGS)
   end
 end

data/lib/tasks/jspackage.rake

@@ -0,0 +1,14 @@
+require 'wax_tasks'
+
+namespace :wax do
+  desc 'write a simple package.json for monitoring js dependencies'
+  task :jspackage do
+    task_runner = WaxTasks::TaskRunner.new
+    package = task_runner.js_package
+    unless package.empty?
+      path = WaxTasks::Utils.make_path(task_runner.site[:source_dir],
+                                       'package.json')
+      File.open(path, 'w') { |f| f.write(package.to_json) }
+    end
+  end
+end

data/lib/tasks/lunr.rake

@@ -0,0 +1,9 @@
+require 'wax_tasks'
+
+namespace :wax do
+  desc 'build lunr search index (with default UI if UI=true)'
+  task :lunr do
+    task_runner = WaxTasks::TaskRunner.new
+    task_runner.lunr(generate_ui: !!ENV['UI'])
+  end
+end

data/lib/{wax/tasks → tasks}/pagemaster.rake

@@ -4,7 +4,8 @@ namespace :wax do
   desc 'generate collection md pages from yaml or csv data source'
   task :pagemaster do
     ARGS = ARGV.drop(1).each { |a| task a.to_sym }
-    abort "You must specify a collection after 'wax:pagemaster'" if ARGS.empty?
-    ARGS.each { |name| PagemasterCollection.new(name).generate_pages }
+    raise 'You must specify a collection after wax:pagemaster' if ARGS.empty?
+    task_runner = WaxTasks::TaskRunner.new
+    task_runner.pagemaster(ARGS)
   end
 end

data/lib/tasks/push.rake

@@ -0,0 +1,13 @@
+require 'wax_tasks'
+
+namespace :wax do
+  desc 'push compiled Jekyll site to git branch BRANCH'
+  task :push do
+    ARGS = ARGV.drop(1).each { |a| task a.to_sym }
+    raise 'You must specify a branch after \'wax:push:branch\'' if ARGS.empty?
+
+    target = WaxTasks::Utils.slug(ARGS.first)
+    task_runner = WaxTasks::TaskRunner.new
+    task_runner.push_branch(target)
+  end
+end

data/lib/{wax/tasks → tasks}/test.rake

@@ -1,7 +1,7 @@
 require 'html-proofer'
 
 namespace :wax do
-  desc 'run htmlproofer, rspec if exists'
+  desc 'run htmlproofer, rspec if .rspec file exists'
   task :test do
     opts = {
       check_external_hash: true,

data/lib/wax_tasks/branch.rb

@@ -0,0 +1,110 @@
+require 'jekyll'
+require 'logger'
+require 'time'
+require 'tmpdir'
+
+module WaxTasks
+  # Parent class representing a Git Branch
+  # that cannot be created directly. Only child classes
+  # (LocalBranch, TravisBranch) can be initialized.
+  #
+  # @attr target      [String] the name of the Git branch to deploy to
+  # @attr origin      [String] the current repository remote
+  # @attr commit_msg  [String] the commit message to use on push
+  # @attr baseurl     [String] the site baseurl to build with (if on gh-pages)
+  # @attr success_msg [String] informative message to be output to console
+  class Branch
+    attr_reader :target, :origin, :commit_msg, :baseurl, :success
+    private_class_method :new
+
+    # This method ensures child classes can be instantiated eventhough
+    # Branch.new cannot be.
+    def self.inherited(*)
+      public_class_method :new
+    end
+
+    # @param site   [Hash]    the site config from (TaskRunner.site)
+    # @param target [String]  the name of the Git branch to deploy to
+    def initialize(site, target)
+      @site   = site
+      @target = target
+    end
+
+    # Rebuild the Jekyll site with branch @baseurl
+    # @return [Nil]
+    def rebuild
+      if @baseurl.empty?
+        msg = 'Building the gh-pages _site without a baseurl is not recommended'
+        Logger.new($stdout).warn(msg.orange)
+      end
+      FileUtils.rm_r(SITE_DIR) if File.directory?(SITE_DIR)
+      opts = {
+        source: '.',
+        destination: SITE_DIR,
+        baseurl:  @baseurl,
+        verbose: true
+      }
+      Jekyll::Site.new(Jekyll.configuration(opts)).process
+    end
+
+    # Add, commmit, and push compiled Jekyll site to @target branch
+    # @return [Nil]
+    def push
+      if @site[:env] == 'prod'
+        rebuild if @target == 'gh-pages'
+        raise Error::MissingSite, "Cannot find #{SITE_DIR}" unless Dir.exist? SITE_DIR
+        Dir.chdir(SITE_DIR)
+        system 'git init && git add .'
+        system "git commit -m '#{@commit_msg}'"
+        system "git remote add origin #{@origin}"
+        puts @success_msg.cyan
+        system "git push origin master:refs/heads/#{@target} --force"
+      else
+        puts "Skipping build for branch '#{@target}' on env='test'".orange
+      end
+    end
+  end
+
+  # Branch object for `$ wax:push` task when run on Travis-CI VM
+  # using encrypted Travis environment vars
+  #
+  # @attr repo_slug   [String] the 'user/repo_name'
+  # @attr user        [String] the GitHub user making the commit/push
+  # @attr token       [String] secret git access token
+  # @attr commit_msg  [String] the commit message to use on push
+  # @attr origin      [String] the current repository remote
+  # @attr baseurl     [String] the site baseurl to build with (if on gh-pages)
+  # @attr success_msg [String] informative message to be output to console
+  class TravisBranch < Branch
+    def initialize(site, target)
+      super(site, target)
+
+      @repo_slug    = ENV['TRAVIS_REPO_SLUG']
+      @user         = @repo_slug.split('/').first
+      @token        = ENV['ACCESS_TOKEN']
+
+      @commit_msg   = "Updated via #{ENV['TRAVIS_COMMIT']} @#{Time.now.utc}"
+      @origin       = "https://#{@user}:#{@token}@github.com/#{@repo_slug}.git"
+      @baseurl      = @repo_slug.split('/').last
+      @success_msg  = "Deploying to #{@target} branch from Travis as #{@user}."
+    end
+  end
+
+  # Branch object for `$ wax:push` task when run on local machine
+  # using local credentials
+  #
+  # @attr origin      [String] the current repository remote
+  # @attr commit_msg  [String] the commit message to use on push
+  # @attr baseurl     [String] the site baseurl to build with (if on gh-pages)
+  # @attr success_msg [String] informative message to be output to console
+  class LocalBranch < Branch
+    def initialize(site, target)
+      super(site, target)
+
+      @origin       = `git config --get remote.origin.url`.strip
+      @commit_msg   = "Updated via local task at #{Time.now.utc}"
+      @baseurl      = @origin.split('/').last.gsub('.git', '')
+      @success_msg  = "Deploying to #{@target} branch from local task."
+    end
+  end
+end

data/lib/wax_tasks/collection.rb

@@ -0,0 +1,61 @@
+module WaxTasks
+  # Parent class representing a Jekyll collection
+  # that cannot be created directly. Only child classes
+  # (IiifCollection, LunrCollection, PagemasterCollection)
+  # can be initialized.
+  #
+  # @attr config    [Hash]    the collection config within site config
+  # @attr name      [String]  the name of the collection in site:collections
+  # @attr page_dir  [String]  the directory path for generated collection pages
+  # @attr site      [Hash]    the site config
+  class Collection
+    attr_reader :name, :page_dir
+    private_class_method :new
+
+    # This method ensures child classes can be instantiated though
+    # Collection.new cannot be.
+    def self.inherited(*)
+      public_class_method :new
+    end
+
+    # Creates a new collection with name @name given site config @site
+    #
+    # @param name     [String]  the name of the collection in site:collections
+    # @param site     [Hash]    the site config
+    def initialize(name, site)
+      @name     = name
+      @site     = site
+      @config   = collection_config
+      @page_dir = Utils.make_path(@site[:source_dir],
+                                  @site[:collections_dir],
+                                  @name)
+    end
+
+    # Finds the collection config within the site config
+    #
+    # @return [Hash] the config for the collection
+    def collection_config
+      @site[:collections].fetch(@name)
+    rescue StandardError => e
+      raise Error::InvalidCollection, "Cannot load collection config for #{@name}.\n#{e}"
+    end
+
+    # Ingests the collection source data as an Array of Hashes
+    #
+    # @param source [String] the path to the CSV, JSON, or YAML source file
+    # @return [Array] the collection data
+    def ingest_file(source)
+      raise Error::MissingSource, "Cannot find #{source}" unless File.exist? source
+
+      case File.extname(source)
+      when '.csv'     then data = WaxTasks::Utils.validate_csv(source)
+      when '.json'    then data = WaxTasks::Utils.validate_json(source)
+      when /\.ya?ml/  then data = WaxTasks::Utils.validate_yaml(source)
+      else raise Error::InvalidSource, "Cannot load #{File.extname(source)} files. Culprit: #{source}"
+      end
+
+      WaxTasks::Utils.assert_pids(data)
+      WaxTasks::Utils.assert_unique(data)
+    end
+  end
+end

data/lib/wax_tasks/error.rb

@@ -0,0 +1,71 @@
+module WaxTasks
+  # Custom WaxTasks Errors module
+  module Error
+    # Custom WaxTasks Error class with magenta console output
+    class WaxTasksError < StandardError
+      def initialize(msg = '')
+        super(msg.magenta)
+      end
+    end
+
+    # Custom Error:
+    # Site config cannot be found / parsed
+    class InvalidSiteConfig < WaxTasksError; end
+
+    # Custom Error:
+    # Collection specified cannot be found / parsed in site config
+    class InvalidCollection < WaxTasksError; end
+
+    # Custom Error:
+    # Collection data source type is not allowed or is an invalid file
+    class InvalidSource     < WaxTasksError; end
+
+    # Custom Error:
+    # Data source file could not be found
+    class MissingSource     < WaxTasksError; end
+
+    # Custom Error:
+    # While loading markdown pages to index, one could not be read
+    class LunrPageLoad      < WaxTasksError; end
+
+    # Custom Error:
+    # Lunr collection does not have fields specified to index
+    class MissingFields     < WaxTasksError; end
+
+    # Custom Error:
+    # Page layout was not specified for a pagemaster collection
+    class MissingLayout     < WaxTasksError; end
+
+    # Custom Error:
+    # Collection item does not have a required pid value
+    class MissingPid        < WaxTasksError; end
+
+    # Custom Error:
+    # Collection item has a non-unique pud value
+    class NonUniquePid      < WaxTasksError; end
+
+    # Custom Error:
+    # Collection page item could not be generated
+    class PageFailure       < WaxTasksError; end
+
+    # Custom Error:
+    # CSV file failed to lint + could not be loaded
+    class InvalidCSV        < WaxTasksError; end
+
+    # Custom Error:
+    # JSON file failed to lint + could not be loaded
+    class InvalidJSON       < WaxTasksError; end
+
+    # Custom Error:
+    # YAML file failed to lint + could not be loaded
+    class InvalidYAML       < WaxTasksError; end
+
+    # Custom Error:
+    # No collections in site config have lunr_index parameters
+    class NoLunrCollections < WaxTasksError; end
+
+    # Custom Error:
+    # Cannot find _site directory to push to GitHub
+    class MissingSite < WaxTasksError; end
+  end
+end

data/lib/wax_tasks/iiif_collection.rb

@@ -0,0 +1,105 @@
+require 'wax_iiif'
+
+module WaxTasks
+  # A Jekyll collection with IIIF configuration + data
+  #
+  # @attr src_data    [String]  the path to the data source file
+  # @attr iiif_config [Hash]    the iiif configuration for the collection
+  # @attr meta        [Array]   metadata k,v rules
+  # @attr variants    [Hash]    image variants to generate e.g. { med: 650 }
+  # @attr src_dir     [String]  path to existing iiif source images
+  # @attr target_dir  [String]  target path for iiif derivatives
+  class IiifCollection < Collection
+    attr_reader :variants, :meta, :target_dir
+
+    # Creates a new IiifCollection with name @name given site config @site
+    def initialize(name, site)
+      super(name, site)
+
+      @src_data     = @config.fetch('source', nil)
+      @iiif_config  = @config.fetch('iiif', {})
+      @meta         = @iiif_config.fetch('meta', nil)
+      @variants     = validated_variants
+      @src_dir      = Utils.make_path(@site[:source_dir],
+                                      '_data/iiif',
+                                      @name)
+      @target_dir   = Utils.make_path(@site[:source_dir],
+                                      'iiif',
+                                      @name)
+    end
+
+    # Main method for generating iiif derivatives and json
+    # @return [Nil]
+    def process
+      raise Error::MissingIiifSrc, "Cannot find IIIF source directory #{@src_dir}" unless Dir.exist?(@src_dir)
+      FileUtils.mkdir_p(@target_dir, verbose: false)
+      builder = iiif_builder
+      builder.load(iiif_records)
+      builder.process_data(true)
+    end
+
+    # Creates a IiifS3::Builder object from collection config
+    # @return [Object]
+    def iiif_builder
+      build_opts = {
+        base_url: "#{@site[:baseurl]}/iiif/#{@name}",
+        output_dir: @target_dir,
+        verbose: true,
+        variants: @variants
+      }
+      IiifS3::Builder.new(build_opts)
+    end
+
+    # Gets custom image variants from collection config if available
+    # Else returns default variants { med: 600, lg: 1140 } to Builder
+    #
+    # @return [Hash]
+    def validated_variants
+      vars = @iiif_config.fetch('variants', false)
+      if vars.is_a?(Array) && vars.all? { |v| v.is_a?(Integer) }
+        variants = {}
+        vars.each_with_index do |v, i|
+          variants["custom_variant_#{i}".to_sym] = v
+        end
+      else
+        variants = { med: 600, lg: 1140 }
+      end
+      variants
+    end
+
+    # Creates an array of IIIfS3 ImageRecords from the collection config
+    # for the IiifS3 Builder to process
+    #
+    # @return [Array]
+    def iiif_records
+      records = []
+      source_images = Dir["#{@src_dir}/*"].sort!
+      if @meta && @src_data
+        src_path = Utils.make_path(@site[:source_dir], '_data', @src_data)
+        metadata = ingest_file(src_path)
+      else
+        metadata = false
+      end
+      source_images.each { |src_img| records << iiif_record(src_img, metadata) }
+      records
+    end
+
+    # Creates an individual IiifS3 ImageRecord
+    # with metadata from source data file if available
+    #
+    # @param  src_img  [String]  path to the original source image
+    # @param  metadata [Hash]    metadata to add to the item if available
+    # @return [Object]
+    def iiif_record(src_img, metadata)
+      basename = File.basename(src_img, '.*').to_s
+      record_opts = { id: basename, path: src_img, label: basename }
+      if metadata
+        src_item = metadata.find { |i| i['pid'].to_s == basename }
+        @meta.each do |i|
+          record_opts[i.first[0].to_sym] = src_item.fetch(i.first[1], '')
+        end
+      end
+      IiifS3::ImageRecord.new(record_opts)
+    end
+  end
+end

data/lib/wax_tasks/lunr_collection.rb

@@ -0,0 +1,59 @@
+module WaxTasks
+  # A Jekyll collection to be Indexed in a Lunr Index / JSON file
+  # for client-side search.
+  #
+  # @attr index_config  [Hash]    the collection's lunr_index config
+  # @attr content       [Boolean] whether/not page content should be indexed
+  # @attr fields        [Array]   the fields (i.e., keys) that should be indexed
+  # @attr data          [Array]   hash array of data from the ingested pages
+  class LunrCollection < Collection
+    attr_accessor :fields, :data
+
+    # Creates a new LunrCollection with name @name given site config @site
+    def initialize(name, site)
+      super(name, site)
+
+      @index_config = @config['lunr_index']
+      @content      = @index_config.fetch('content', false)
+      @fields       = @index_config.fetch('fields', [])
+      @data         = ingest_pages
+
+      raise Error::MissingFields, "There are no fields for #{@name}.".magenta if @fields.empty?
+    end
+
+    # Finds the @page_dir of markdown pages for the collection and ingests
+    # them as an array of hashes
+    #
+    # @return [Array] array of the loaded markdown pages loaded as hashes
+    def ingest_pages
+      data  = []
+      pages = Dir.glob("#{@page_dir}/*.md")
+      puts "There are no pages in #{@page_dir} to index.".cyan if pages.empty?
+      pages.each do |p|
+        begin
+          data << load_page(p)
+        rescue StandardError => e
+          raise Error::LunrPageLoad, "Cannot load page #{p}\n#{e}"
+        end
+      end
+      data
+    end
+
+    # Reads in a markdown file and converts it to a hash
+    # with the values from @fields.
+    # Adds the content of the file (below the YAML) if @content == true
+    #
+    # @param page [String] the path to a markdown page to load
+    def load_page(page)
+      yaml = YAML.load_file(page)
+      hash = {
+        'link' => "{{'#{yaml.fetch('permalink')}' | relative_url }}",
+        'collection' => @name
+      }
+      hash['content'] = File.read(page).html_strip.remove_diacritics if @content
+      fields = @fields.push('pid').uniq
+      fields.each { |f| hash[f] = yaml[f].normalize }
+      hash
+    end
+  end
+end

data/lib/wax_tasks/lunr_index.rb

@@ -0,0 +1,67 @@
+module WaxTasks
+  # A LunrIndex document that combines data from all collections
+  # in site config that have `lunr_index` parameters.
+  #
+  # @attr collections [Array] a list of LunrCollection objects
+  # @attr fields [Array] shared list of fields to index among LunrCollections
+  class LunrIndex
+    attr_accessor :collections, :fields
+
+    # Creates a new LunrIndex object
+    def initialize(collections)
+      @collections  = collections
+      @fields       = total_fields
+    end
+
+    # @return [Array] shared list of fields to index among LunrCollections
+    def total_fields
+      total_fields = @collections.map(&:fields).reduce([], :concat)
+      total_fields.uniq
+    end
+
+    # @return [String] writes index data as pretty JSON with YAML front-matter
+    def to_s
+      data = @collections.map(&:data).flatten
+      data.each_with_index.map { |d, id| d['lunr_index'] = id }
+      "---\nlayout: none\n---\n#{JSON.pretty_generate(data)}"
+    end
+
+    # Creates a default LunrUI / JS file for displaying the Index
+    #
+    # @return [String]
+    def default_ui
+      <<~HEREDOC
+        ---
+        layout: none
+        ---
+        $.getJSON({{ site.baseurl }}/js/lunr-index.json, function(index_json) {
+          window.index = new elasticlunr.Index;
+          window.store = index_json;
+          index.saveDocument(false);
+          index.setRef('lunr_id');
+          #{@fields.map { |f| "index.addField('#{f}');" }.join("\n")}
+          // add docs
+          for (i in store){
+            index.addDoc(store[i]);
+          }
+          $('input#search').on('keyup', function() {
+            var results_div = $('#results');
+            var query = $(this).val();
+            var results = index.search(query, { boolean: 'AND', expand: true });
+            results_div.empty();
+            if (results.length > 10) {
+              results_div.prepend("<p><small>Displaying 10 of " + results.length + " results.</small></p>");
+            }
+            for (var r in results.slice(0, 9)) {
+              var ref = results[r].ref;
+              var item = store[ref];
+              #{@fields.map { |f| "var #{f} = item.#{f};" }.join("\n")}
+              var result = '<div class="result"><b><a href="' + item.link + '">' + title + '</a></b></p></div>';
+              results_div.append(result);
+            }
+          });
+        });
+      HEREDOC
+    end
+  end
+end

data/lib/wax_tasks/pagemaster_collection.rb

@@ -0,0 +1,67 @@
+module WaxTasks
+  # A Jekyll collection with a data source file that
+  # can generate markdown pages from that data.
+  #
+  # @attr source  [String]  the path to the data source file
+  # @attr layout  [String]  the Jekyll layout to be used by the generated pages
+  # @attr data    [Array]   array of hashes representing the ingested data file
+  # @attr ordered [Boolean] whether/not the order of items should be preserved
+  class PagemasterCollection < Collection
+    attr_reader :source, :layout, :data, :ordered
+
+    # Creates a new PagemasterCollection with name @name given site config @site
+    def initialize(name, site)
+      super(name, site)
+
+      @source   = source_path
+      @layout   = assert_layout
+      @data     = ingest_file(@source)
+      @ordered  = @config.fetch('keep_order', false)
+    end
+
+    # Constructs the path to the data source file
+    #
+    # @return [String] the path to the data source file
+    def source_path
+      raise WaxTasks::Error::MissingSource, "Missing collection source in _config.yml for #{@name}" unless @config.key? 'source'
+      WaxTasks::Utils.make_path(@site[:source_dir], '_data', @config['source'])
+    end
+
+    # Confirms + requires `layout` value in the collection @config
+    #
+    # @return [String] the Jekyll layout to be used by the generated pages
+    def assert_layout
+      raise WaxTasks::Error::MissingLayout, "Missing collection layout in _config.yml for #{@name}" unless @config.key? 'layout'
+      @config['layout']
+    end
+
+    # Writes markdown pages from the ingested data to @page_dir
+    # with layout, permalink, and order info added (if applicable)
+    #
+    # @return [Array] a copy of the pages as hashes, for testing
+    def generate_pages
+      FileUtils.mkdir_p(@page_dir)
+      pages = []
+      @data.each_with_index do |item, idx|
+        page_slug         = item.fetch('pid').to_s.slug
+        path              = "#{@page_dir}/#{page_slug}.md"
+        item['permalink'] = "/#{@name}/#{page_slug}#{@site[:permalink]}"
+        item['layout']    = @layout
+        item['order']     = padded_int(idx, @data.length) if @ordered
+        pages << item
+        next "#{page_slug}.md already exits. Skipping." if File.exist?(path)
+        File.open(path, 'w') { |f| f.write("#{item.to_yaml}---") }
+      end
+      puts "#{@data.length} pages were generated to #{@page_dir} directory.".cyan
+      pages
+    end
+
+    # Constructs the order variable for each page (if the collection
+    # needs to preserve the order of items from the file)
+    #
+    # @return [Integer] the order if the item padded with '0's for sorting
+    def padded_int(idx, max_idx)
+      idx.to_s.rjust(Math.log10(max_idx).to_i + 1, '0')
+    end
+  end
+end