bulkrax 1.0.0

data/lib/bulkrax/engine.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'oai'
+
+module Bulkrax
+  class Engine < ::Rails::Engine
+    isolate_namespace Bulkrax
+    initializer :append_migrations do |app|
+      if !app.root.to_s.match(root.to_s) && app.root.join('db/migrate').children.none? { |path| path.fnmatch?("*.bulkrax.rb") }
+        config.paths["db/migrate"].expanded.each do |expanded_path|
+          app.config.paths["db/migrate"] << expanded_path
+        end
+      end
+    end
+
+    config.generators do |g|
+      g.test_framework :rspec
+      begin
+        g.fixture_replacement :factory_bot, dir: 'spec/factories'
+      rescue
+        nil
+      end
+    end
+
+    config.after_initialize do
+      my_engine_root = Bulkrax::Engine.root.to_s
+      paths = ActionController::Base.view_paths.collect(&:to_s)
+      hyrax_path = paths.detect { |path| path.match('/hyrax-') }
+      paths = if hyrax_path
+                paths.insert(paths.index(hyrax_path), my_engine_root + '/app/views')
+              else
+                paths.insert(0, my_engine_root + '/app/views')
+              end
+      ActionController::Base.view_paths = paths
+    end
+  end
+end

data/lib/bulkrax/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  VERSION = '1.0.0'
+end

data/lib/generators/bulkrax/install_generator.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+class Bulkrax::InstallGenerator < Rails::Generators::Base
+  source_root File.expand_path('../templates', __FILE__)
+
+  desc 'This generator installs Bulkrax.'
+
+  def banner
+    say_status("info", "Generating Bulkrax installation", :blue)
+  end
+
+  def add_to_gemfile
+    gem 'willow_sword', github: 'notch8/willow_sword'
+
+    Bundler.with_clean_env do
+      run "bundle install"
+    end
+  end
+
+  def mount_route
+    route "mount Bulkrax::Engine, at: '/'"
+  end
+
+  def create_config
+    copy_file 'config/initializers/bulkrax.rb', 'config/initializers/bulkrax.rb'
+
+    hyrax = "\n# set bulkrax default work type to first curation_concern if it isn't already set\nif Bulkrax.default_work_type.blank?\n  Bulkrax.default_work_type = Hyrax.config.curation_concerns.first.to_s\nend\n"
+
+    return if File.read('config/initializers/hyrax.rb').include?(hyrax)
+    append_to_file 'config/initializers/hyrax.rb' do
+      hyrax
+    end
+  end
+
+  def create_bulkrax_api
+    copy_file 'config/bulkrax_api.yml', 'config/bulkrax_api.yml'
+  end
+
+  def create_cmd_script
+    copy_file 'bin/importer', 'bin/importer'
+  end
+
+  def create_local_processing
+    copy_file 'app/models/concerns/bulkrax/has_local_processing.rb', 'app/models/concerns/bulkrax/has_local_processing.rb'
+  end
+
+  def add_javascripts
+    file = 'app/assets/javascripts/application.js'
+    file_text = File.read(file)
+    js = '//= require bulkrax/application'
+
+    return if file_text.include?(js)
+    insert_into_file file, before: /\/\/= require_tree ./ do
+      "#{js}\n"
+    end
+  end
+
+  def add_css
+    ['css', 'scss', 'sass'].map do |ext|
+      file = "app/assets/stylesheets/application.#{ext}"
+      next unless File.exist?(file)
+
+      file_text = File.read(file)
+      css = "*= require 'bulkrax/application'"
+      next if file_text.include?(css)
+
+      insert_into_file file, before: /\s\*= require_self/ do
+        "\s#{css}\n"
+      end
+    end
+  end
+
+  def display_readme
+    readme 'README'
+  end
+
+  def add_removed_image
+    copy_file 'app/assets/images/bulkrax/removed.png', 'app/assets/images/bulkrax/removed.png'
+  end
+end

data/lib/generators/bulkrax/templates/README

@@ -0,0 +1,3 @@
+Please ensure you have queues for import and export setup.
+
+For example, if using Sidekiq add the queues in config/sidekiq.yml

data/lib/generators/bulkrax/templates/app/models/concerns/bulkrax/has_local_processing.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Bulkrax::HasLocalProcessing
+  # This method is called during build_metadata
+  # add any special processing here, for example to reset a metadata property
+  # to add a custom property from outside of the import data
+  def add_local; end
+end

data/lib/generators/bulkrax/templates/bin/importer

@@ -0,0 +1,140 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../config/environment'
+
+require 'slop'
+
+def main(opts = {})
+  if opts[:importer_id].blank? && invalid?(opts)
+    puts 'Missing required parameters'
+    help
+  end
+
+  if opts[:auth_token].blank?
+    puts 'Missing Authentication Token --auth_token'
+    exit
+  end
+
+  update = opts[:importer_id].present?
+  url = build_url(opts.delete(:importer_id), opts.delete(:url))
+
+  headers = { 'Content-Type' => 'application/json' }
+  headers['Authorization'] = "Token: #{opts.delete(:auth_token)}"
+  params = build_params(opts)
+
+  logger.info("POST to #{url} - PARAMS #{params}")
+
+  conn = Faraday.new(
+    url: url,
+    headers: headers
+  )
+
+  response = if update
+               conn.put do |request|
+                 request.body = params.to_json
+               end
+             else
+               conn.post do |request|
+                 request.body = params.to_json
+               end
+             end
+
+  puts "#{response.status} - #{response.body.truncate(200)}"
+end
+
+def invalid?(opts)
+  required_params.each do |p|
+    return true if opts[p.to_sym].blank?
+  end
+  return false
+end
+
+def required_params
+  Bulkrax.api_definition['bulkrax']['importer'].map { |key, value| key if value['required'] == true }.compact
+end
+
+def build_params(opts = {})
+  params = {}
+  params[:commit] = opts.delete(:commit)
+  parser_fields = {
+    metadata_file_name: opts.delete(:metadata_file_name),
+    metadata_format: opts.delete(:metadata_format),
+    rights_statement: opts.delete(:rights_statement),
+    override_rights_statement: opts.delete(:override_rights_statement),
+    import_file_path: opts.delete(:import_file_path),
+    metadata_prefix: opts.delete(:metadata_prefix),
+    set: opts.delete(:set),
+    collection_name: opts.delete(:collection_name)
+  }.compact
+  params[:importer] = opts.compact
+  params[:importer][:user_id] = opts.delete(:user_id)
+  params[:importer][:admin_set_id] = opts.delete(:admin_set_id)
+  params[:importer][:parser_fields] = parser_fields || {}
+  return params.compact
+end
+
+def build_url(importer_id, url)
+  if url.nil?
+    protocol = Rails.application.config.force_ssl ? 'https://' : 'http://'
+    host = Rails.application.config.action_mailer.default_url_options[:host]
+    url = "#{protocol}#{host}"
+  end
+  path = Bulkrax::Engine.routes.url_helpers.polymorphic_path(Bulkrax::Importer)
+  url = File.join(url, path)
+  url = File.join(url, importer_id) if importer_id
+  return url
+end
+
+def logger
+  Rails.logger
+end
+
+def version
+  puts "Bulkrax #{Bulkrax::VERSION}"
+  puts "Slop #{Slop::VERSION}"
+end
+
+# Format the help for the CLI
+def help
+  puts 'CREATE:'
+  puts '  bin/importer --name "My Import" --parser_klass Bulkrax::CsvParser --commit "Create and Import" --import_file_path /data/tmp/import.csv --auth_token 12345'
+  puts 'UPDATE:'
+  puts '  bin/importer --importer_id 1 --commit "Update and Re-Import (update metadata only)" --import_file_path /data/tmp/import.csv --auth_token 12345'
+  puts 'PARAMETERS:'
+  Bulkrax.api_definition['bulkrax']['importer'].each_pair do |key, value|
+    next if key == 'parser_fields'
+    puts "  --#{key}"
+    value.each_pair do |k, v|
+      next if k == 'contained_in'
+      puts "    #{k}: #{v}"
+    end
+  end
+  puts '  --url'
+  puts "    Repository URL"
+  exit
+end
+
+# Setup the options
+options = Slop.parse do |o|
+  o.on '--version', 'Print the version' do
+    version
+    exit
+  end
+
+  o.on '--help', 'Print help' do
+    help
+    exit
+  end
+
+  Bulkrax.api_definition['bulkrax']['importer'].each_pair do |key, value|
+    if value['required'].blank?
+      o.string "--#{key}", value['definition'], default: nil
+    else
+      o.string "--#{key}", value['definition']
+    end
+  end
+  o.string '--url', 'Repository URL'
+end
+
+main(options.to_hash)

data/lib/generators/bulkrax/templates/config/bulkrax_api.yml

@@ -0,0 +1,84 @@
+bulkrax:
+  importer:
+    importer_id:
+      definition: 'The ID of the Importer. Used for Update only. CLI only.'
+    name: 
+      definition: 'Name for the Importer'
+      required: true
+    admin_set_id:
+      definition: 'AdminSet ID. If not supplied the default AdminSet is used.'
+    parser_fields:
+      definition: 'Contains additional information which varies depending on the parser. Some is required.'
+      type: container
+    rights_statement: 
+      definition: 'Supply a URI from the application rights statements'
+      valid_values: <%= Hyrax.config.rights_statement_service_class.new.select_active_options.map(&:last) %>
+      contained_in: parser_fields
+    override_rights_statement:
+      definition: 'Override any existing rights statements: 1 for yes | 0 for no'
+      valid_values: 
+        - 0
+        - 1
+      contained_in: parser_fields
+    import_file_path:
+      definition: 'Path to the import file - required for CSV and BagIt only'
+      required:
+        - 'Bulkrax::BagitParser'
+        - 'Bulkrax::CsvParser'
+      contained_in: parser_fields
+    selected_files:
+      definition: 'Used by browse everything to retrieve files from cloud URLs'
+    metadata_file_name:
+      definition: 'Filename used for metadata files - required for BagIt only'
+      required: 
+        - 'Bulkrax::BagitParser'
+      contained_in: parser_fields
+    metadata_format:
+      definition: 'RDF or CSV - required for BagIt only'
+      required:
+        - 'Bulkrax::BagitParser'
+      valid_values: 
+        - 'Bulkrax::RdfEntry'
+        - 'Bulkrax::CsvEntry'
+      contained_in: parser_fields
+    set:
+      definition: 'The OAI setSpec - required for OAI only'
+      required:
+        - 'Bulkrax::OaiParser'
+      contained_in: parser_fields
+    collection_name:
+      definition: 'The OAI setName - required for required for OAI only'
+      required:
+        - 'Bulkrax::OaiParser'
+      contained_in: parser_fields
+    parser_klass:
+      definition: 'Parser name eg. Bulkrax::CsvParser'
+      required: true
+      valid_values: <%= Bulkrax.parsers.map {|p|p[:class_name]} %>
+    limit:
+      definition: 'Number of records to import; blank for all'
+    frequency:
+      definition: ISO 8601 Durations <%= Bulkrax::Importer.frequency_enums.map(&:first).join('; ') %>
+      valid_values: <%= Bulkrax::Importer.frequency_enums.map(&:last) %>
+    field_mapping:
+      definition: 'See bulkrax/lib/bulkrax.rb for further information'
+    user_id:
+      definition: 'ID of the User creating the Importer. If not supplied, the batch_user will be used.'
+    base_url:
+      definition: 'Base URL for the Endpoint - required for OAI Only'
+      required:
+        - 'Bulkrax::OaiParser'
+    commit:
+      definition: 'Commit Message, Bulkrax uses commit messages to decide what action to take.'
+      required: true
+      valid_values:
+        - "Create"
+        - "Create and Import"
+        - "Update Importer"
+        - "Update and Re-Import (update metadata and replace files)"
+        - "Update and Harvest Updated Items', 'Update and Re-Harvest All Items"
+        - "Update and Re-Import (update metadata only)"
+        - "Update and Import (importer has not yet been run)"
+    auth_token:
+      definition: 'Authentication token. Required for JSON requests only.'
+      required: true

data/lib/generators/bulkrax/templates/config/initializers/bulkrax.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+Bulkrax.setup do |config|
+  # Add local parsers
+  # config.parsers += [
+  #   { name: 'MODS - My Local MODS parser', class_name: 'Bulkrax::ModsXmlParser', partial: 'mods_fields' },
+  # ]
+
+  # WorkType to use as the default if none is specified in the import
+  # Default is the first returned by Hyrax.config.curation_concerns
+  # config.default_work_type = MyWork
+
+  # Path to store pending imports
+  # config.import_path = 'tmp/imports'
+
+  # Path to store exports before download
+  # config.export_path = 'tmp/exports'
+
+  # Server name for oai request header
+  # config.server_name = 'my_server@name.com'
+
+  # Field_mapping for establishing a parent-child relationship (FROM parent TO child)
+  # This can be a Collection to Work, or Work to Work relationship
+  # This value IS NOT used for OAI, so setting the OAI Entries here will have no effect
+  # The mapping is supplied per Entry, provide the full class name as a string, eg. 'Bulkrax::CsvEntry'
+  # Example:
+  #   {
+  #     'Bulkrax::RdfEntry'  => 'http://opaquenamespace.org/ns/contents',
+  #     'Bulkrax::CsvEntry'  => 'children'
+  #   }
+  # By default no parent-child relationships are added
+  # config.parent_child_field_mapping = { }
+
+  # Field_mapping for establishing a collection relationship (FROM work TO collection)
+  # This value IS NOT used for OAI, so setting the OAI parser here will have no effect
+  # The mapping is supplied per Entry, provide the full class name as a string, eg. 'Bulkrax::CsvEntry'
+  # The default value for CSV is collection
+  # Add/replace parsers, for example:
+  # config.collection_field_mapping['Bulkrax::RdfEntry'] = 'http://opaquenamespace.org/ns/set'
+
+  # Field mappings
+  # Create a completely new set of mappings by replacing the whole set as follows
+  #   config.field_mappings = {
+  #     "Bulkrax::OaiDcParser" => { **individual field mappings go here*** }
+  #   }
+
+  # Add to, or change existing mappings as follows
+  #   e.g. to exclude date
+  #   config.field_mappings["Bulkrax::OaiDcParser"]["date"] = { from: ["date"], excluded: true  }
+  #
+  # #   e.g. to add the required source_identifier field
+  #   #   config.field_mappings["Bulkrax::CsvParser"]["source_id"] = { from: ["old_source_id"], source_identifier: true  }
+  # If you want Bulkrax to fill in source_identifiers for you, see below
+
+  # To duplicate a set of mappings from one parser to another
+  #   config.field_mappings["Bulkrax::OaiOmekaParser"] = {}
+  #   config.field_mappings["Bulkrax::OaiDcParser"].each {|key,value| config.field_mappings["Bulkrax::OaiOmekaParser"][key] = value }
+
+  # Should Bulkrax make up source identifiers for you? This allow round tripping
+  # and download errored entries to still work, but does mean if you upload the
+  # same source record in two different files you WILL get duplicates.
+  # It is given two aruguments, self at the time of call and the index of the reocrd
+  #    config.fill_in_blank_source_identifiers = ->(parser, index) { "b-#{parser.importer.id}-#{index}"}
+  # or use a uuid
+  #    config.fill_in_blank_source_identifiers = ->(parser, index) { SecureRandom.uuid }
+
+  # Properties that should not be used in imports/exports. They are reserved for use by Hyrax.
+  # config.reserved_properties += ['my_field']
+end
+
+# Sidebar for hyrax 3+ support
+Hyrax::DashboardController.sidebar_partials[:repository_content] << "hyrax/dashboard/sidebar/bulkrax_sidebar_additions" if Object.const_defined?(:Hyrax) && ::Hyrax::DashboardController&.respond_to?(:sidebar_partials)

data/lib/tasks/bulkrax_tasks.rake

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# desc "Explaining what the task does"
+# task :bulkrax do
+#   # Task goes here
+# end