gdor-indexer 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5e1bc32ca2df58b2018dbc9f27aa2ce897cde0e4
+  data.tar.gz: c9a3378b959896f6007a7bdbb187069102a7897c
+SHA512:
+  metadata.gz: c401ef958ff8bd02ce867cc066cbf9f40d6127bbc4d4916a79e08d158d694ea2ad4c27e633e2a0f27b8dccf5c623259b4f23df779692d0551f58979040ff0cf6
+  data.tar.gz: f0a797d7d3df7bfbdd4daa75d19a215682eb08ca885c437341a7c02b02e6bfca0b35fdc165d9116cbfb0f9b6eec1cd89de84501a724d1c5b53ca0d4969bc98c7

data/.gitignore

@@ -0,0 +1,31 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+.travis
+.rvmrc
+.ruby-version
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+solrmarc-sw
+solrmarc.log*
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+spec/test_logs
+test/tmp
+test/version_tmp
+tmp
+logs
+.DS_Store
+*.tmproj
+tmtags
+.idea/*
+config/solr.yml
+config/dor-fetcher-client.yml
+config/collections

data/.hound.yml

@@ -0,0 +1,2 @@
+ruby:
+  config_file: .rubocop.yml

data/.rubocop.yml

@@ -0,0 +1,3 @@
+require: rubocop-rspec
+
+inherit_from: .rubocop_todo.yml

data/.rubocop_todo.yml

@@ -0,0 +1,131 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2015-10-26 16:04:23 -0700 using RuboCop version 0.34.2.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 6
+Lint/AmbiguousRegexpLiteral:
+  Exclude:
+    - 'spec/unit/indexer_spec.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: AlignWith, SupportedStyles, AutoCorrect.
+Lint/EndAlignment:
+  Enabled: false
+
+# Offense count: 6
+Lint/UselessAssignment:
+  Exclude:
+    - 'lib/gdor/indexer.rb'
+    - 'spec/unit/gdor_mods_fields_spec.rb'
+    - 'spec/unit/indexer_spec.rb'
+    - 'spec/unit/solr_doc_builder_spec.rb'
+
+# Offense count: 1
+Lint/Void:
+  Exclude:
+    - 'spec/unit/gdor_mods_fields_spec.rb'
+
+# Offense count: 16
+Metrics/AbcSize:
+  Max: 82
+
+# Offense count: 1
+# Configuration parameters: CountComments.
+Metrics/ClassLength:
+  Max: 233
+
+# Offense count: 4
+Metrics/CyclomaticComplexity:
+  Max: 9
+
+# Offense count: 304
+# Configuration parameters: AllowURI, URISchemes.
+Metrics/LineLength:
+  Max: 258
+
+# Offense count: 15
+# Configuration parameters: CountComments.
+Metrics/MethodLength:
+  Max: 45
+
+# Offense count: 3
+Metrics/PerceivedComplexity:
+  Max: 10
+
+# Offense count: 5
+# Configuration parameters: CustomTransform.
+RSpec/FilePath:
+  Exclude:
+    - 'spec/unit/gdor_mods_fields_spec.rb'
+    - 'spec/unit/indexer_spec.rb'
+    - 'spec/unit/public_xml_fields_spec.rb'
+    - 'spec/unit/solr_doc_builder_spec.rb'
+    - 'spec/unit/solr_doc_hash_spec.rb'
+
+# Offense count: 356
+RSpec/InstanceVariable:
+  Exclude:
+    - 'spec/unit/gdor_mods_fields_spec.rb'
+    - 'spec/unit/indexer_spec.rb'
+    - 'spec/unit/public_xml_fields_spec.rb'
+    - 'spec/unit/solr_doc_builder_spec.rb'
+
+# Offense count: 7
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/BracesAroundHashParameters:
+  Exclude:
+    - 'spec/unit/indexer_spec.rb'
+
+# Offense count: 4
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/ClassAndModuleChildren:
+  Exclude:
+    - 'lib/gdor/indexer/mods_fields.rb'
+    - 'lib/gdor/indexer/public_xml_fields.rb'
+    - 'lib/gdor/indexer/solr_doc_builder.rb'
+    - 'lib/gdor/indexer/solr_doc_hash.rb'
+
+# Offense count: 4
+# Configuration parameters: Exclude.
+Style/Documentation:
+  Exclude:
+    - 'lib/gdor/indexer.rb'
+    - 'lib/gdor/indexer/nokogiri_xml_node_mixin.rb'
+    - 'lib/gdor/indexer/solr_doc_hash.rb'
+    - 'lib/gdor/indexer/version.rb'
+
+# Offense count: 1
+Style/DoubleNegation:
+  Exclude:
+    - 'lib/gdor/indexer/solr_doc_hash.rb'
+
+# Offense count: 3
+# Configuration parameters: MinBodyLength.
+Style/GuardClause:
+  Exclude:
+    - 'lib/gdor/indexer.rb'
+
+# Offense count: 1
+# Configuration parameters: NamePrefix, NamePrefixBlacklist.
+Style/PredicateName:
+  Exclude:
+    - 'lib/gdor/indexer/mods_fields.rb'
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles, AllowInnerSlashes.
+Style/RegexpLiteral:
+  Exclude:
+    - 'spec/unit/gdor_mods_fields_spec.rb'
+    - 'spec/unit/indexer_spec.rb'
+
+# Offense count: 1
+Style/UnlessElse:
+  Exclude:
+    - 'lib/gdor/indexer.rb'

data/.yardopts

@@ -0,0 +1,3 @@
+--title 'Walters Indexer Documentation'
+lib/**/*.rb -
+README.rdoc

data/Capfile

@@ -0,0 +1,26 @@
+# Load DSL and Setup Up Stages
+require 'capistrano/setup'
+
+# Includes default deployment tasks
+require 'capistrano/deploy'
+
+# Includes tasks from other gems included in your Gemfile
+#
+# For documentation on these, see for example:
+#
+# https://github.com/capistrano/rvm
+# https://github.com/capistrano/rbenv
+# https://github.com/capistrano/chruby
+# https://github.com/capistrano/bundler
+# https://github.com/capistrano/rails
+#
+# require 'capistrano/rbenv'
+# require 'capistrano/chruby'
+require 'capistrano/bundler'
+# require 'capistrano/rails'
+require 'capistrano/rvm' # gdor-indexer needs jruby until merge-manager
+
+require 'dlss/capistrano'
+
+# Loads custom tasks from `lib/capistrano/tasks' if you have any defined.
+Dir.glob('lib/capistrano/tasks/*.cap').each { |r| import r }

data/Gemfile

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in spotlight-dor-resources.gemspec
+gemspec
+
+group :deployment do
+  gem 'capistrano', '~> 3.2'
+  gem 'capistrano-bundler'
+  gem 'capistrano-rvm' # gdor-indexer used to need jruby for merged records
+  gem 'lyberteam-capistrano-devel'
+  gem 'rainbow' # for color output
+end

data/LICENSE.txt

@@ -0,0 +1,5 @@
+Copyright (c) 2014.  The Board of Trustees of the Leland Stanford Junior University. All rights reserved.
+
+Redistribution and use of this distribution in source and binary forms, with or without modification, are permitted provided that: The above copyright notice and this permission notice appear in all copies and supporting documentation; The name, identifiers, and trademarks of The Board of Trustees of the Leland Stanford Junior University are not used in advertising or publicity without the express prior written permission of The Board of Trustees of the Leland Stanford Junior University; Recipients acknowledge that this distribution is made available as a research courtesy, "as is", potentially with defects, without any obligation on the part of The Board of Trustees of the Leland Stanford Junior University to provide support, services, or repair;
+
+THE BOARD OF TRUSTEES OF THE LELAND STANFORD JUNIOR UNIVERSITY DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, WITH REGARD TO THIS SOFTWARE, INCLUDING WITHOUT LIMITATION ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, AND IN NO EVENT SHALL THE BOARD OF TRUSTEES OF THE LELAND STANFORD JUNIOR UNIVERSITY BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, TORT (INCLUDING NEGLIGENCE) OR STRICT LIABILITY, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

data/README.md

@@ -0,0 +1,67 @@
+[![Dependency Status](https://gemnasium.com/sul-dlss/gdor-indexer.svg)](https://gemnasium.com/sul-dlss/gdor-indexer) [![Gem Version](https://badge.fury.io/rb/gdor-indexer.svg)](http://badge.fury.io/rb/gdor-indexer)
+
+# gdor-indexer
+
+Code to harvest DOR druids via DOR Fetcher service, mods from PURL, and use it to index items into a Solr index, such as that for SearchWorks.
+
+## Prerequisites
+
+1. ruby
+2. bundler gem must be installed
+
+## Install steps for running locally
+
+Add this line to your application's Gemfile:
+
+    gem 'harvestdor-indexer'
+
+Then execute:
+
+    $ bundle
+
+## Configuration
+
+#### Create a collections folder in the config directory:
+
+    $ cd /path/to/gdor-indexer/config
+    $ mkdir collections
+
+#### Create a yml config file for your collection(s) to be harvested and indexed.
+
+See ```spec/config/walters_integration_spec.yml``` for an example.  Copy that file to ```config/collections``` and change the following settings:
+
+* whitelist
+* dor_fetcher service_url
+* harvestdor log_dir and log_name
+* solr_url
+
+##### whitelist
+
+The whitelist is how you specify which objects to index.  The whitelist can be
+
+* an Array of druids inline in the config yml file
+* a filename containing a list of druids (one per line)
+
+If a druid, per the object's identityMetadata at purl page, is for a
+
+* collection record:  then we process all the item druids in that collection (as if they were included individually in the whitelist)
+* non-collection record: then we process the druid as an individual item
+
+#### Run the indexer script
+
+    $ cd /path/to/gdor-indexer
+    $ nohup ./bin/indexer -c my_collection &>path/to/nohup.output
+
+## Running the tests
+
+  ```$ rake```
+
+## Contributing
+
+* Fork it (https://help.github.com/articles/fork-a-repo/)
+* Create your feature branch (`git checkout -b my-new-feature`)
+* Write code and tests.
+* Commit your changes (`git commit -am 'Added some feature'`)
+* Push to the branch (`git push origin my-new-feature`)
+* Create new Pull Request (https://help.github.com/articles/creating-a-pull-request/)
+

data/Rakefile

@@ -0,0 +1,57 @@
+require 'rake'
+require 'bundler'
+require 'bundler/gem_tasks'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts 'Run `bundle install` to install missing gems'
+  exit e.status_code
+end
+
+# add tasks defined in lib/tasks
+# Dir.glob('lib/tasks/*.rake').each { |r| import r }
+
+# desc 'Open an irb session preloaded with this library'
+# task :console do
+#  sh 'irb -rubygems -I lib  -r ./frda_indexer.rb'
+# end
+
+task default: [:ci, :rubocop]
+
+desc 'run continuous integration suite (tests, coverage, docs)'
+task ci: [:rspec, :doc, :rubocop]
+
+task spec: :rspec
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:rspec) do |spec|
+  spec.rspec_opts = ['-c', '-f progress', '--tty', '-r ./spec/spec_helper.rb']
+end
+
+RSpec::Core::RakeTask.new(:rspec_wip) do |spec|
+  spec.rspec_opts = ['-c', '-f d', '--tty', '-r ./spec/spec_helper.rb', '-t wip']
+end
+
+require 'rubocop/rake_task'
+RuboCop::RakeTask.new(:rubocop)
+
+# Use yard to build docs
+require 'yard'
+require 'yard/rake/yardoc_task'
+begin
+  project_root = File.expand_path(File.dirname(__FILE__))
+  doc_dest_dir = File.join(project_root, 'doc')
+
+  YARD::Rake::YardocTask.new(:doc) do |yt|
+    yt.files = Dir.glob(File.join(project_root, 'lib', '**', '*.rb')) +
+      [File.join(project_root, 'README.rdoc')]
+    yt.options = ['--output-dir', doc_dest_dir, '--readme', 'README.rdoc', '--title', 'Gryphondor Indexer Documentation']
+  end
+rescue LoadError
+  desc 'Generate YARD Documentation'
+  task :doc do
+    abort 'Please install the YARD gem to generate rdoc.'
+  end
+end

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/bin/indexer

@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+
+# Look in the lib directory for .rb files
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+
+require 'rubygems'
+require 'bundler/setup'
+require 'trollop'
+require 'gdor/indexer'
+
+# Define the collections that this script knows how to harvest per yml files in the config/collections directory
+# @return Array
+def collections
+  config_dir = File.join(File.dirname(__FILE__), '..', 'config', 'collections')
+  files = Dir.glob("#{config_dir}/*.yml")
+  files.map! { |x| File.basename(x, '.yml') }
+end
+
+# use Trollop to declare options and help text.
+@opts = Trollop.options do
+  version 'indexer v0.2.0 (c) 2012-2014 Stanford University http://searchworks.stanford.edu'
+  banner <<-EOM
+  The indexer script harvests items in a collection from DOR Fetcher Service,
+  harvests MODS from PURL pages, then
+  maps them to Solr document hashes appropriate for SearchWorks, and writes them to Solr
+  Usage:
+    ./indexer [options]
+  where [options] are:
+  EOM
+
+  opt :collection,
+      'Index a given collection. Possible values are #{collections.sort}',
+      default: nil,
+      short: 'c',
+      type: String,
+      multi: true
+  opt :nocommit,
+      "Index the collection but don't commit",
+      default: nil,
+      short: 'n'
+  opt :all, 'Index all collections'
+end
+
+# Trollop::die :collection, "must be a known collection. Possible values are: #{collections.inspect} You entered #{@opts[:collection]}" unless collections.include?@opts[:collection]
+
+def time
+  start = Time.now.getlocal
+  yield
+  elapsed = Time.now.getlocal - start
+  puts "This set took #{elapsed} seconds to run."
+end
+
+# Get all of the specified collections (from the command line) into an Array
+@collections = []
+if @opts[:collection].instance_of? Array
+  @collections = @opts[:collection]
+else
+  @collections << @opts[:collection]
+end
+
+# process each collection from the command line
+@collections.each do |coll|
+  config_yml_path = File.join(File.dirname(__FILE__), '..', 'config', 'collections', "#{coll}.yml")
+  @indexer = GDor::Indexer.new(config_yml_path)
+  puts "Indexing into Solr server #{@indexer.config[:solr][:url]}"
+  puts "Logging output to #{@indexer.config[:harvestdor][:log_dir]}/#{@indexer.config[:harvestdor][:log_name]}"
+
+  time do
+    @indexer.harvest_and_index
+  end
+end

data/config/deploy.rb

@@ -0,0 +1,31 @@
+set :application, 'gdor-indexer'
+set :repo_url, 'https://github.com/sul-dlss/gdor-indexer.git'
+
+# Default branch is :master
+# ask :branch, proc { `git rev-parse --abbrev-ref HEAD`.chomp }
+
+# gdor-indexer needs jruby until merge-manager
+set :rvm_ruby_version, 'jruby-1.7.10'
+
+set :user, 'lyberadmin'
+set :deploy_to, "/home/#{fetch(:user)}/#{fetch(:application)}"
+
+set :linked_dirs, %w(logs config/collections tmp solrmarc-sw)
+set :linked_files, %w(.ruby-version config/solr.yml bin/index-prod-image.sh bin/index-prod-hydrus.sh config/dor-fetcher-client.yml)
+
+set :stages, %w(dev stage prod fetcher)
+
+# Default value for :log_level is :debug
+set :log_level, :info
+
+# Default value for :format is :pretty
+# set :format, :pretty
+
+# Default value for :pty is false
+# set :pty, true
+
+# Default value for default_env is {}
+# set :default_env, { path: "/opt/ruby/bin:$PATH" }
+
+# Default value for keep_releases is 5
+set :keep_releases, 10