base_indexer 3.0.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7c8632ff625459ac2ee0a4b34e95e2fa38ed4306
-  data.tar.gz: b3eaab7a1ef88ee04ae3cfa8429e66075af7ce5a
+  metadata.gz: 9044a8dc8006eb311f4a709b071fd22bdc635115
+  data.tar.gz: d89c940569d6c4003e3fc6f2a9a68b4971e98c66
 SHA512:
-  metadata.gz: fbfabfa97ae959e72e1638d4d2c0abeedac7efda67d625c82537ab6ef10805270c20cd5bf79d74b87e6b9072c7562f91c85e23b98803e4b377e568545224bddc
-  data.tar.gz: 0e266c03b6556806684761f5f7c32bb85f7e95ea586748771533fa0d8defd7760e71fca8e7f4062ef83e1938d0a334f03f9c30c4caf475f1b9b63ccf2bba76fc
+  metadata.gz: 364f6a44ea77b4470279180a0b35f0124ab68dca3c5a77389dc6a6283d9ed5b81b0c6455e1f4bdc03abef2a7f9f9380209dcc99119c0a6e30a97e365f906e32f
+  data.tar.gz: 65b5a40a35a4cd47f65d990e30cd94554ae4ab9cd2ad77539e9bced88f56653f4e04b639c4722cdbafa7787deae45cffe02edab10c0d49a99f19f988ab716f6e

data/.gitignore

@@ -0,0 +1,40 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+Gemfile.lock
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+.internal_test_app
+/spec/internal
+./spec/internal

data/.travis.yml

@@ -0,0 +1,17 @@
+notifications:
+  email: false
+
+rvm:
+  - 2.3.1
+  - 2.2.5
+
+env:
+  global:
+    - NOKOGIRI_USE_SYSTEM_LIBRARIES=true
+
+sudo: false
+language: ruby
+cache: bundler
+
+after_success:
+  coveralls

data/Gemfile

@@ -0,0 +1,49 @@
+source 'https://rubygems.org'
+
+# Declare your gem's dependencies in base_indexer.gemspec.
+# Bundler will treat runtime dependencies like base dependencies, and
+# development dependencies will be added by default to the :development group.
+gemspec
+
+# Declare any dependencies that are still in development here instead of in
+# your gemspec. These might include edge Rails or gems from your path or
+# Git. Remember to move these dependencies to your gemspec before releasing
+# your gem to rubygems.org.
+
+# To use debugger
+# gem 'debugger'
+
+# BEGIN ENGINE_CART BLOCK
+# engine_cart: 0.10.0
+# engine_cart stanza: 0.10.0
+# the below comes from engine_cart, a gem used to test this Rails engine gem in the context of a Rails app.
+file = File.expand_path('Gemfile', ENV['ENGINE_CART_DESTINATION'] || ENV['RAILS_ROOT'] || File.expand_path('.internal_test_app', File.dirname(__FILE__)))
+if File.exist?(file)
+  begin
+    eval_gemfile file
+  rescue Bundler::GemfileError => e
+    Bundler.ui.warn '[EngineCart] Skipping Rails application dependencies:'
+    Bundler.ui.warn e.message
+  end
+else
+  Bundler.ui.warn "[EngineCart] Unable to find test application dependencies in #{file}, using placeholder dependencies"
+
+  if ENV['RAILS_VERSION']
+    if ENV['RAILS_VERSION'] == 'edge'
+      gem 'rails', github: 'rails/rails'
+      ENV['ENGINE_CART_RAILS_OPTIONS'] = '--edge --skip-turbolinks'
+    else
+      gem 'rails', ENV['RAILS_VERSION']
+    end
+  end
+
+  case ENV['RAILS_VERSION']
+  when /^4.2/
+    gem 'responders', '~> 2.0'
+    gem 'sass-rails', '>= 5.0'
+    gem 'coffee-rails', '~> 4.1.0'
+  when /^4.[01]/
+    gem 'sass-rails', '< 5.0'
+  end
+end
+# END ENGINE_CART BLOCK

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2014 The Board of Trustees of the Leland Stanford Junior University.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,151 @@
+[![Build Status](https://travis-ci.org/sul-dlss/base_indexer.svg?branch=master)](https://travis-ci.org/sul-dlss/base_indexer) [![Coverage Status](https://coveralls.io/repos/github/sul-dlss/base_indexer/badge.svg?branch=master)](https://coveralls.io/github/sul-dlss/base_indexer) [![Gem Version](https://badge.fury.io/rb/base_indexer.png)](http://badge.fury.io/rb/base_indexer)
+
+# BaseIndexer
+
+## Running tests
+
+First time setup and to generate all docs:
+
+$ rake                  
+
+Just run the tests next time around:
+
+$ bundle exec rake spec
+
+
+## Steps to hook the base_indexer engine in your app
+### Generate new rails app
+$ rails new my_indexer_app
+
+### Edit Gemfile and add the base_indexer gem name
+gem 'base_indexer'
+
+### Run bundle install to download the gem
+$ bundle install
+
+## Run the install generator
+```sh
+$ rails g base_indexer:install
+```
+
+## Basic configuration
+The engine is looking for the following values
+
+config.solr_config_file_path = "#{config.root}/config/solr.yml"
+DiscoveryIndexer::PURL_DEFAULT='https://purl.stanford.edu'
+
+## Advanced features
+
+The engine gives the developer the ability to extend any of its classes
+
+To extend any of indexer features (purl-reader, mods-reader, mapper, solr-writer)
+
+1. Create a new class that inherits from BaseIndexer::MainIndexerEngine
+2. Create a new file named config/initializers/base_indexer.rb
+3. In this file, add the following line. replace 'MyIndexerClassName' with the fully qualifed actual class name. The name should be between double qoutes
+BaseIndexer.indexer_class = "MyIndexerClassName"
+4. In the new indexer class, you can override any of the functions that you need to change its implementation. For example, if you need to use a new mapper, you will override map function.
+
+To extend mapper functionality.
+1. Create a new class e.g., MyMapper that inherits from GeneralMapper or IndexMapper.
+2. Implement MyMapper.map to converts the input to solr doc hash.
+3. Override MyIndexerClassName.map to call your new class instead of the default one.
+
+## Rake Tasks For Indexing Druids
+
+All rake tasks that perform batch indexing will generate log files in the "log" folder within the app itself.  You can tail the log file to watch the progress.  The log file is also useful since you can pass it to the "reindexer" rake task to retry just the errored out druids.  The name of the log file will depend on which rake task you are running, and will be timestamped to be unique.
+
+### Index a single druid:
+
+$ rake index RAILS_ENV=production target=revs_prod druid=oo000oo0001
+
+### Index a list of druids from a pre-assembly run, a remeditaion run, or a simple CSV:
+
+$ rake log_indexer RAILS_ENV=production target=revs_prod log_file=/tmp/mailander_1.yaml log_type=preassembly  = preassembly run
+
+$ nohup rake log_indexer RAILS_ENV=production target=revs_prod log_file=/tmp/mailander_1.yaml log_type=preassembly &  = for a long running process, which will be most runs that have more than a few dozen druids, nohup it
+
+$ rake log_indexer RAILS_ENV=production target=revs_prod log_file=/tmp/mailander_1_remediate.yaml log_type=remediate = remediation run
+
+$ rake log_indexer RAILS_ENV=production target=revs_prod log_file=/tmp/mailander.csv log_type=csv = a simple csv file -- it must have a header line, with the header of "druid" definining the items you wish to index
+
+### Index an entire collection, including the collection itself, along with all of its members (be sure to check the fetcher_url parameter in the Rails environment you are running under to be sure it is connecting where you expect):
+
+$ rake collection_indexer RAILS_ENV=production target=revs_prod collection_druid=oo000oo0001
+$ nohup rake collection_indexer RAILS_ENV=production target=revs_prod collection_druid=oo000oo0001 &   = for a long running process, e.g. a collection with more than a few dozen druids, nohup it
+
+### Re-Index Just Errored Out Items
+
+If you had errors when indexing from a preassembly/remediation log or from indexing an entire collection, you can re-run the errored out druids only with the log file.  All log files are kept in the log folder in the revs-indexer-service app.
+
+$ rake reindexer RAILS_ENV=production target=revs_prod file=log/logfile.log
+
+$ nohup rake reindexer RAILS_ENV=production target=revs_prod file=log/logfile.log & = probably no need to nohup unless there were alot of errors
+
+### Delete Druids
+
+Delete a list of druids specified in a CSV/txt file.  Be careful, this will delete from all targets!  Put one druid per line, no header is necessary.
+
+$ rake delete_druids RAILS_ENV=production file=druid_list.txt
+
+### Delete a single druid
+
+$ rake delete RAILS_ENV=production druid=oo000oo0001
+
+# HTTP API
+
+### Items
+
+#### `/items/:druid`
+
+`DELETE /items/:druid`
+
+##### Summary
+Deletes a druid from all registered subtargets.
+
+##### Parameters
+Name | Located In | Description | Required | Schema | Default
+---- | ---------- | ----------- | -------- | ------ | -------
+`druid` | url | object identifier | yes | String |
+
+##### Responses
+Code | Description
+---- | -----------
+`200` | Request received and successfully processed for all subtargets
+`500` | Request received but did not complete successfully (one or more subtargets may have not been deleted)
+
+#### `/items/:druid/subtargets/:subtarget`
+
+`DELETE /items/:druid/subtargets/:subtarget`
+
+##### Summary
+Deletes a druid from a specific subtarget.
+
+##### Parameters
+Name | Located In | Description | Required | Schema | Default
+---- | ---------- | ----------- | -------- | ------ | -------
+`druid` | url | object identifier | yes | String |
+`subtarget` | url | subtarget name (usually capitialized) | yes | String |
+
+##### Responses
+Code | Description
+---- | -----------
+`200` | Request received and successfully processed for subtarget
+`500` | Request received but did not complete successfully
+
+`PATCH/PUT /items/:druid/subtargets/:subtarget`
+
+##### Summary
+Indexes a druid from a specific subtarget.
+
+##### Parameters
+Name | Located In | Description | Required | Schema | Default
+---- | ---------- | ----------- | -------- | ------ | -------
+`druid` | url | object identifier | yes | String |
+`subtarget` | url | subtarget name (usually capitialized) | yes | String |
+
+##### Responses
+Code | Description
+---- | -----------
+`200` | Request received and successfully processed for subtarget
+`500` | Request received but did not complete successfully

data/app/controllers/base_indexer/about_controller.rb

@@ -6,7 +6,7 @@ module BaseIndexer
 
     def version
       @result = { app_name: Rails.configuration.app_name, rails_env: Rails.env, version: Rails.configuration.app_version, last_restart: (File.exist?('tmp/restart.txt') ? File.new('tmp/restart.txt').mtime : 'n/a'), last_deploy: (File.exist?('REVISION') ? File.new('REVISION').mtime : 'n/a'), revision: (File.exist?('REVISION') ? File.read('REVISION') : 'n/a') }
-      @result.update(solr_cores: BaseIndexer.solr_configuration_class_name.constantize.instance.get_configuration_hash)
+      @result.update(solr_cores: Settings.SOLR_TARGETS)
       @result.update(gems: Gem.loaded_specs) if request.format.html?
 
       respond_to do |format|

data/base_indexer.gemspec

@@ -0,0 +1,40 @@
+$LOAD_PATH.push File.expand_path('../lib', __FILE__)
+
+# Maintain your gem's version:
+require 'base_indexer/version'
+
+# Describe your gem and declare its dependencies:
+Gem::Specification.new do |s|
+  s.name        = 'base_indexer'
+  s.version     = BaseIndexer::VERSION
+  s.authors     = ['Ahmed Alsum','Laney McGlohon']
+  s.email       = ['laneymcg@stanford.edu']
+  s.summary     = 'Summary of BaseIndexer.'
+  s.description = 'Description of BaseIndexer.'
+  s.license     = 'Apache 2'
+
+  s.files         = `git ls-files -z`.split("\x0")
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  s.bindir        = 'exe'
+  s.executables   = s.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  
+  s.add_dependency 'rails', '>= 4', '< 6'
+  s.add_dependency 'discovery-indexer', '>= 2', '< 4'
+  s.add_dependency 'retries'
+  s.add_dependency 'dor-fetcher'
+  s.add_dependency 'config'
+
+  s.add_development_dependency 'sqlite3'
+
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'rspec-rails'
+  s.add_development_dependency 'capybara'
+  s.add_development_dependency 'vcr'
+  s.add_development_dependency 'webmock'
+  s.add_development_dependency 'coveralls'
+  s.add_development_dependency 'yard' # for documentation
+
+  s.add_development_dependency 'engine_cart'
+  s.add_development_dependency 'jettywrapper'
+end

data/config/initializers/base_indexer.rb

@@ -2,7 +2,5 @@
 # Define the indexer class that will be used by the engine.
 # The engine consumer app should override this class
 BaseIndexer.indexer_class = 'BaseIndexer::MainIndexerEngine'
-BaseIndexer.solr_configuration_class_name = 'BaseIndexer::SolrConfigurationFromFile'
-# BaseIndexer.solr_configuration_class.constantize.new(Rails.configuration.solr_config_file_path)
 BaseIndexer.mapper_class_name = 'DiscoveryIndexer::GeneralMapper'
 BaseIndexer.solr_writer_class_name = 'BaseIndexer::Solr::Writer'

data/config/initializers/config.rb

@@ -0,0 +1,6 @@
+require 'config'
+
+Config.setup do |config|
+  # Name of the constant exposing loaded settings
+  config.const_name = 'Settings'
+end

data/lib/base_indexer.rb

@@ -1,8 +1,6 @@
 require 'base_indexer/engine'
 
 require 'base_indexer/main_indexer_engine'
-require 'base_indexer/config/solr_configuration'
-require 'base_indexer/config/solr_configuration_from_file'
 require 'base_indexer/solr/client'
 require 'base_indexer/solr/writer'
 require 'discovery-indexer'
@@ -10,6 +8,5 @@ module BaseIndexer
   mattr_accessor :indexer_class
   mattr_accessor :mapper_class_name
   mattr_accessor :solr_writer_class_name
-  mattr_accessor :solr_configuration_class_name
   mattr_accessor :fetcher_class
 end

data/lib/base_indexer/engine.rb

@@ -1,5 +1,3 @@
-require 'active_support/core_ext/numeric/bytes'
-
 module BaseIndexer
   class Engine < ::Rails::Engine
     isolate_namespace BaseIndexer
@@ -14,13 +12,6 @@ module BaseIndexer
     config.app_name = '[You have to override this name in your app]'
 
     config.after_initialize do
-      config.solr_config_file = File.join(Rails.root, 'config', 'solr.yml')
-
-      if File.exist? config.solr_config_file
-        # Reads the SOLR configuration file
-        BaseIndexer.solr_configuration_class_name.constantize.instance.read(config.solr_config_file)
-      end
-
       # Initializes the DiscoveryIndexer log with Rails logger, so all the messages will go to
       #   the same log file
       DiscoveryIndexer::Logging.logger = Rails.logger

data/lib/base_indexer/main_indexer_engine.rb

@@ -21,8 +21,7 @@ module BaseIndexer
     # It is the main indexing function
     #
     # @param druid [String] is the druid for an object e.g., ab123cd4567
-    # @param targets [Array] is an array with the targets list to index towards,
-    #   if it is nil, the method will read the target list from release_tags
+    # @param targets [Array] is an array with the targets list to index towards
     #
     # @raise it will raise erros if there is any problems happen in any level
     def index(druid, targets = nil)
@@ -30,15 +29,15 @@ module BaseIndexer
       solr_doc = BaseIndexer.mapper_class_name.constantize.new(druid).convert_to_solr_doc
 
       # Get SOLR configuration and write
-      solr_targets_configs = BaseIndexer.solr_configuration_class_name.constantize.instance.get_configuration_hash
-      BaseIndexer.solr_writer_class_name.constantize.new.process(druid, solr_doc, targets, solr_targets_configs)
+      BaseIndexer.solr_writer_class_name.constantize.new
+                 .process(druid, solr_doc, targets, Settings.SOLR_TARGETS.to_hash.deep_stringify_keys)
     end
 
     # It deletes an item defined by druid from all registered solr core
     # @param druid [String] is the druid for an object e.g., ab123cd4567
     def delete(druid)
-      solr_targets_configs = BaseIndexer.solr_configuration_class_name.constantize.instance.get_configuration_hash
-      BaseIndexer.solr_writer_class_name.constantize.new.solr_delete_from_all(druid, solr_targets_configs)
+      BaseIndexer.solr_writer_class_name.constantize.new
+                 .solr_delete_from_all(druid, Settings.SOLR_TARGETS.to_hash.deep_stringify_keys)
     end
 
   end

data/lib/base_indexer/solr/client.rb

@@ -42,14 +42,14 @@ module BaseIndexer
           DiscoveryIndexer::Logging.logger.debug "Attempt #{attempt} for #{id}"
 
           if is_delete
-            DiscoveryIndexer::Logging.logger.info "Deleting #{id} on attempt #{attempt}"
             solr_connector.delete_by_id(id, :add_attributes => {:commitWithin => 10000})
+            DiscoveryIndexer::Logging.logger.info "Deleting #{id} on attempt #{attempt}"
           elsif allow_update?(solr_connector) && doc_exists?(id, solr_connector)
-            DiscoveryIndexer::Logging.logger.info "Updating #{id} on attempt #{attempt}"
             update_solr_doc(id, solr_doc, solr_connector)
+            DiscoveryIndexer::Logging.logger.info "Updating #{id} on attempt #{attempt}"
           else
-            DiscoveryIndexer::Logging.logger.info "Indexing #{id} on attempt #{attempt}"
             solr_connector.add(solr_doc, :add_attributes => {:commitWithin => 10000})
+            DiscoveryIndexer::Logging.logger.info "Indexing #{id} on attempt #{attempt}"
           end
           #solr_connector.commit
           DiscoveryIndexer::Logging.logger.info "Completing #{id} successfully on attempt #{attempt}"