cdmbl 0.7.2 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cd1dcc251930032444c67fd600f933acc1d8a25f
-  data.tar.gz: 8c5df4dc3bc694e063961cb6e2a320e75bacf1d3
+  metadata.gz: 818b769be7195ae37538de0892e275545da57a44
+  data.tar.gz: 5a1a6c25b618ea2cd1d047e3ef3047e29e54db87
 SHA512:
-  metadata.gz: e8b1baf77f906f90cc8341d0af96e89afefeeb1b59d14961b9be5f3021f759d38d2d9b49bdb5e03f1211d926557f2cb4f4209fcc6e987be2352d1db7c6ad3431
-  data.tar.gz: 612b9147b36693f6dc742d2a6e46432aa8b5cc88165676bd5b5ef5a158faa33ce26e687dce837f20c6ae5e080ac130978e660d7c727fe4792d64e882648fdf50
+  metadata.gz: aba7fdaefa7ca9e8031b9ac42c9384ebd41ca7bff629d978715aa57cb416248c2b4f5e5e234cf223b61406ffa597591e1906584e4657ed39fc60a361641c7c1d
+  data.tar.gz: 7a1e280b31c57cdf98207481f02946bfdaa6abe2dbfebe36ae9722ea9b444da3dec939859d8db66ce01565b476b4e211ec65d679c707a1378c3f882d3fd9e9bd

data/.rubocop.yml

@@ -0,0 +1,4 @@
+Layout/IndentationWidth:
+  # Number of spaces for each indentation level.
+  Width: 2
+  IgnoredPatterns: []

data/README.md

@@ -4,7 +4,7 @@
 
 Use [Blacklight](https://github.com/projectblacklight/blacklight) as a front end for your CONTENTdm instance.
 
-At the moment, CDMBL consists only of a micro [ETL](https://en.wikipedia.org/wiki/Extract,_transform,_load) system dedicated to extracting metadata records from a CONTENTdm instance (using the [CONTENTdm API gem](https://github.com/UMNLibraries/contentdm_api), transforming them into Solr documents, and loading them into Solr. After initially populating the entire index, CDMBL allows for selective harvesting for incremental Solr index updates.
+At the moment, CDMBL consists only of a micro [ETL](https://en.wikipedia.org/wiki/Extract,_transform,_load) system dedicated to extracting metadata records from a CONTENTdm instance (using the [CONTENTdm API gem](https://github.com/UMNLibraries/contentdm_api), transforming them into Solr documents, and loading them into Solr.
 
 ## Installation
 
@@ -41,22 +41,24 @@ export export GEONAMES_USER="yourusernamehere"
 
 Run the ingester
 
-rake cdmbl:ingest[solr_url,oai_endpoint,cdm_endpoint,minimum_date]
+rake cdmbl:batch[solr_url,oai_endpoint,cdm_endpoint,set_spec, batch_size, max_compounds]
 
 |Argument| Definition|
 |--:|---|
 |solr_url| The full URL to your Solr core instance (same as your blacklight.yml solr url)|
-|oai_endpoint| A URL to your OAI instance (e.g. http://reflections.mndigital.org/oai/oai.php)   |
+|oai_endpoint| A URL to your OAI instance (e.g. https://server16022.contentdm.oclc.org/oai/oai.php)   |
 |cdm_endpoint| A URL to your CONTENTdm API endpoint (e.g. https://server16022.contentdm.oclc.org/dmwebservices/index.php) |
-|minimum_date| Date from which to [selectively harvest](https://www.openarchives.org/OAI/openarchivesprotocol.html#SelectiveHarvesting) identifiers from the OAI endpoint. These identifiers are used to determine which records to delete from your index and which records to request from the CONTENTdm API|
+|set_spec| Selectively harvest from a single collection with [setSpec](http://www.openarchives.org/OAI/openarchivesprotocol.html#Set)|
+|batch_size| The number of records to transform at a time. **Note**: it is within the record transformation process that the CONTENTdm API is requested. This API can be sluggish, so we conservatively transform batches of ten records at a time to prevent timeouts.|
+|max_compounds| CONTENTdm records with many compounds can take a long time to load from the CONTENTdm API as multiple requests must happen in order to get the metadata for each child record of a parent compound object. For this reason, records with ten or more compound children are, by default, processed in batches of one. This setting allows you to override this behavior.|
 
 For example:
 
 ```ruby
-rake "cdmbl:ingest[http://solr:8983/solr/foo-bar-core, http://reflections.mndigital.org/oai/oai.php, https://server16022.contentdm.oclc.org/dmwebservices/index.php, 2015-01-01]"
+rake "cdmbl:ingest[http://solr:8983/solr/foo-bar-core, https://server16022.contentdm.oclc.org/oai/oai.php, https://server16022.contentdm.oclc.org/dmwebservices/index.php, 2015-01-01]"
 ```
 
-### Custom Rake Task
+### Custom Rake Tasks
 
 You might also create your own rake task to run your modified field transformers:
 
@@ -64,14 +66,21 @@ You might also create your own rake task to run your modified field transformers
 require 'cdmbl'
 
 namespace :cdmbl do
-  desc 'Launch a background job to index metadata from CONTENTdm int Solr.'
-  task :ingest do
-    solr_config = { url: 'http://solr:8983/solr/foo-bar-core' }
-    etl_config  = { oai_endpoint: 'http://reflections.mndigital.org/oai/oai.php',
-                    cdm_endpoint: 'https://server16022.contentdm.oclc.org/dmwebservices/index.php',
-                    field_mappings: my_field_mappings,
-                    minimum_date:  '2016-09-01'}
-    CDMBL::ETLWorker.perform_async(solr_config, etl_config)
+  desc "ingest batches of records"
+  ##
+  # e.g. rake mdl_ingester:ingest[2015-09-14, 2]
+  task :batch, [:batch_size, :set_spec] => :environment  do |t, args|
+    config  =
+      {
+        oai_endpoint: 'http://cdm16022.contentdm.oclc.org/oai/oai.php',
+        cdm_endpoint: 'https://server16022.contentdm.oclc.org/dmwebservices/index.php',
+        set_spec: (args[:set_spec] != '""') ? args[:set_spec] : nil,
+        max_compounds: (args[:max_compounds]) ? args[:max_compounds] : 2,
+        batch_size: (args[:batch_size]) ? args[:batch_size] : 30,
+        solr_config: solr_config
+      }
+    CDMBL::ETLWorker.perform_async(config)
+  end
 end
 ```
 ### Your Own Custom Solr Field Mappings (see above code snippet)

data/cdmbl.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'rsolr', '~> 2.0'
   # This gem generally wants to be in a rails app, but just to avoid adding
   # another external dependency for XML procssing, we rely on activesupport's
-  # Has.to_xml feature for testing and to allow this gem to function
+  # Has.to_jsonl feature for testing and to allow this gem to function
   # independently from a rails app
   spec.add_dependency 'activesupport', '>= 4.2'
 

data/lib/cdmbl/batch_deleter_worker.rb

@@ -3,8 +3,7 @@ module CDMBL
   class BatchDeleterWorker
     include Sidekiq::Worker
     attr_reader :start, :prefix, :oai_url, :solr_url
-    attr_accessor :batch_deleter_klass, :oai_client, :solr_client
-    sidekiq_options :backtrace => true
+    attr_writer :batch_deleter_klass, :oai_client, :solr_client
     def perform(start = 0, prefix = '', oai_url = '', solr_url = '')
       @start    = start
       @prefix   = prefix

data/lib/cdmbl/compound_filter.rb

@@ -0,0 +1,45 @@
+module CDMBL
+    # Takes a list of record id/collection data, uses CompoundLookup to
+    # identifiy records with large numbers of compounds and sorts them
+    # into a large and a small heap
+  class CompoundFilter
+    attr_reader :record_ids,
+                :max_compounds,
+                :cdm_endpoint,
+                :compound_lookup_klass
+    def initialize(record_ids: [],
+                   max_compounds: 10,
+                   cdm_endpoint: '',
+                   compound_lookup_klass: CompoundLookup)
+      @record_ids            = record_ids
+      @max_compounds         = max_compounds
+      @cdm_endpoint          = cdm_endpoint
+      @compound_lookup_klass = compound_lookup_klass
+    end
+
+    def filter(large: true)
+      ids(records.select { |record| record[:large] == large })
+    end
+
+    private
+
+    def ids(records)
+      records.map { |record| record[:id] }
+    end
+
+    def records
+     @records ||= record_ids.map do |identifier|
+        {
+          large: count(*identifier) >= max_compounds,
+          id: identifier
+        }
+      end
+    end
+
+    def count(collection, id)
+      compound_lookup_klass.new(cdm_endpoint: cdm_endpoint,
+                                collection: collection,
+                                id: id).count
+    end
+  end
+end

data/lib/cdmbl/compound_lookup.rb

@@ -0,0 +1,43 @@
+module CDMBL
+    # Fetching the full metadata for compound records is expensive. This class
+    # lets us check on how many compounds a CDM record has so that we know
+  class CompoundLookup
+      attr_reader :cdm_endpoint,
+                  :collection,
+                  :id,
+                  :request_klass,
+                  :service_klass
+
+    def initialize(cdm_endpoint: '',
+                   collection: '',
+                   id: '',
+                   request_klass: CONTENTdmAPI::Request,
+                   service_klass: CONTENTdmAPI::Service)
+      @cdm_endpoint  = cdm_endpoint
+      @collection    = collection
+      @id            = id
+      @request_klass = request_klass
+      @service_klass = service_klass
+    end
+
+    def count
+      page.respond_to?(:length) ? page.length : 0
+    end
+
+    private
+
+    def page
+      JSON.parse(request).fetch('page', [])
+    end
+
+    def service
+      @service ||= service_klass.new(function: 'dmGetCompoundObjectInfo',
+                                     params: [collection, id])
+    end
+
+    def request
+      @request ||= request_klass.new(base_url: cdm_endpoint,
+                                     service: service).fetch
+    end
+  end
+end

data/lib/cdmbl/default_solr.rb

@@ -26,7 +26,6 @@ module CDMBL
 
     def add(records)
       connection.add records
-      connection.commit
     end
 
     def delete(ids)

data/lib/cdmbl/etl_worker.rb

@@ -1,89 +1,140 @@
 require 'sidekiq'
 module CDMBL
+  # Extract records from OAI, delete records marked for deletion, sort the
+  # remaning records them into "big and small" record piles based upon how many
+  # compounds a record has, chunk the small records into batches and the big
+  # records individuall and then send these records to a transformation worker
   class ETLWorker
     include Sidekiq::Worker
-
-    attr_reader :solr_config,
-                :etl_config,
+    attr_reader :config,
+                :solr_config,
+                :cdm_endpoint,
+                :oai_endpoint,
+                :field_mappings,
+                :resumption_token,
+                :set_spec,
+                :max_compounds,
                 :batch_size,
-                :is_recursive,
-                :identifiers,
-                :deletables
-
-
-    def perform(solr_config,
-                etl_config,
-                batch_size = 5,
-                is_recursive = true,
-                identifiers = [],
-                deletables = [])
-
-      @etl_config   = etl_config.symbolize_keys
-      @solr_config  = solr_config.symbolize_keys
-      @batch_size   = batch_size.to_i
-      @is_recursive = is_recursive
-      @identifiers  = identifiers
-      @deletables   = deletables
-
-      if !identifiers.empty?
-        load!
+                :is_recursive
+
+    attr_writer :compound_filter_klass,
+                :extractor_klass,
+                :etl_worker_klass,
+                :load_worker_klass,
+                :completed_callback_klass,
+                :transform_worker_klass
+
+    def perform(config)
+      # Sidekiq stores params in JSON, so we can't inject dependencies. This
+      # results in the long set of arguments that follows. Otherwise, we'd
+      # simply inject the OAI request and extractor objects
+      @config           = config
+      @solr_config      = config.fetch('solr_config').symbolize_keys
+      @cdm_endpoint     = config.fetch('cdm_endpoint')
+      @oai_endpoint     = config.fetch('oai_endpoint')
+      @field_mappings   = config.fetch('field_mappings', false)
+      @resumption_token = config.fetch('resumption_token', nil)
+      @set_spec         = config.fetch('set_spec', nil)
+      @max_compounds    = config.fetch('max_compounds', 10)
+      @batch_size       = config.fetch('batch_size', 5).to_i
+      @is_recursive     = config.fetch('is_recursive', true)
+      extract_batch!
+      next_batch!
+    end
+
+    # Because Sidekiq serializes params to JSON, we provide custom setters
+    # for dependencies (normally these would be default params in the
+    # constructor) so that they may be mocked and tested
+    def completed_callback_klass
+      @completed_callback_klass ||= CDMBL::CompletedCallback
+    end
+
+    def etl_worker_klass
+      @etl_worker_klass ||= ETLWorker
+    end
+
+    def compound_filter_klass
+      @compound_filter_klass ||= CompoundFilter
+    end
+
+    def extractor_klass
+      @extractor_klass ||= Extractor
+    end
+
+    def load_worker_klass
+      @load_worker_klass ||= LoadWorker
+    end
+
+    def transform_worker_klass
+      @transform_worker_klass ||= TransformWorker
+    end
+
+    # Recurse through OAI batches one at a time
+    def next_batch!
+      if next_resumption_token && is_recursive
+        etl_worker_klass.perform_async(next_config)
       else
-        ingest_batches!
-        if extraction.next_resumption_token && is_recursive
-          # Call the next batch of records
-          ETLWorker.perform_async(solr_config, next_etl_config, batch_size)
-        else
-          CDMBL::CompletedCallback.call!(solr_client)
-        end
+        completed_callback_klass.call!(solr_config)
       end
     end
 
     private
 
-    # Break down extractions into batches of IDs for ingestion
-    def ingest_batches!
-      sent_deleted = false
-      extraction.local_identifiers.each_slice(batch_size) do |ids|
-        delete_ids = (sent_deleted == false) ? extraction.deletable_ids : []
-        ETLWorker.perform_async(solr_config,
-                                etl_config,
-                                batch_size,
-                                is_recursive,
-                                ids,
-                                delete_ids)
-        sent_deleted = true
-      end
+    # Extract an oai response - a batch of records
+    def extract_batch!
+      # Delete records that OAI has marked for deletion
+      delete_deletables!
+      # Records with few compounds are processed in batches
+      transform_small_records!
+      # Large records are all transformed and loaded one by one to avoid
+      # timeouts
+      transform_large_records!
     end
 
-    def load!
-      CDMBL::LoaderNotification.call!(transformation.records, deletables)
-      etl_run.load!(deletables, transformation.records)
+    def next_config
+      config.merge(resumption_token: next_resumption_token)
     end
 
-    def transformation
-      @transformation ||= etl_run.transform(extraction.set_lookup, records)
+    def next_resumption_token
+      @next_resumption_token ||= extraction.next_resumption_token
     end
 
-    def records
-      identifiers.map do |identifier|
-        extraction.cdm_request(*identifier)
+    def transform_small_records!
+      compound_filter.filter(large: false).each_slice(batch_size) do |ids|
+        transform!(ids)
       end
     end
 
-    def extraction
-      @extraction ||= etl_run.extract
+    def transform_large_records!
+      compound_filter.filter(large: true).each do |id|
+        transform!([id])
+      end
     end
 
-    def etl_run
-      ETLRun.new(etl_config.merge(solr_client: solr_client))
+    def transform!(ids)
+        transform_worker_klass.perform_async(ids,
+                                             solr_config,
+                                             cdm_endpoint,
+                                             oai_endpoint,
+                                             field_mappings)
     end
 
-    def solr_client
-      @solr_client ||= CDMBL::Solr.new(solr_config)
+    def delete_deletables!
+      load_worker_klass.perform_async([], extraction.deletable_ids, solr_config)
     end
 
-    def next_etl_config
-      etl_config.merge(resumption_token: extraction.next_resumption_token)
+    def compound_filter
+      @compound_filter ||=
+        compound_filter_klass.new(record_ids: extraction.local_identifiers,
+                                  cdm_endpoint: cdm_endpoint,
+                                  max_compounds: max_compounds)
+    end
+
+    def extraction
+      @extraction ||=
+        extractor_klass.new(oai_endpoint: oai_endpoint,
+                            resumption_token: resumption_token,
+                            set_spec: set_spec)
     end
   end
-end
+end

data/lib/cdmbl/extract_worker.rb

@@ -0,0 +1,141 @@
+# require 'sidekiq'
+# module CDMBL
+#   # Extract records from OAI, delete records marked for deletion, sort the
+#   # remaning records them into "big and small" record piles based upon how many
+#   # compounds a record has, chunk the small records into batches and the big
+#   # records individuall and then send these records to a transformation worker
+#   class ExtractWorker
+#     include Sidekiq::Worker
+
+#     attr_reader :config,
+#                 :solr_config,
+#                 :cdm_endpoint,
+#                 :oai_endpoint,
+#                 :field_mappings,
+#                 :resumption_token,
+#                 :set_spec,
+#                 :max_compounds,
+#                 :batch_size,
+#                 :is_recursive
+
+#     attr_writer :compound_filter_klass,
+#                 :extractor_klass,
+#                 :extraction_worker_klass,
+#                 :load_worker_klass,
+#                 :completed_callback_klass,
+#                 :transform_worker_klass
+
+#     def perform(config)
+#       # Sidekiq stores params in JSON, so we can't inject dependencies. This
+#       # results in the long set of arguments that follows. Otherwise, we'd
+#       # simply inject the OAI request and extractor objects
+#       @config           = config
+#       @solr_config      = config.fetch('solr_config').symbolize_keys
+#       @cdm_endpoint     = config.fetch('cdm_endpoint')
+#       @oai_endpoint     = config.fetch('oai_endpoint')
+#       @field_mappings   = config.fetch('field_mappings', [])
+#       @resumption_token = config.fetch('resumption_token', nil)
+#       @set_spec         = config.fetch('set_spec', nil)
+#       @max_compounds    = config.fetch('max_compounds', 10)
+#       @batch_size       = config.fetch('batch_size', 5).to_i
+#       @is_recursive     = config.fetch('is_recursive', true)
+#       extract_batch!
+#       next_batch!
+#     end
+
+#     # Because Sidekiq serializes params to JSON, we provide custom setters
+#     # for dependencies (normally these would be default params in the
+#     # constructor) so that they may be mocked and tested
+#     def completed_callback_klass
+#       @completed_callback_klass ||= CDMBL::CompletedCallback
+#     end
+
+#     def extraction_worker_klass
+#       @extraction_worker_klass ||= ExtractionWorker
+#     end
+
+#     def compound_filter_klass
+#       @compound_filter_klass ||= CompoundFilter
+#     end
+
+#     def extractor_klass
+#       @extractor_klass ||= Extractor
+#     end
+
+#     def load_worker_klass
+#       @load_worker_klass ||= LoadWorker
+#     end
+
+#     def transform_worker_klass
+#       @transform_worker_klass ||= TransformWorker
+#     end
+
+#     # Recurse through OAI batches one at a time
+#     def next_batch!
+#       if next_resumption_token && is_recursive
+#         extraction_worker_klass.perform_async(next_config)
+#       else
+#         completed_callback_klass.call!(solr_config)
+#       end
+#     end
+
+#     private
+
+#     # Extract an oai response - a batch of records
+#     def extract_batch!
+#       # Delete records that OAI has marked for deletion
+#       delete_deletables!
+#       # Records with few compounds are processed in batches
+#       transform_small_records!
+#       # Large records are all transformed and loaded one by one to avoid
+#       # timeouts
+#       transform_large_records!
+#     end
+
+#     def next_config
+#       config.merge(resumption_token: next_resumption_token)
+#     end
+
+#     def next_resumption_token
+#       @next_resumption_token ||= extraction.next_resumption_token
+#     end
+
+#     def transform_small_records!
+#       compound_filter.filter(large: false).each_slice(batch_size) do |ids|
+#         transform!(ids)
+#       end
+#     end
+
+#     def transform_large_records!
+#       compound_filter.filter(large: true).each do |id|
+#         transform!([id])
+#       end
+#     end
+
+#     def transform!(ids)
+#         transform_worker_klass.perform_async(ids,
+#                                              solr_config,
+#                                              cdm_endpoint,
+#                                              oai_endpoint,
+#                                              field_mappings)
+#     end
+
+#     def delete_deletables!
+#       load_worker_klass.perform_async([], extraction.deletable_ids, solr_config)
+#     end
+
+#     def compound_filter
+#       @compound_filter ||=
+#         compound_filter_klass.new(record_ids: extraction.local_identifiers,
+#                                   cdm_endpoint: cdm_endpoint,
+#                                   max_compounds: max_compounds)
+#     end
+
+#     def extraction
+#       @extraction ||=
+#         extractor_klass.new(oai_endpoint: oai_endpoint,
+#                             resumption_token: resumption_token,
+#                             set_spec: set_spec)
+#     end
+#   end
+# end

data/lib/cdmbl/extractor.rb

@@ -4,32 +4,27 @@ require 'hash_at_path'
 require 'forwardable'
 
 module CDMBL
-  # This extractor uses the SimpleGet extractor initially and then makes
-  # subsequent passes at the full ContentDM API with identifiers taken from
-  # the contentdm api
+  # Retrieve OAI records and sort them into add/updatables and deletables
   class Extractor
     extend ::Forwardable
     def_delegators :@oai_request, :sets, :identifiers
     attr_reader :oai_request,
-                :cdm_item,
-                :cdm_endpoint,
-                :oai_set_lookup,
-                :oai_filter
+                :oai_request_klass,
+                :oai_filter_klass,
+                :oai_set_lookup_klass
 
-    def initialize(oai_request: OaiRequest.new,
-                   cdm_endpoint: '',
-                   oai_set_lookup: OAISetLookup,
-                   cdm_item: CONTENTdmAPI::Item,
-                   oai_filter: OAIFilter)
-      @oai_request    = oai_request
-      @cdm_item       = cdm_item
-      @cdm_endpoint   = cdm_endpoint
-      @oai_set_lookup = oai_set_lookup
-      @oai_filter     = oai_filter
-    end
-
-    def set_lookup
-      oai_set_lookup.new(oai_sets: sets).keyed
+    def initialize(oai_endpoint: '',
+                   resumption_token: nil,
+                   set_spec: nil,
+                   oai_request_klass: OaiRequest,
+                   oai_filter_klass: OAIFilter,
+                   oai_set_lookup_klass: OAISetLookup)
+      @oai_request_klass    = oai_request_klass
+      @oai_filter_klass     = oai_filter_klass
+      @oai_set_lookup_klass = oai_set_lookup_klass
+      @oai_request          = oai_requester(oai_endpoint,
+                                            resumption_token,
+                                            set_spec)
     end
 
     def deletable_ids
@@ -44,16 +39,21 @@ module CDMBL
       oai_identifiers.at_path('OAI_PMH/ListIdentifiers/resumptionToken')
     end
 
-    # e.g. local_identifiers.map { |identifier| extractor.cdm_request(*identifier) }
-    def cdm_request(collection, id)
-      CDMBL::CdmNotification.call!(collection, id, cdm_endpoint)
-      cdm_item.new(base_url: cdm_endpoint, collection: collection, id: id).metadata
+    def oai_ids
+      oai_filter_klass.new(headers: oai_headers)
+    end
+
+    def set_lookup
+      oai_set_lookup_klass.new(oai_sets: sets).keyed
     end
 
     private
 
-    def oai_ids
-      oai_filter.new(headers: oai_headers)
+    def oai_requester(oai_endpoint, resumption_token, set_spec)
+      @oai_requester ||=
+        oai_request_klass.new(base_uri: oai_endpoint,
+                              resumption_token: resumption_token,
+                              set: set_spec)
     end
 
     # Get the local collection and id from an OAI namespaced identifier
@@ -67,7 +67,7 @@ module CDMBL
     end
 
     def oai_identifiers
-      identifiers
+      @oai_identifiers ||= identifiers
     end
   end
-end
+end

data/lib/cdmbl/load_worker.rb

@@ -0,0 +1,35 @@
+require 'sidekiq'
+module CDMBL
+  # Load Records into a solr index
+  class LoadWorker
+    include Sidekiq::Worker
+    attr_reader :solr_config, :records, :deletables
+    attr_writer :loader_klass, :solr_klass
+    def perform(records = [], deletables = [], solr_config = {})
+      @solr_config = solr_config.symbolize_keys
+      @records     = records
+      @deletables  = deletables
+      load!
+    end
+
+    def loader_klass
+      @loader_klass ||= Loader
+    end
+
+    def solr_klass
+      @solr_klass ||= DefaultSolr
+    end
+
+    def load!
+      loader_klass.new(records: records,
+                       deletable_ids: deletables,
+                       solr_client: solr_client).load!
+    end
+
+    private
+
+    def solr_client
+      @solr_client ||= solr_klass.new(solr_config)
+    end
+  end
+end

data/lib/cdmbl/oai_request.rb

@@ -4,19 +4,16 @@ module CDMBL
       attr_reader :base_uri,
                   :resumption_token,
                   :client,
-                  :from,
                   :set,
                   :identifier
       def initialize(base_uri: '',
-                     resumption_token: false,
-                     from: false,
-                     set: false,
+                     resumption_token: nil,
+                     set: nil,
                      identifier: '',
                      client: Net::HTTP)
           @base_uri         = base_uri
           @resumption_token = resumption_token
           @client           = client
-          @from             = (from) ? "&from=#{from}" : ''
           @set              = (set) ? "&set=#{set}" : ''
           @identifier       = identifier
       end
@@ -32,7 +29,7 @@ module CDMBL
       private
 
       def first_batch_uri
-        "#{base_uri}?verb=ListIdentifiers&metadataPrefix=oai_dc#{from}#{set}"
+        "#{base_uri}?verb=ListIdentifiers&metadataPrefix=oai_dc#{set}"
       end
 
       def batch_uri

data/lib/cdmbl/tasks/etl.rake

@@ -2,17 +2,37 @@ require 'cdmbl'
 
 namespace :cdmbl do
   desc 'Launch a background job to index metadata from CONTENTdm to Solr.'
-  task :ingest, [:solr_url, :oai_endpoint, :cdm_endpoint, :minimum_date, :batch_size, :set_spec] do |t, args|
-    solr_config = { url: args[solr_url] }
-    etl_config  = {
-                    oai_endpoint: args[:oai_endpoint],
-                    cdm_endpoint: args[:cdm_endpoint],
-                    minimum_date: args[:minimum_date],
-                    set_spec: args[:set_spec]
-                  }
-    etl_config = (args[:resumption_token]) ? etl_cofig.merge(args[:resumption_token]) : etl_config
-    batch_size = (args[:batch_size]) ? args[:batch_size] : 10
-    CDMBL::ETLWorker.perform_async(solr_config, etl_config, batch_size, true)
+  task :batch, [
+    :solr_url,
+    :oai_endpoint,
+    :cdm_endpoint,
+    :set_spec,
+    :batch_size,
+    :max_compounds
+  ] do |t, args|
+    CDMBL::ETLWorker.perform_async(
+      solr_config: { url: args.fetch(:solr_url) },
+      oai_endpoint: args.fetch(:oai_endpoint),
+      cdm_endpoint: args.fetch(:cdm_endpoint),
+      set_spec: args[:set_spec] != '""' ? args[:set_spec] : nil,
+      batch_size: args.fetch(:batch_size, 10),
+      max_compounds: args.fetch(:max_compounds, 10)
+    )
   end
-end
 
+  desc 'Launch a background job to index a single record.'
+  task :record, [
+    :collection,
+    :id,
+    :solr_url,
+    :cdm_endpoint,
+    :oai_endpoint
+  ] do |t, args|
+    CDMBL::TransformWorker.perform_async(
+      [[args.fetch(:collection), args.fetch(:id)]],
+      { url: args.fetch(:solr_url) },
+      args.fetch(:cdm_endpoint),
+      args.fetch(:oai_endpoint)
+    )
+  end
+end

data/lib/cdmbl/tasks/extract.rake

@@ -0,0 +1,9 @@
+require 'cdmbl'
+
+namespace :cdmbl do
+  desc 'Extract OAI results to the local file system.'
+  task :extract, [:oai_endpoint, :storage_dir] do |t, args|
+    CDMBL::OAIWorker.perform_async(oai_endpoint, false, storage_dir)
+  end
+end
+

data/lib/cdmbl/transform_worker.rb

@@ -0,0 +1,93 @@
+require 'sidekiq'
+module CDMBL
+  class TransformWorker
+    include Sidekiq::Worker
+    attr_reader :identifiers,
+                :solr_config,
+                :cdm_endpoint,
+                :oai_endpoint,
+                :field_mappings
+
+    attr_writer :cdm_api_klass,
+                  :oai_request_klass,
+                  :oai_set_lookup_klass,
+                  :cdm_notification_klass,
+                  :load_worker_klass,
+                  :transformer_klass
+
+    def perform(identifiers,
+                solr_config,
+                cdm_endpoint,
+                oai_endpoint,
+                field_mappings)
+
+      @identifiers    = identifiers
+      @solr_config    = solr_config
+      @cdm_endpoint   = cdm_endpoint
+      @oai_endpoint   = oai_endpoint
+      @field_mappings = field_mappings
+
+      transform_and_load!
+    end
+
+    def oai_set_lookup_klass
+      @oai_set_lookup_klass ||= OAISetLookup
+    end
+
+    def oai_request_klass
+      @oai_request_klass ||= OaiRequest
+    end
+
+    def cdm_api_klass
+      @cdm_api_klass ||= CONTENTdmAPI::Item
+    end
+
+    def cdm_notification_klass
+      @cdm_notification_klass ||= CdmNotification
+    end
+
+    def transformer_klass
+      @transformer_klass ||= Transformer
+    end
+
+    def load_worker_klass
+      @load_worker_klass ||= LoadWorker
+    end
+
+    private
+
+    def transform_and_load!
+      load_worker_klass.perform_async(transformed_records, [], solr_config)
+    end
+
+    def transformed_records
+      @transformation ||=
+        transformer_klass.new(cdm_records: records,
+                              oai_sets: set_lookup,
+                              field_mappings: field_mappings).records
+    end
+
+    def set_lookup
+      oai_set_lookup_klass.new(oai_sets: sets).keyed
+    end
+
+    def records
+      identifiers.map do |identifier|
+        cdm_request(*identifier)
+      end
+    end
+
+    # e.g. local_identifiers.map { |identifier| extractor.cdm_request(*identifier) }
+    def cdm_request(collection, id)
+      cdm_notification_klass.call!(collection, id, cdm_endpoint)
+      cdm_api_klass.new(base_url: cdm_endpoint,
+                        collection: collection,
+                        id: id).metadata
+    end
+
+    def sets
+      @oai_request ||=
+        oai_request_klass.new(base_uri: oai_endpoint).sets
+    end
+  end
+end

data/lib/cdmbl/version.rb

@@ -1,3 +1,3 @@
 module CDMBL
-  VERSION = "0.7.2"
+  VERSION = "0.8.0"
 end

data/lib/cdmbl.rb

@@ -21,4 +21,8 @@ require 'cdmbl/oai_client'
 require 'cdmbl/oai_get_record'
 require 'cdmbl/oai_deletables'
 require 'cdmbl/batch_deleter'
-require 'cdmbl/batch_deleter_worker'
+require 'cdmbl/batch_deleter_worker'
+require 'cdmbl/compound_lookup'
+require 'cdmbl/compound_filter'
+require 'cdmbl/load_worker'
+require 'cdmbl/transform_worker'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cdmbl
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.8.0
 platform: ruby
 authors:
 - chadfennell
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-07-12 00:00:00.000000000 Z
+date: 2017-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: hash_at_path
@@ -198,6 +198,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rubocop.yml"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -210,6 +211,8 @@ files:
 - lib/cdmbl.rb
 - lib/cdmbl/batch_deleter.rb
 - lib/cdmbl/batch_deleter_worker.rb
+- lib/cdmbl/compound_filter.rb
+- lib/cdmbl/compound_lookup.rb
 - lib/cdmbl/default_cdm_notification.rb
 - lib/cdmbl/default_completed_callback.rb
 - lib/cdmbl/default_loader_notification.rb
@@ -217,11 +220,13 @@ files:
 - lib/cdmbl/default_solr.rb
 - lib/cdmbl/etl_run.rb
 - lib/cdmbl/etl_worker.rb
+- lib/cdmbl/extract_worker.rb
 - lib/cdmbl/extractor.rb
 - lib/cdmbl/field_formatter.rb
 - lib/cdmbl/field_transformer.rb
 - lib/cdmbl/formatters.rb
 - lib/cdmbl/hooks.rb
+- lib/cdmbl/load_worker.rb
 - lib/cdmbl/loader.rb
 - lib/cdmbl/oai_client.rb
 - lib/cdmbl/oai_deletables.rb
@@ -233,6 +238,8 @@ files:
 - lib/cdmbl/record_transformer.rb
 - lib/cdmbl/tasks/delete.rake
 - lib/cdmbl/tasks/etl.rake
+- lib/cdmbl/tasks/extract.rake
+- lib/cdmbl/transform_worker.rb
 - lib/cdmbl/transformer.rb
 - lib/cdmbl/version.rb
 - travis.yml