cdmdexer 0.17.7 → 0.21.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8d3dccb39ef4048b79af386f1d3696f9d7497efb3c53e388480e1505e027d99b
-  data.tar.gz: 262051a8e4e246be6092a5cbc54245cbed91449c859f5c408bde2f7b8d9ae068
+  metadata.gz: ea1e0a2b54ce8d063a2d84d18987d662ef583abf044142356a46d5bcd85790f5
+  data.tar.gz: a9012a9ae4fcee9bdd37314a0acd5c19238c60691ff60a95668329c89e5c650b
 SHA512:
-  metadata.gz: 41aeca2b754fc5681e5bfe120e4d690e4f56f5b767bd924f50a0363ca2fdfad2238e2c58507d61e79805c37e0218267903e662c968768c5d20fce807d98cecc9
-  data.tar.gz: 2a99d19a2639e15b5811692101d9630a67296b9ee1a32bda4e1cf70be5dcf4a0aba8815602b9172b935b8bf1482531026f5ebc8ee5289439a4baee76a7b6873c
+  metadata.gz: bf5450e86279e1e3a16fc8aa862f69ac865047bb4a9dc1a067a547152437c9d849acc2cf78f1107b664d4b930c8140913bad5cb0d21a35ada2e720465427a187
+  data.tar.gz: 5ae4ef6681742ec8bd883a388d571c08057e58f23044b1c09278b305159add9aec4ddde656024c813e0afb7c4427bab06d81286874c4c7fe104a2a937134d40e

data/README.md

@@ -116,19 +116,51 @@ You might also want to simply override some of the default mappings or add your
 ```ruby
 mappings = CDMDEXER::Transformer.default_mappings.merge(your_custom_field_mappings)
 ```
-## A Custom Post-indexing Callback
 
-If you would like to perform some action (e.g. send an email) following the completion of the CDMDEXER indexing process, you may declare your own callback hook (anything with "Callback" in the class name declared within the CDMDEXER module space will be used). To do so in Rails, create a Rails initializer file `config/initializers/cdmdexer.rb`:
+### Callbacks
+
+CDMDEXER comes with a set of lifecycle hooks that are called at various points during the ETL process. Downstream applications may want to take advantage of these in order to perform logging or notification tasks.  Create a Rails initializer at `config/initializers/cdmdexer.rb` in order to take advantage of these hooks.
+
+**IMPORTANT NOTE:**  Errors (except for http timeouts) are **not raised** but are rather sent to the `CdmError` notification hook below. This prevents sidekiq from piling-up with errors that will never resolve via retries but still allows you to capture the error and be notified of error events.
+
+E.g.:
 
 ```ruby
 module CDMDEXER
-  class Callback
-    def self.call!
-      Rails.logger.info("My Custom CDMDEXER Callback")
+  class CompletedCallback
+    def self.call!(config)
+      # e.g. commit records  - ::SolrClient.new.commit
+      Rails.logger.info "Processing last batch for: #{config['set_spec']}"
+    end
+  end
+
+  class OaiNotification
+    def self.call!(location)
+      Rails.logger.info "CDMDEXER: Requesting: #{location}"
+    end
+  end
+
+  class CdmNotification
+    def self.call!(collection, id, endpoint)
+      Rails.logger.info "CDMDEXER: Requesting: #{collection}:#{id}"
+    end
+  end
+
+  class LoaderNotification
+    def self.call!(ingestables, deletables)
+      Rails.logger.info "CDMDEXER: Loading #{ingestables.length} records and deleting #{deletables.length}"
+    end
+  end
+
+  class CdmError
+    def self.call!(error)
+      Rails.logger.info "CDMDEXER: #{error}"
+      # e.g. push error to a slack channel or send an email alert
     end
   end
 end
 ```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/cdmdexer.gemspec

@@ -1,5 +1,6 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'cdmdexer/version'
 
@@ -9,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.authors       = ['chadfennell']
   spec.email         = ['fenne035@umn.edu']
 
-  spec.summary       = %q{Load CONTENTdm data into a Solr Index. CDMDEXER expects to run inside a Rails application.}
+  spec.summary       = 'Load CONTENTdm data into a Solr Index. CDMDEXER expects to run inside a Rails application.'
   spec.license       = 'MIT'
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
@@ -17,11 +18,11 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'hash_at_path', '~> 0.1'
   spec.add_dependency 'contentdm_api', '~> 0.5.0'
+  spec.add_dependency 'hash_at_path', '~> 0.1.6'
+  spec.add_dependency 'rsolr', '~> 2.0'
   spec.add_dependency 'sidekiq', '>= 3.5'
   spec.add_dependency 'titleize', '~> 1.4'
-  spec.add_dependency 'rsolr', '~> 2.0'
   # CDMDEXER expects to run in a rails app, but just to avoid adding
   # another external dependency for XML procssing, we rely on activesupport's
   # Has.to_jsonl feature for testing and to allow this gem to function
@@ -29,7 +30,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'rails', '>= 5.2'
 
   spec.add_development_dependency 'bundler', '~> 1.12'
-  spec.add_development_dependency 'rake', '~> 12.0'
   spec.add_development_dependency 'minitest', '~> 5.0'
+  spec.add_development_dependency 'rake', '~> 12.0'
   spec.add_development_dependency 'yard', '~> 0.9.0'
 end

data/lib/cdmdexer.rb

@@ -23,4 +23,6 @@ require 'cdmdexer/filtered_set_specs'
 require 'cdmdexer/etl_by_set_specs'
 require 'cdmdexer/regex_filter_callback'
 require 'cdmdexer/field_mapping'
-require 'cdmdexer/cdm_item'
+require 'cdmdexer/cdm_item'
+require 'cdmdexer/default_cdm_error'
+require 'cdmdexer/transformation_error_message'

data/lib/cdmdexer/default_cdm_error.rb

@@ -0,0 +1,8 @@
+module CDMDEXER
+  # An example callback
+  class DefaultCdmError
+    def self.call!(error)
+      puts "CDMDEXER Error: #{error}"
+    end
+  end
+end

data/lib/cdmdexer/etl_worker.rb

@@ -38,11 +38,20 @@ module CDMDEXER
       @resumption_token  = config.fetch('resumption_token', nil)
       @batch_size        = config.fetch('batch_size', 5).to_i
       @is_recursive      = config.fetch('is_recursive', true)
+      after_date         = config.fetch('after_date', false)
 
       @oai_request = oai_request_klass.new(
         endpoint_url: oai_endpoint,
         resumption_token: resumption_token,
-        set_spec: config.fetch('set_spec', nil)
+        set_spec: config.fetch('set_spec', nil),
+        # Optionally only select records that have been updated after a
+        # certain date. You may need to manually update a parent record
+        # after updating a child in order to signify to the indexer that
+        # some record in the parent's children has been updated. This indexer
+        # expects to only see parent records in the OAI responses.
+        # The default here is to skip indexing based on date.
+        # Rails example for getting a date: `after_date: 2.weeks.ago`
+        after_date: after_date
       )
 
       run_batch!
@@ -77,7 +86,7 @@ module CDMDEXER
       if next_resumption_token && is_recursive
         etl_worker_klass.perform_async(next_config)
       else
-        completed_callback_klass.call!(solr_config)
+        completed_callback_klass.call!(config)
       end
     end
 

data/lib/cdmdexer/field_transformer.rb

@@ -35,7 +35,7 @@ module CDMDEXER
     def transform_field
       formatter_klass.new(value: field_value, formatters: formatters).format!
     rescue StandardError => e
-      raise "Mapping Error:#{field_mapping.config} Error:#{e.message}"
+      raise "Mapping: #{field_mapping.config} Error:#{e.message}"
     end
   end
 end

data/lib/cdmdexer/formatters.rb

@@ -120,7 +120,7 @@ module CDMDEXER
 
   class AddSetSpecFormatter
     def self.format(value)
-      value.merge('setSpec' => value['id'].split('/').first)
+      value.merge('setSpec' => value['id'].split(':').first)
     end
   end
 

data/lib/cdmdexer/hooks.rb

@@ -10,6 +10,8 @@ module CDMDEXER
       hook(pattern: name.to_s, default: DefaultLoaderNotification)
     elsif name.to_s == 'CdmNotification'
       hook(pattern: name.to_s, default: DefaultCdmNotification)
+    elsif name.to_s == 'CdmError'
+      hook(pattern: name.to_s, default: DefaultCdmError)
     end
   end
 

data/lib/cdmdexer/oai_request.rb

@@ -1,5 +1,8 @@
+# frozen_string_literal: true
+
 require 'hash_at_path'
 require 'json'
+require 'time'
 
 module CDMDEXER
   # Light wrapper around OAI requests
@@ -10,16 +13,19 @@ module CDMDEXER
     attr_reader :endpoint_url,
                 :resumption_token,
                 :client,
-                :set_spec
+                :set_spec,
+                :after_date
 
     def initialize(endpoint_url: '',
                    resumption_token: nil,
                    set_spec: nil,
-                   client: Net::HTTP)
+                   client: Net::HTTP,
+                   after_date: false)
       @endpoint_url     = endpoint_url
       @resumption_token = resumption_token
       @client           = client
       @set_spec         = set_spec ? "&set=#{set_spec}" : ''
+      @after_date       = after_date
     end
 
     def records
@@ -44,12 +50,23 @@ module CDMDEXER
     end
 
     def deletable_ids
-      records.select { |record| record['status'] == 'deleted' }
-             .map { |record| record[:id] }
+      records.select do |record|
+        if record['status'] == 'deleted'
+          after_date ? Time.parse(record['datestamp']) >= after_date : true
+        end
+      end.map { |record| record[:id] }
     end
 
     def updatables
-      records.reject { |record| record['status'] == 'deleted' }
+      records.reject do |record|
+        if record['status'] == 'deleted'
+          true
+        elsif after_date && Time.parse(record['datestamp']) < after_date
+          true
+        else
+          false
+        end
+      end
     end
 
     private
@@ -62,7 +79,7 @@ module CDMDEXER
     # Ensure results are a single level array
     # (single row sets, records, etc)
     def force_array(result)
-      [result].flatten
+      [result].flatten.compact
     end
 
     def to_key(set)

data/lib/cdmdexer/record_transformer.rb

@@ -1,22 +1,31 @@
 module CDMDEXER
+  # "Record Transformation Error: #{message}"
   class RecordTransformer
-    attr_reader :record, :field_mappings, :field_transformer
+    attr_reader :record, :field_mappings, :field_transformer, :error_klass
     def initialize(record: {},
                    field_mappings: [],
-                   field_transformer: FieldTransformer)
+                   field_transformer: FieldTransformer,
+                   error_klass: TransformationErrorMessage)
       @record            = record
       @field_mappings    = field_mappings
       @field_transformer = field_transformer
+      @error_klass = error_klass
     end
 
     def transform!
       field_mappings.inject({}) do |dest_record, field_mapping|
         dest_record.merge(transform_field(record, field_mapping))
       end
+    rescue StandardError => error
+      error_klass.new(message: message(error)).notify
     end
 
     private
 
+    def message(error)
+      "Record Transformation Error (Record #{record['id']}): #{error}"
+    end
+
     def transform_field(record, field_mapping)
       field_transformer.new(field_mapping: field_mapping,
                             record: record).reduce

data/lib/cdmdexer/transformation_error_message.rb

@@ -0,0 +1,23 @@
+module CDMDEXER
+  # Raise anything but timeout errors or other http connection errors
+  # Notify downstream in case users want to log the non-timeout errors
+  class TransformationErrorMessage
+    attr_reader :message, :notification_klass
+    def initialize(message: :MISSING_ERROR_MESSAGE,
+                   notification_klass: CDMDEXER::CdmError)
+      @notification_klass = notification_klass
+      @message = message
+    end
+
+    def notify
+      notification_klass.call! message
+      raise message if http_error?
+    end
+
+    private
+
+    def http_error?
+      !(message =~ /ConnectionError/).nil?
+    end
+  end
+end

data/lib/cdmdexer/version.rb

@@ -1,3 +1,3 @@
 module CDMDEXER
-  VERSION = "0.17.7"
+  VERSION = "0.21.0"
 end

metadata

@@ -1,85 +1,85 @@
 --- !ruby/object:Gem::Specification
 name: cdmdexer
 version: !ruby/object:Gem::Version
-  version: 0.17.7
+  version: 0.21.0
 platform: ruby
 authors:
 - chadfennell
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-03-13 00:00:00.000000000 Z
+date: 2020-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: hash_at_path
+  name: contentdm_api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: 0.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: 0.5.0
 - !ruby/object:Gem::Dependency
-  name: contentdm_api
+  name: hash_at_path
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 0.1.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 0.1.6
 - !ruby/object:Gem::Dependency
-  name: sidekiq
+  name: rsolr
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: titleize
+  name: sidekiq
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '3.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '3.5'
 - !ruby/object:Gem::Dependency
-  name: rsolr
+  name: titleize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
@@ -109,33 +109,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.12'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '5.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
@@ -170,6 +170,7 @@ files:
 - cdmdexer.gemspec
 - lib/cdmdexer.rb
 - lib/cdmdexer/cdm_item.rb
+- lib/cdmdexer/default_cdm_error.rb
 - lib/cdmdexer/default_cdm_notification.rb
 - lib/cdmdexer/default_completed_callback.rb
 - lib/cdmdexer/default_loader_notification.rb
@@ -193,6 +194,7 @@ files:
 - lib/cdmdexer/tasks/delete.rake
 - lib/cdmdexer/tasks/etl.rake
 - lib/cdmdexer/transform_worker.rb
+- lib/cdmdexer/transformation_error_message.rb
 - lib/cdmdexer/transformer.rb
 - lib/cdmdexer/version.rb
 - travis.yml
@@ -215,7 +217,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.6
 signing_key: 
 specification_version: 4
 summary: Load CONTENTdm data into a Solr Index. CDMDEXER expects to run inside a Rails