cdmdexer 0.19.0 → 0.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 59d43d9c22b7fe79a88b39a9d44909f41062cbe24f97daa2d9458c70de3a7c56
-  data.tar.gz: cf687f21c8f8740dba42eb9b86d8272df812ea7456bfae6eb1314ec5f270189b
+  metadata.gz: 60ef91c0ea6d70b0242465b27945c265175312254e540b40a2ba15ffbb609413
+  data.tar.gz: dce7221c4a962bad08f35ef39c2bf4bd0288650d97b2b1c3d1d6751226bc7a14
 SHA512:
-  metadata.gz: f75efb0c7b642a71a6231eaa07fd6a7fe8ffa994e90175a0e7ee953795500458adf3e23b6f9461b2116144494d331e221ac7c63c717bd1bb55ab4efd981cf561
-  data.tar.gz: 2d9817ad1099ac49117f6bde773134c66b0da166ab5928ca513226996a93279b3d49f18b4905de17db751674c5219cbf5acec44e36f85f156981038d3ae21c33
+  metadata.gz: fd92890578c5e3c9642766b605b381219f789854918c9895ef3d8a895979dc5bde6a3e2522fc21b03893d304a4660914c45d080cb67066b10a998d3d15897d1a
+  data.tar.gz: 05d038f8b6149dbfe7500dced962f853c7f3a2ed49821d253d02e71baf0af82dfb4ac9cfc526de4dc4e5cca5552712c8b955ec9a532c437ed9d0570852a4cc70

data/README.md

@@ -116,19 +116,51 @@ You might also want to simply override some of the default mappings or add your
 ```ruby
 mappings = CDMDEXER::Transformer.default_mappings.merge(your_custom_field_mappings)
 ```
-## A Custom Post-indexing Callback
 
-If you would like to perform some action (e.g. send an email) following the completion of the CDMDEXER indexing process, you may declare your own callback hook (anything with "Callback" in the class name declared within the CDMDEXER module space will be used). To do so in Rails, create a Rails initializer file `config/initializers/cdmdexer.rb`:
+### Callbacks
+
+CDMDEXER comes with a set of lifecycle hooks that are called at various points during the ETL process. Downstream applications may want to take advantage of these in order to perform logging or notification tasks.  Create a Rails initializer at `config/initializers/cdmdexer.rb` in order to take advantage of these hooks.
+
+**IMPORTANT NOTE:**  Errors (except for http timeouts) are **not raised** but are rather sent to the `CdmError` notification hook below. This prevents sidekiq from piling-up with errors that will never resolve via retries but still allows you to capture the error and be notified of error events.
+
+E.g.:
 
 ```ruby
 module CDMDEXER
-  class Callback
-    def self.call!
-      Rails.logger.info("My Custom CDMDEXER Callback")
+  class CompletedCallback
+    def self.call!(config)
+      # e.g. commit records  - ::SolrClient.new.commit
+      Rails.logger.info "Processing last batch for: #{config['set_spec']}"
+    end
+  end
+
+  class OaiNotification
+    def self.call!(location)
+      Rails.logger.info "CDMDEXER: Requesting: #{location}"
+    end
+  end
+
+  class CdmNotification
+    def self.call!(collection, id, endpoint)
+      Rails.logger.info "CDMDEXER: Requesting: #{collection}:#{id}"
+    end
+  end
+
+  class LoaderNotification
+    def self.call!(ingestables, deletables)
+      Rails.logger.info "CDMDEXER: Loading #{ingestables.length} records and deleting #{deletables.length}"
+    end
+  end
+
+  class CdmError
+    def self.call!(error)
+      Rails.logger.info "CDMDEXER: #{error}"
+      # e.g. push error to a slack channel or send an email alert
     end
   end
 end
 ```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/cdmdexer.rb

@@ -24,3 +24,5 @@ require 'cdmdexer/etl_by_set_specs'
 require 'cdmdexer/regex_filter_callback'
 require 'cdmdexer/field_mapping'
 require 'cdmdexer/cdm_item'
+require 'cdmdexer/default_cdm_error'
+require 'cdmdexer/transformation_error_message'

data/lib/cdmdexer/default_cdm_error.rb

@@ -0,0 +1,8 @@
+module CDMDEXER
+  # An example callback
+  class DefaultCdmError
+    def self.call!(error)
+      puts "CDMDEXER Error: #{error}"
+    end
+  end
+end

data/lib/cdmdexer/field_transformer.rb

@@ -4,12 +4,11 @@ module CDMDEXER
   class FieldTransformer
     extend Forwardable
     def_delegators :@field_mapping, :origin_path, :dest_path, :formatters
-    attr_reader :field_value, :field_mapping, :formatter_klass, :record_id
+    attr_reader :field_value, :field_mapping, :formatter_klass
     def initialize(field_mapping: FieldMapping.new,
                    record: {},
                    formatter_klass: FieldFormatter)
       @field_mapping   = field_mapping
-      @record_id       = record.fetch('id', 'MISSING_RECORD_ID')
       @field_value     = compact(record.at_path(origin_path))
       @formatter_klass = formatter_klass
     end
@@ -36,7 +35,7 @@ module CDMDEXER
     def transform_field
       formatter_klass.new(value: field_value, formatters: formatters).format!
     rescue StandardError => e
-      raise "FieldTransformer Mapping Error - Record: #{record_id} Mapping: #{field_mapping.config} Error:#{e.message}"
+      raise "Mapping: #{field_mapping.config} Error:#{e.message}"
     end
   end
 end

data/lib/cdmdexer/hooks.rb

@@ -10,6 +10,8 @@ module CDMDEXER
       hook(pattern: name.to_s, default: DefaultLoaderNotification)
     elsif name.to_s == 'CdmNotification'
       hook(pattern: name.to_s, default: DefaultCdmNotification)
+    elsif name.to_s == 'CdmError'
+      hook(pattern: name.to_s, default: DefaultCdmError)
     end
   end
 

data/lib/cdmdexer/record_transformer.rb

@@ -1,22 +1,31 @@
 module CDMDEXER
+  # "Record Transformation Error: #{message}"
   class RecordTransformer
-    attr_reader :record, :field_mappings, :field_transformer
+    attr_reader :record, :field_mappings, :field_transformer, :error_klass
     def initialize(record: {},
                    field_mappings: [],
-                   field_transformer: FieldTransformer)
+                   field_transformer: FieldTransformer,
+                   error_klass: TransformationErrorMessage)
       @record            = record
       @field_mappings    = field_mappings
       @field_transformer = field_transformer
+      @error_klass = error_klass
     end
 
     def transform!
       field_mappings.inject({}) do |dest_record, field_mapping|
         dest_record.merge(transform_field(record, field_mapping))
       end
+    rescue StandardError => error
+      error_klass.new(message: message(error)).notify
     end
 
     private
 
+    def message(error)
+      "Record Transformation Error (Record #{record['id']}): #{error}"
+    end
+
     def transform_field(record, field_mapping)
       field_transformer.new(field_mapping: field_mapping,
                             record: record).reduce

data/lib/cdmdexer/transformation_error_message.rb

@@ -0,0 +1,23 @@
+module CDMDEXER
+  # Raise anything but timeout errors or other http connection errors
+  # Notify downstream in case users want to log the non-timeout errors
+  class TransformationErrorMessage
+    attr_reader :message, :notification_klass
+    def initialize(message: :MISSING_ERROR_MESSAGE,
+                   notification_klass: CDMDEXER::CdmError)
+      @notification_klass = notification_klass
+      @message = message
+    end
+
+    def notify
+      notification_klass.call! message
+      raise message if http_error?
+    end
+
+    private
+
+    def http_error?
+      !(message =~ /ConnectionError/).nil?
+    end
+  end
+end

data/lib/cdmdexer/version.rb

@@ -1,3 +1,3 @@
 module CDMDEXER
-  VERSION = "0.19.0"
+  VERSION = "0.20.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cdmdexer
 version: !ruby/object:Gem::Version
-  version: 0.19.0
+  version: 0.20.0
 platform: ruby
 authors:
 - chadfennell
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-07-09 00:00:00.000000000 Z
+date: 2019-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: hash_at_path
@@ -170,6 +170,7 @@ files:
 - cdmdexer.gemspec
 - lib/cdmdexer.rb
 - lib/cdmdexer/cdm_item.rb
+- lib/cdmdexer/default_cdm_error.rb
 - lib/cdmdexer/default_cdm_notification.rb
 - lib/cdmdexer/default_completed_callback.rb
 - lib/cdmdexer/default_loader_notification.rb
@@ -193,6 +194,7 @@ files:
 - lib/cdmdexer/tasks/delete.rake
 - lib/cdmdexer/tasks/etl.rake
 - lib/cdmdexer/transform_worker.rb
+- lib/cdmdexer/transformation_error_message.rb
 - lib/cdmdexer/transformer.rb
 - lib/cdmdexer/version.rb
 - travis.yml
@@ -215,8 +217,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.8
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Load CONTENTdm data into a Solr Index. CDMDEXER expects to run inside a Rails