cdmdexer 0.18.0 → 0.21.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d9864c85d2d86c48decf721e05582718c7edca604b5b60a58221b66abc5632c9
-  data.tar.gz: d1cae834155be4e7595a8126130b38866744cec68178d8850f25dfb5b4e3d1d7
+  metadata.gz: '09844cd37ea7523a05b092aeaaa92383b7871a75ae1892d598aeb16a6ef050b5'
+  data.tar.gz: 11384ed2379d8ea5115f1d5d04346261a0309f96342d910f0942b1f4c224123e
 SHA512:
-  metadata.gz: 4b869bdeaee31345509c7f1e73d992ba0426f4dbf742f5c89fccb97731aa1d4e73306f0edc0e0b83f5a18eaba55f3c9bb24b2b0c6b2f2fe59bf58c629619e375
-  data.tar.gz: 8552430d8a82618e9494982a0392a06376db7ac406c3921f3960eac87d8705e3dbcc4836ca313b5b89f353ec67b6e7e9ffecab3ee28ef726d000b1fdc5c0f5af
+  metadata.gz: e5213725ac3cf7459e21ccd537c15549beb4b52ff13f64f99af3e96b172b64ab49367777b0685dc9d86ffb86dd0ac91fb3b1ebb7e1ebf77f417b4f4f1c10a467
+  data.tar.gz: e544f8e2fbc0528a31550c2946ce2ba2504996ec8894971393acafee40dadf5f3d32f1b5b202ca715fa4bd19e553ddfde7722fab4ad7266dd26502c714894442

data/.env.example

@@ -0,0 +1,2 @@
+GEONAMES_USER=foo
+GEONAMES_TOKEN=bar

data/.github/workflows/ruby.yml

@@ -0,0 +1,39 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Ruby
+
+on: [push]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ['2.6']
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+    # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
+    # change this to (see https://github.com/ruby/setup-ruby#versioning):
+    # uses: ruby/setup-ruby@v1
+      uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+    - name: Pin bundler
+      run: gem install bundler:1.17.3
+    - name: Lock bundler
+      run: bundle _1.17.3_ lock
+    - name: Run bundler
+      run: bundle install
+    - name: Run tests
+      env:
+        GEONAMES_USER: ${{ secrets.GEONAMES_USER }}
+        GEONAMES_TOKEN: ${{ secrets.GEONAMES_TOKEN }}
+      run: bundle exec rake test

data/.gitignore

@@ -8,3 +8,5 @@
 /spec/reports/
 /tmp/
 *.gem
+
+.env

data/README.md

@@ -30,9 +30,14 @@ require 'cdmdexer/rake_task'
 
 In order to make use of the GeoNames service, you must purchase a [GeoNames Premium Webservices Account](http://www.geonames.org/commercial-webservices.html). If you do not have a `geonam` field in your CONTENTdm schema, you may ignore this instruction. Add your credentials to your shell environment once you have secured a GeoNames user:
 
+
 ```
-# e.g. within your .bash_profile or .zprofile file
-export export GEONAMES_USER="yourusernamehere"
+cp .env.example .env
+nano .env
+
+# Add these vars to the .env file
+GEONAMES_USER=foo
+GEONAMES_TOKEN=bar
 ```
 
 ## Usage
@@ -116,19 +121,51 @@ You might also want to simply override some of the default mappings or add your
 ```ruby
 mappings = CDMDEXER::Transformer.default_mappings.merge(your_custom_field_mappings)
 ```
-## A Custom Post-indexing Callback
 
-If you would like to perform some action (e.g. send an email) following the completion of the CDMDEXER indexing process, you may declare your own callback hook (anything with "Callback" in the class name declared within the CDMDEXER module space will be used). To do so in Rails, create a Rails initializer file `config/initializers/cdmdexer.rb`:
+### Callbacks
+
+CDMDEXER comes with a set of lifecycle hooks that are called at various points during the ETL process. Downstream applications may want to take advantage of these in order to perform logging or notification tasks.  Create a Rails initializer at `config/initializers/cdmdexer.rb` in order to take advantage of these hooks.
+
+**IMPORTANT NOTE:**  Errors (except for http timeouts) are **not raised** but are rather sent to the `CdmError` notification hook below. This prevents sidekiq from piling-up with errors that will never resolve via retries but still allows you to capture the error and be notified of error events.
+
+E.g.:
 
 ```ruby
 module CDMDEXER
-  class Callback
-    def self.call!
-      Rails.logger.info("My Custom CDMDEXER Callback")
+  class CompletedCallback
+    def self.call!(config)
+      # e.g. commit records  - ::SolrClient.new.commit
+      Rails.logger.info "Processing last batch for: #{config['set_spec']}"
+    end
+  end
+
+  class OaiNotification
+    def self.call!(location)
+      Rails.logger.info "CDMDEXER: Requesting: #{location}"
+    end
+  end
+
+  class CdmNotification
+    def self.call!(collection, id, endpoint)
+      Rails.logger.info "CDMDEXER: Requesting: #{collection}:#{id}"
+    end
+  end
+
+  class LoaderNotification
+    def self.call!(ingestables, deletables)
+      Rails.logger.info "CDMDEXER: Loading #{ingestables.length} records and deleting #{deletables.length}"
+    end
+  end
+
+  class CdmError
+    def self.call!(error)
+      Rails.logger.info "CDMDEXER: #{error}"
+      # e.g. push error to a slack channel or send an email alert
     end
   end
 end
 ```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
@@ -146,4 +183,4 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/UMNLib
 ## TODO
 
 * Make StripFormatter the default formatter so it doesn't need to be declared for every field
-* Re-brand project: CONTENTdm Indexer. CDMDEXER doesn't necessarily require Blacklight. Moreover only handles indexing.
+* Re-brand project: CONTENTdm Indexer. CDMDEXER doesn't necessarily require Blacklight. Moreover only handles indexing.

data/cdmdexer.gemspec

@@ -1,5 +1,6 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'cdmdexer/version'
 
@@ -9,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.authors       = ['chadfennell']
   spec.email         = ['fenne035@umn.edu']
 
-  spec.summary       = %q{Load CONTENTdm data into a Solr Index. CDMDEXER expects to run inside a Rails application.}
+  spec.summary       = 'Load CONTENTdm data into a Solr Index. CDMDEXER expects to run inside a Rails application.'
   spec.license       = 'MIT'
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
@@ -17,19 +18,20 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'hash_at_path', '~> 0.1'
   spec.add_dependency 'contentdm_api', '~> 0.5.0'
+  spec.add_dependency 'hash_at_path', '~> 0.1.6'
+  spec.add_dependency 'rsolr', '~> 2.0'
   spec.add_dependency 'sidekiq', '>= 3.5'
   spec.add_dependency 'titleize', '~> 1.4'
-  spec.add_dependency 'rsolr', '~> 2.0'
   # CDMDEXER expects to run in a rails app, but just to avoid adding
   # another external dependency for XML procssing, we rely on activesupport's
   # Has.to_jsonl feature for testing and to allow this gem to function
   # independently from a rails app
-  spec.add_dependency 'rails', '>= 5.2'
+  spec.add_dependency 'rails', '~> 6.0.0'
 
+  spec.add_development_dependency 'dotenv-rails', '~> 2.7.6'
   spec.add_development_dependency 'bundler', '~> 1.12'
-  spec.add_development_dependency 'rake', '~> 12.0'
   spec.add_development_dependency 'minitest', '~> 5.0'
+  spec.add_development_dependency 'rake', '~> 12.0'
   spec.add_development_dependency 'yard', '~> 0.9.0'
 end

data/lib/cdmdexer/default_cdm_error.rb

@@ -0,0 +1,8 @@
+module CDMDEXER
+  # An example callback
+  class DefaultCdmError
+    def self.call!(error)
+      puts "CDMDEXER Error: #{error}"
+    end
+  end
+end

data/lib/cdmdexer/etl_worker.rb

@@ -38,11 +38,20 @@ module CDMDEXER
       @resumption_token  = config.fetch('resumption_token', nil)
       @batch_size        = config.fetch('batch_size', 5).to_i
       @is_recursive      = config.fetch('is_recursive', true)
+      after_date         = config.fetch('after_date', false)
 
       @oai_request = oai_request_klass.new(
         endpoint_url: oai_endpoint,
         resumption_token: resumption_token,
-        set_spec: config.fetch('set_spec', nil)
+        set_spec: config.fetch('set_spec', nil),
+        # Optionally only select records that have been updated after a
+        # certain date. You may need to manually update a parent record
+        # after updating a child in order to signify to the indexer that
+        # some record in the parent's children has been updated. This indexer
+        # expects to only see parent records in the OAI responses.
+        # The default here is to skip indexing based on date.
+        # Rails example for getting a date: `after_date: 2.weeks.ago`
+        after_date: after_date
       )
 
       run_batch!

data/lib/cdmdexer/field_transformer.rb

@@ -35,7 +35,7 @@ module CDMDEXER
     def transform_field
       formatter_klass.new(value: field_value, formatters: formatters).format!
     rescue StandardError => e
-      raise "Mapping Error:#{field_mapping.config} Error:#{e.message}"
+      raise "Mapping: #{field_mapping.config} Error:#{e.message}"
     end
   end
 end

data/lib/cdmdexer/formatters.rb

@@ -166,4 +166,4 @@ module CDMDEXER
     end
   end
 
-end
+end

data/lib/cdmdexer/hooks.rb

@@ -10,6 +10,8 @@ module CDMDEXER
       hook(pattern: name.to_s, default: DefaultLoaderNotification)
     elsif name.to_s == 'CdmNotification'
       hook(pattern: name.to_s, default: DefaultCdmNotification)
+    elsif name.to_s == 'CdmError'
+      hook(pattern: name.to_s, default: DefaultCdmError)
     end
   end
 

data/lib/cdmdexer/oai_request.rb

@@ -1,5 +1,8 @@
+# frozen_string_literal: true
+
 require 'hash_at_path'
 require 'json'
+require 'time'
 
 module CDMDEXER
   # Light wrapper around OAI requests
@@ -10,16 +13,19 @@ module CDMDEXER
     attr_reader :endpoint_url,
                 :resumption_token,
                 :client,
-                :set_spec
+                :set_spec,
+                :after_date
 
     def initialize(endpoint_url: '',
                    resumption_token: nil,
                    set_spec: nil,
-                   client: Net::HTTP)
+                   client: Net::HTTP,
+                   after_date: false)
       @endpoint_url     = endpoint_url
       @resumption_token = resumption_token
       @client           = client
       @set_spec         = set_spec ? "&set=#{set_spec}" : ''
+      @after_date       = after_date
     end
 
     def records
@@ -44,12 +50,23 @@ module CDMDEXER
     end
 
     def deletable_ids
-      records.select { |record| record['status'] == 'deleted' }
-             .map { |record| record[:id] }
+      records.select do |record|
+        if record['status'] == 'deleted'
+          after_date ? Time.parse(record['datestamp']) >= after_date : true
+        end
+      end.map { |record| record[:id] }
     end
 
     def updatables
-      records.reject { |record| record['status'] == 'deleted' }
+      records.reject do |record|
+        if record['status'] == 'deleted'
+          true
+        elsif after_date && Time.parse(record['datestamp']) < after_date
+          true
+        else
+          false
+        end
+      end
     end
 
     private

data/lib/cdmdexer/record_transformer.rb

@@ -1,22 +1,31 @@
 module CDMDEXER
+  # "Record Transformation Error: #{message}"
   class RecordTransformer
-    attr_reader :record, :field_mappings, :field_transformer
+    attr_reader :record, :field_mappings, :field_transformer, :error_klass
     def initialize(record: {},
                    field_mappings: [],
-                   field_transformer: FieldTransformer)
+                   field_transformer: FieldTransformer,
+                   error_klass: TransformationErrorMessage)
       @record            = record
       @field_mappings    = field_mappings
       @field_transformer = field_transformer
+      @error_klass = error_klass
     end
 
     def transform!
       field_mappings.inject({}) do |dest_record, field_mapping|
         dest_record.merge(transform_field(record, field_mapping))
       end
+    rescue StandardError => error
+      error_klass.new(message: message(error)).notify
     end
 
     private
 
+    def message(error)
+      "Record Transformation Error (Record #{record['id']}): #{error}"
+    end
+
     def transform_field(record, field_mapping)
       field_transformer.new(field_mapping: field_mapping,
                             record: record).reduce

data/lib/cdmdexer/transformation_error_message.rb

@@ -0,0 +1,23 @@
+module CDMDEXER
+  # Raise anything but timeout errors or other http connection errors
+  # Notify downstream in case users want to log the non-timeout errors
+  class TransformationErrorMessage
+    attr_reader :message, :notification_klass
+    def initialize(message: :MISSING_ERROR_MESSAGE,
+                   notification_klass: CDMDEXER::CdmError)
+      @notification_klass = notification_klass
+      @message = message
+    end
+
+    def notify
+      notification_klass.call! message
+      raise message if http_error?
+    end
+
+    private
+
+    def http_error?
+      !(message =~ /ConnectionError/).nil?
+    end
+  end
+end

data/lib/cdmdexer/version.rb

@@ -1,3 +1,3 @@
 module CDMDEXER
-  VERSION = "0.18.0"
-end
+  VERSION = "0.21.1"
+end

data/lib/cdmdexer.rb

@@ -24,3 +24,5 @@ require 'cdmdexer/etl_by_set_specs'
 require 'cdmdexer/regex_filter_callback'
 require 'cdmdexer/field_mapping'
 require 'cdmdexer/cdm_item'
+require 'cdmdexer/default_cdm_error'
+require 'cdmdexer/transformation_error_message'

metadata

@@ -1,43 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: cdmdexer
 version: !ruby/object:Gem::Version
-  version: 0.18.0
+  version: 0.21.1
 platform: ruby
 authors:
 - chadfennell
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-03-18 00:00:00.000000000 Z
+date: 2021-12-20 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: contentdm_api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
 - !ruby/object:Gem::Dependency
   name: hash_at_path
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: 0.1.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: 0.1.6
 - !ruby/object:Gem::Dependency
-  name: contentdm_api
+  name: rsolr
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
@@ -67,33 +81,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.4'
 - !ruby/object:Gem::Dependency
-  name: rsolr
+  name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 6.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 6.0.0
 - !ruby/object:Gem::Dependency
-  name: rails
+  name: dotenv-rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
-  type: :runtime
+        version: 2.7.6
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: 2.7.6
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -109,33 +123,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.12'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '5.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
@@ -157,6 +171,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".env.example"
+- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rubocop.yml"
 - ".travis.yml"
@@ -170,6 +186,7 @@ files:
 - cdmdexer.gemspec
 - lib/cdmdexer.rb
 - lib/cdmdexer/cdm_item.rb
+- lib/cdmdexer/default_cdm_error.rb
 - lib/cdmdexer/default_cdm_notification.rb
 - lib/cdmdexer/default_completed_callback.rb
 - lib/cdmdexer/default_loader_notification.rb
@@ -193,6 +210,7 @@ files:
 - lib/cdmdexer/tasks/delete.rake
 - lib/cdmdexer/tasks/etl.rake
 - lib/cdmdexer/transform_worker.rb
+- lib/cdmdexer/transformation_error_message.rb
 - lib/cdmdexer/transformer.rb
 - lib/cdmdexer/version.rb
 - travis.yml