cdmbl 0.14.0 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 417415def1b0f28a66e6640201a3c53becf43737
-  data.tar.gz: 36d22cc2acaab58453bb908877af79f2fe93ddba
+SHA256:
+  metadata.gz: 98471f931e9b4535c9f1019cfe954cfba9eb7e9742fdb4f2eda70fe914d4fd3e
+  data.tar.gz: 32524fe0f8ae2d4ea5b45954d0b368a46153b9f7b3298cb064a5202cd75049ee
 SHA512:
-  metadata.gz: 6a4a95dbdf25f2cfd26d2d037b900d07850bf4be2c50112319bcfd220cbddcf7e0325b62ff45267b9cddb0fc135291d946509e8bfe5a78486c5231e1665f7354
-  data.tar.gz: bc2795036b04aed45ac8be18829450001c08091b2df7c7af089b7faf1779d422ffb14e0610d5a9c3306b981bcbedcb584ef3b89b70a8fcdf94a81ff6dcd55752
+  metadata.gz: 6205cbb83cc23192f3157341e4980b49b4bd6cb89a615ddec3ae13d829f26738d293e6cb174e56741c6fe1406c501568896fdf3ca148be55743b0318452c75ff
+  data.tar.gz: c9b9f37bc27e3a7d21fb30035310dae9109e5f0d73d8bf8a45f7b0f175251766e06185de27a03763a50eb8d034777437f910b73e2446e388c98f7371c3a04d8d

data/cdmbl.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'hash_at_path', '~> 0.1'
-  spec.add_dependency 'contentdm_api', '~> 0.3.11'
+  spec.add_dependency 'contentdm_api', '~> 0.5.0'
   spec.add_dependency 'sidekiq', '>= 3.5'
   spec.add_dependency 'titleize', '~> 1.4'
   spec.add_dependency 'rsolr', '~> 2.0'
@@ -32,6 +32,4 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake', '~> 12.0'
   spec.add_development_dependency 'minitest', '~> 5.0'
   spec.add_development_dependency 'yard', '~> 0.9.0'
-  spec.add_development_dependency 'webmock', '~> 1.24', '>= 1.24.0'
-  spec.add_development_dependency 'vcr', '~> 3.0', '>= 3.0.1'
 end

data/lib/cdmbl/cdm_request_worker.rb

@@ -0,0 +1,90 @@
+require 'sidekiq'
+module CDMBL
+  class CdmRequestWorker
+    include Sidekiq::Worker
+    attr_reader :collection,
+                :id,
+                :set_lookup,
+                :field_mappings
+
+    attr_writer :cdm_api_klass,
+                  :oai_request_klass,
+                  :oai_set_lookup_klass,
+                  :cdm_notification_klass,
+                  :load_worker_klass,
+                  :transformer_klass
+
+    def perform(collection, id)
+
+      @identifiers       = identifiers
+      @solr_config       = solr_config
+      @cdm_endpoint      = cdm_endpoint
+      @oai_endpoint      = oai_endpoint
+      @field_mappings    = field_mappings
+      @extract_compounds = extract_compounds
+      transform_and_load!
+    end
+
+    def oai_set_lookup_klass
+      @oai_set_lookup_klass ||= OAISetLookup
+    end
+
+    def oai_request_klass
+      @oai_request_klass ||= OaiRequest
+    end
+
+    def cdm_api_klass
+      @cdm_api_klass ||= CONTENTdmAPI::Item
+    end
+
+    def cdm_notification_klass
+      @cdm_notification_klass ||= CdmNotification
+    end
+
+    def transformer_klass
+      @transformer_klass ||= Transformer
+    end
+
+    def load_worker_klass
+      @load_worker_klass ||= LoadWorker
+    end
+
+    private
+
+    def transform_and_load!
+      load_worker_klass.perform_async(transformed_records, [], solr_config)
+    end
+
+    def transformed_records
+      @transformation ||=
+        transformer_klass.new(cdm_records: records,
+                              oai_sets: set_lookup,
+                              field_mappings: field_mappings,
+                              extract_compounds: extract_compounds).records
+    end
+
+    def set_lookup
+      oai_set_lookup_klass.new(oai_sets: sets).keyed
+    end
+
+    def records
+      identifiers.map do |identifier|
+        cdm_request(*identifier)
+      end
+    end
+
+    # e.g. local_identifiers.map { |identifier| extractor.cdm_request(*identifier) }
+    def cdm_request(collection, id)
+      cdm_notification_klass.call!(collection, id, cdm_endpoint)
+      cdm_api_klass.new(base_url: cdm_endpoint,
+                        collection: collection,
+                        with_compounds: false,
+                        id: id)
+    end
+
+    def sets
+      @oai_request ||=
+        oai_request_klass.new(base_uri: oai_endpoint).sets
+    end
+  end
+end

data/lib/cdmbl/etl_worker.rb

@@ -16,7 +16,8 @@ module CDMBL
                 :set_spec,
                 :max_compounds,
                 :batch_size,
-                :is_recursive
+                :is_recursive,
+                :from
 
     attr_writer :compound_filter_klass,
                 :extractor_klass,
@@ -40,6 +41,7 @@ module CDMBL
       @max_compounds     = config.fetch('max_compounds', 10)
       @batch_size        = config.fetch('batch_size', 5).to_i
       @is_recursive      = config.fetch('is_recursive', true)
+      @from              = config.fetch('from', nil)
       extract_batch!
       next_batch!
     end
@@ -114,12 +116,14 @@ module CDMBL
     end
 
     def transform!(ids)
-      transform_worker_klass.perform_async(ids,
-                                           solr_config,
-                                           cdm_endpoint,
-                                           oai_endpoint,
-                                           field_mappings,
-                                           extract_compounds)
+      transform_worker_klass.perform_async(
+        ids,
+        solr_config,
+        cdm_endpoint,
+        oai_endpoint,
+        field_mappings,
+        extract_compounds
+      )
     end
 
     def delete_deletables!
@@ -127,17 +131,20 @@ module CDMBL
     end
 
     def compound_filter
-      @compound_filter ||=
-        compound_filter_klass.new(record_ids: extraction.local_identifiers,
-                                  cdm_endpoint: cdm_endpoint,
-                                  max_compounds: max_compounds)
+      @compound_filter ||= compound_filter_klass.new(
+        record_ids: extraction.local_identifiers,
+        cdm_endpoint: cdm_endpoint,
+        max_compounds: max_compounds
+      )
     end
 
     def extraction
-      @extraction ||=
-        extractor_klass.new(oai_endpoint: oai_endpoint,
-                            resumption_token: resumption_token,
-                            set_spec: set_spec)
+      @extraction ||= extractor_klass.new(
+        oai_endpoint: oai_endpoint,
+        resumption_token: resumption_token,
+        set_spec: set_spec,
+        from: from
+      )
     end
   end
 end

data/lib/cdmbl/extractor.rb

@@ -16,6 +16,7 @@ module CDMBL
     def initialize(oai_endpoint: '',
                    resumption_token: nil,
                    set_spec: nil,
+                   from: nil,
                    oai_request_klass: OaiRequest,
                    oai_filter_klass: OAIFilter,
                    oai_set_lookup_klass: OAISetLookup)
@@ -24,7 +25,8 @@ module CDMBL
       @oai_set_lookup_klass = oai_set_lookup_klass
       @oai_request          = oai_requester(oai_endpoint,
                                             resumption_token,
-                                            set_spec)
+                                            set_spec,
+                                            from)
     end
 
     def deletable_ids
@@ -49,11 +51,16 @@ module CDMBL
 
     private
 
-    def oai_requester(oai_endpoint, resumption_token, set_spec)
-      @oai_requester ||=
-        oai_request_klass.new(base_uri: oai_endpoint,
-                              resumption_token: resumption_token,
-                              set: set_spec)
+    def oai_requester(oai_endpoint, resumption_token, set_spec, from)
+      @oai_requester ||= begin
+        args = {
+          base_uri: oai_endpoint,
+          resumption_token: resumption_token,
+          set: set_spec,
+        }
+        args[:from] = from if from
+        oai_request_klass.new(args)
+      end
     end
 
     # Get the local collection and id from an OAI namespaced identifier

data/lib/cdmbl/formatters.rb

@@ -72,7 +72,7 @@ module CDMBL
   class Titlieze
     def self.format(value)
       if value.respond_to?(:map)
-        value.map {|value| value.titleize }
+        value.map(&:titleize)
       else
         value.titleize
       end
@@ -172,4 +172,4 @@ module CDMBL
     end
   end
 
-end
+end

data/lib/cdmbl/load_worker.rb

@@ -3,6 +3,7 @@ module CDMBL
   # Load Records into a solr index
   class LoadWorker
     include Sidekiq::Worker
+    sidekiq_options queue: 'critical'
     attr_reader :solr_config, :records, :deletables
     attr_writer :loader_klass, :solr_klass
     def perform(records = [], deletables = [], solr_config = {})

data/lib/cdmbl/oai_client.rb

@@ -1,10 +1,12 @@
+
 require 'json'
+require 'http'
 module CDMBL
     class OaiClient
-      attr_reader :base_url, :http_client
-      def initialize(base_url: '', http_client: Net::HTTP)
+      attr_reader :base_url, :client
+      def initialize(base_url: '', client: HTTP)
         @base_url    = base_url
-        @http_client = http_client
+        @client = client
       end
 
       def request(query)
@@ -14,7 +16,7 @@ module CDMBL
       private
 
       def get(url)
-        http_client.get_response(URI(url)).body
+        client.get(url).to_s
       end
 
       def hashify(xml)

data/lib/cdmbl/oai_request.rb

@@ -1,48 +1,51 @@
 require 'json'
 module CDMBL
-    class OaiRequest
-      attr_reader :base_uri,
-                  :resumption_token,
-                  :client,
-                  :set,
-                  :identifier
-      def initialize(base_uri: '',
-                     resumption_token: nil,
-                     set: nil,
-                     identifier: '',
-                     client: Net::HTTP)
-          @base_uri         = base_uri
-          @resumption_token = resumption_token
-          @client           = client
-          @set              = (set) ? "&set=#{set}" : ''
-          @identifier       = identifier
-      end
+  class OaiRequest
+    attr_reader :base_uri,
+                :resumption_token,
+                :client,
+                :set,
+                :identifier,
+                :from
+    def initialize(base_uri: '',
+                   resumption_token: nil,
+                   set: nil,
+                   identifier: '',
+                   from: nil,
+                   client: Net::HTTP)
+      @base_uri         = base_uri
+      @resumption_token = resumption_token
+      @client           = client
+      @set              = (set) ? "&set=#{set}" : ''
+      @from             = from ? "&from=#{from}" : ''
+      @identifier       = identifier
+    end
 
-      def identifiers
-        @ids ||= (resumption_token) ? request(batch_uri) : request(first_batch_uri)
-      end
+    def identifiers
+      @ids ||= (resumption_token) ? request(batch_uri) : request(first_batch_uri)
+    end
 
-      def sets
-        @sets ||= request(sets_uri)
-      end
+    def sets
+      @sets ||= request(sets_uri)
+    end
 
-      private
+    private
 
-      def first_batch_uri
-        "#{base_uri}?verb=ListIdentifiers&metadataPrefix=oai_dc#{set}"
-      end
+    def first_batch_uri
+      "#{base_uri}?verb=ListIdentifiers&metadataPrefix=oai_dc#{set}#{from}"
+    end
 
-      def batch_uri
-        "#{base_uri}?verb=ListIdentifiers&resumptionToken=#{resumption_token}"
-      end
+    def batch_uri
+      "#{base_uri}?verb=ListIdentifiers&resumptionToken=#{resumption_token}"
+    end
 
-      def sets_uri
-        "#{base_uri}?verb=ListSets"
-      end
+    def sets_uri
+      "#{base_uri}?verb=ListSets"
+    end
 
-      def request(location)
-        CDMBL::OaiNotification.call!(location)
-        Hash.from_xml(client.get_response(URI(location)).body)
-      end
+    def request(location)
+      CDMBL::OaiNotification.call!(location)
+      Hash.from_xml(client.get_response(URI(location)).body)
     end
-end
+  end
+end

data/lib/cdmbl/transform_worker.rb

@@ -10,11 +10,12 @@ module CDMBL
                 :extract_compounds
 
     attr_writer :cdm_api_klass,
-                  :oai_request_klass,
-                  :oai_set_lookup_klass,
-                  :cdm_notification_klass,
-                  :load_worker_klass,
-                  :transformer_klass
+                :oai_request_klass,
+                :oai_set_lookup_klass,
+                :cdm_notification_klass,
+                :load_worker_klass,
+                :transformer_klass,
+                :cache_klass
 
     def perform(identifiers,
                 solr_config,
@@ -56,6 +57,10 @@ module CDMBL
       @load_worker_klass ||= LoadWorker
     end
 
+    def cache_klass
+      @cache_klass ||= Rails
+    end
+
     private
 
     def transform_and_load!
@@ -90,7 +95,9 @@ module CDMBL
 
     def sets
       @oai_request ||=
-        oai_request_klass.new(base_uri: oai_endpoint).sets
+        cache_klass.cache.fetch("cdmbl_set_specs", expires_in: 10.minutes) do
+          oai_request_klass.new(base_uri: oai_endpoint).sets
+        end
     end
   end
 end

data/lib/cdmbl/version.rb

@@ -1,3 +1,3 @@
 module CDMBL
-  VERSION = "0.14.0"
-end
+  VERSION = "0.18.0"
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cdmbl
 version: !ruby/object:Gem::Version
-  version: 0.14.0
+  version: 0.18.0
 platform: ruby
 authors:
 - chadfennell
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-01-07 00:00:00.000000000 Z
+date: 2021-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: hash_at_path
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.11
+        version: 0.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.11
+        version: 0.5.0
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
@@ -150,47 +150,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.0
-- !ruby/object:Gem::Dependency
-  name: webmock
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.24'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.24.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.24'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.24.0
-- !ruby/object:Gem::Dependency
-  name: vcr
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.0.1
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.0.1
-description: 
+description:
 email:
 - fenne035@umn.edu
 executables: []
@@ -211,6 +171,7 @@ files:
 - lib/cdmbl.rb
 - lib/cdmbl/batch_deleter.rb
 - lib/cdmbl/batch_deleter_worker.rb
+- lib/cdmbl/cdm_request_worker.rb
 - lib/cdmbl/compound_filter.rb
 - lib/cdmbl/compound_lookup.rb
 - lib/cdmbl/default_cdm_notification.rb
@@ -245,11 +206,11 @@ files:
 - lib/cdmbl/transformer.rb
 - lib/cdmbl/version.rb
 - travis.yml
-homepage: 
+homepage:
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -264,9 +225,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.13
-signing_key: 
+rubygems_version: 3.0.8
+signing_key:
 specification_version: 4
 summary: Load CONTENTdm data into a Solr Index. CDMBL expects to run inside a Rails
   application.