cdmbl 0.17.1 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bdd98b9eb3cd0356fb5af7d9a33b3589aa7d5b6b75813a76996b7d0252446d94
-  data.tar.gz: ac1be708c80dc478d7ecaf3ab6996c527a9f78a746565d146a630c45b3295efd
+  metadata.gz: 98471f931e9b4535c9f1019cfe954cfba9eb7e9742fdb4f2eda70fe914d4fd3e
+  data.tar.gz: 32524fe0f8ae2d4ea5b45954d0b368a46153b9f7b3298cb064a5202cd75049ee
 SHA512:
-  metadata.gz: 1e9b85e89816e9ee280d5ff8606a71123299e2b0e557a65831d05c5d0a83d5260b43ab05bc2fbc7f38849ed3369e439e8d8836b8b07781977912c28bc51b6e82
-  data.tar.gz: d50f00a76b0064bd0e98c236d157a12e70ce5aa8cbd33bd38967cc1fd1d072dab558924a53760f5f333147621bef5198bc2f2c6904c7d49d390c9bf2c292774f
+  metadata.gz: 6205cbb83cc23192f3157341e4980b49b4bd6cb89a615ddec3ae13d829f26738d293e6cb174e56741c6fe1406c501568896fdf3ca148be55743b0318452c75ff
+  data.tar.gz: c9b9f37bc27e3a7d21fb30035310dae9109e5f0d73d8bf8a45f7b0f175251766e06185de27a03763a50eb8d034777437f910b73e2446e388c98f7371c3a04d8d

data/lib/cdmbl/etl_worker.rb

@@ -16,7 +16,8 @@ module CDMBL
                 :set_spec,
                 :max_compounds,
                 :batch_size,
-                :is_recursive
+                :is_recursive,
+                :from
 
     attr_writer :compound_filter_klass,
                 :extractor_klass,
@@ -40,6 +41,7 @@ module CDMBL
       @max_compounds     = config.fetch('max_compounds', 10)
       @batch_size        = config.fetch('batch_size', 5).to_i
       @is_recursive      = config.fetch('is_recursive', true)
+      @from              = config.fetch('from', nil)
       extract_batch!
       next_batch!
     end
@@ -114,12 +116,14 @@ module CDMBL
     end
 
     def transform!(ids)
-      transform_worker_klass.perform_async(ids,
-                                           solr_config,
-                                           cdm_endpoint,
-                                           oai_endpoint,
-                                           field_mappings,
-                                           extract_compounds)
+      transform_worker_klass.perform_async(
+        ids,
+        solr_config,
+        cdm_endpoint,
+        oai_endpoint,
+        field_mappings,
+        extract_compounds
+      )
     end
 
     def delete_deletables!
@@ -127,17 +131,20 @@ module CDMBL
     end
 
     def compound_filter
-      @compound_filter ||=
-        compound_filter_klass.new(record_ids: extraction.local_identifiers,
-                                  cdm_endpoint: cdm_endpoint,
-                                  max_compounds: max_compounds)
+      @compound_filter ||= compound_filter_klass.new(
+        record_ids: extraction.local_identifiers,
+        cdm_endpoint: cdm_endpoint,
+        max_compounds: max_compounds
+      )
     end
 
     def extraction
-      @extraction ||=
-        extractor_klass.new(oai_endpoint: oai_endpoint,
-                            resumption_token: resumption_token,
-                            set_spec: set_spec)
+      @extraction ||= extractor_klass.new(
+        oai_endpoint: oai_endpoint,
+        resumption_token: resumption_token,
+        set_spec: set_spec,
+        from: from
+      )
     end
   end
 end

data/lib/cdmbl/extractor.rb

@@ -16,6 +16,7 @@ module CDMBL
     def initialize(oai_endpoint: '',
                    resumption_token: nil,
                    set_spec: nil,
+                   from: nil,
                    oai_request_klass: OaiRequest,
                    oai_filter_klass: OAIFilter,
                    oai_set_lookup_klass: OAISetLookup)
@@ -24,7 +25,8 @@ module CDMBL
       @oai_set_lookup_klass = oai_set_lookup_klass
       @oai_request          = oai_requester(oai_endpoint,
                                             resumption_token,
-                                            set_spec)
+                                            set_spec,
+                                            from)
     end
 
     def deletable_ids
@@ -49,11 +51,16 @@ module CDMBL
 
     private
 
-    def oai_requester(oai_endpoint, resumption_token, set_spec)
-      @oai_requester ||=
-        oai_request_klass.new(base_uri: oai_endpoint,
-                              resumption_token: resumption_token,
-                              set: set_spec)
+    def oai_requester(oai_endpoint, resumption_token, set_spec, from)
+      @oai_requester ||= begin
+        args = {
+          base_uri: oai_endpoint,
+          resumption_token: resumption_token,
+          set: set_spec,
+        }
+        args[:from] = from if from
+        oai_request_klass.new(args)
+      end
     end
 
     # Get the local collection and id from an OAI namespaced identifier

data/lib/cdmbl/formatters.rb

@@ -72,7 +72,7 @@ module CDMBL
   class Titlieze
     def self.format(value)
       if value.respond_to?(:map)
-        value.map {|value| value.titleize }
+        value.map(&:titleize)
       else
         value.titleize
       end
@@ -172,4 +172,4 @@ module CDMBL
     end
   end
 
-end
+end

data/lib/cdmbl/oai_request.rb

@@ -1,48 +1,51 @@
 require 'json'
 module CDMBL
-    class OaiRequest
-      attr_reader :base_uri,
-                  :resumption_token,
-                  :client,
-                  :set,
-                  :identifier
-      def initialize(base_uri: '',
-                     resumption_token: nil,
-                     set: nil,
-                     identifier: '',
-                     client: Net::HTTP)
-          @base_uri         = base_uri
-          @resumption_token = resumption_token
-          @client           = client
-          @set              = (set) ? "&set=#{set}" : ''
-          @identifier       = identifier
-      end
+  class OaiRequest
+    attr_reader :base_uri,
+                :resumption_token,
+                :client,
+                :set,
+                :identifier,
+                :from
+    def initialize(base_uri: '',
+                   resumption_token: nil,
+                   set: nil,
+                   identifier: '',
+                   from: nil,
+                   client: Net::HTTP)
+      @base_uri         = base_uri
+      @resumption_token = resumption_token
+      @client           = client
+      @set              = (set) ? "&set=#{set}" : ''
+      @from             = from ? "&from=#{from}" : ''
+      @identifier       = identifier
+    end
 
-      def identifiers
-        @ids ||= (resumption_token) ? request(batch_uri) : request(first_batch_uri)
-      end
+    def identifiers
+      @ids ||= (resumption_token) ? request(batch_uri) : request(first_batch_uri)
+    end
 
-      def sets
-        @sets ||= request(sets_uri)
-      end
+    def sets
+      @sets ||= request(sets_uri)
+    end
 
-      private
+    private
 
-      def first_batch_uri
-        "#{base_uri}?verb=ListIdentifiers&metadataPrefix=oai_dc#{set}"
-      end
+    def first_batch_uri
+      "#{base_uri}?verb=ListIdentifiers&metadataPrefix=oai_dc#{set}#{from}"
+    end
 
-      def batch_uri
-        "#{base_uri}?verb=ListIdentifiers&resumptionToken=#{resumption_token}"
-      end
+    def batch_uri
+      "#{base_uri}?verb=ListIdentifiers&resumptionToken=#{resumption_token}"
+    end
 
-      def sets_uri
-        "#{base_uri}?verb=ListSets"
-      end
+    def sets_uri
+      "#{base_uri}?verb=ListSets"
+    end
 
-      def request(location)
-        CDMBL::OaiNotification.call!(location)
-        Hash.from_xml(client.get_response(URI(location)).body)
-      end
+    def request(location)
+      CDMBL::OaiNotification.call!(location)
+      Hash.from_xml(client.get_response(URI(location)).body)
     end
-end
+  end
+end

data/lib/cdmbl/version.rb

@@ -1,3 +1,3 @@
 module CDMBL
-  VERSION = "0.17.1"
-end
+  VERSION = "0.18.0"
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cdmbl
 version: !ruby/object:Gem::Version
-  version: 0.17.1
+  version: 0.18.0
 platform: ruby
 authors:
 - chadfennell
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-03-11 00:00:00.000000000 Z
+date: 2021-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: hash_at_path
@@ -150,7 +150,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.0
-description: 
+description:
 email:
 - fenne035@umn.edu
 executables: []
@@ -206,11 +206,11 @@ files:
 - lib/cdmbl/transformer.rb
 - lib/cdmbl/version.rb
 - travis.yml
-homepage: 
+homepage:
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -225,8 +225,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.2
-signing_key: 
+rubygems_version: 3.0.8
+signing_key:
 specification_version: 4
 summary: Load CONTENTdm data into a Solr Index. CDMBL expects to run inside a Rails
   application.