cdmbl 0.2.25 → 0.2.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a38f93dada47610d74fd9724fc057a86ab46d9db
-  data.tar.gz: a410ba1672d49c893de4e862047eae7bbd7316df
+  metadata.gz: e7d48b4a55adff68d3096b6f4e04f20b26cbcb61
+  data.tar.gz: 7c670f721ef8617a51994becbca83e7ca9900f30
 SHA512:
-  metadata.gz: e2bde7dc1b0879186db2149092c967db2d3f4ff5cead71acb71dd8f0cb6bdad394ff2eee4d676e4d998f320ef2486b285ecfa093647cb54395c42db79b820923
-  data.tar.gz: 63df148bd2914fefba9972dc0b220493a8bb3398eed410ca6bfeece20b15c1a205acc89d9b22e98a53855e027ea8f20b908f215f83ac3ebdc929282871bce624
+  metadata.gz: 8802661c746e0c8ea8d162b00777c26be2f9928d8dc0ec36b4691dbf8635c51e5af80bf5c3950c9330592236e3812d9dadfc85637338ef5bd792f534e8b00e9e
+  data.tar.gz: f58528feef5a8d52ef4d76f9a98127414e037b7e38e89dcc8ed233a8e901b28ee7a0a47b54d0a51f3c2fb5ccfc2ad7bc59047ab65a322119f038be651915ae39

data/lib/cdmbl/etl_run.rb

@@ -6,6 +6,7 @@ module CDMBL
                 :resumption_token,
                 :field_mappings,
                 :minimum_date,
+                :set_spec,
                 :oai_requester,
                 :extractor,
                 :transformer,
@@ -16,6 +17,7 @@ module CDMBL
                    resumption_token: false,
                    field_mappings: false,
                    minimum_date: nil,
+                   set_spec: false,
                    oai_requester: OaiRequest,
                    extractor: Extractor,
                    transformer: Transformer,
@@ -28,6 +30,7 @@ module CDMBL
       @field_mappings   = field_mappings
       @oai_requester    = oai_requester
       @minimum_date     = minimum_date
+      @set_spec         = set_spec
       @extractor        = extractor
       @transformer      = transformer
       @loader           = loader
@@ -54,7 +57,8 @@ module CDMBL
     def oai_request
       @oai_request ||= oai_requester.new(base_uri: oai_endpoint,
                                          resumption_token: resumption_token,
-                                         from: minimum_date)
+                                         from: minimum_date,
+                                         set: set_spec)
     end
   end
 

data/lib/cdmbl/etl_worker.rb

@@ -1,30 +1,28 @@
 require 'sidekiq'
 module CDMBL
   class ETLWorker
+    include Sidekiq::Worker
+
     attr_reader :solr_config,
                 :etl_config,
-                :batch_size,
                 :is_recursive,
-                :identifiers,
+                :identifier,
                 :deletables
 
-    include Sidekiq::Worker
 
     def perform(solr_config,
                 etl_config,
-                batch_size = 10,
                 is_recursive = true,
-                identifiers = [],
+                identifier = false,
                 deletables = [])
-  
+
       @etl_config   = etl_config.symbolize_keys
       @solr_config  = solr_config.symbolize_keys
       @is_recursive = is_recursive
-      @identifiers  = identifiers
+      @identifier   = identifier
       @deletables   = deletables
-      @batch_size   = batch_size
-  
-      if !identifiers.empty?
+
+      if identifier
         load!
       else
         ingest_batches!
@@ -40,13 +38,12 @@ module CDMBL
 
     def ingest_batches!
       sent_deleted = false
-      extraction.local_identifiers.each_slice(batch_size) do |ids|
+      extraction.local_identifiers.each do |id|
         delete_ids = (sent_deleted == false) ? extraction.deletable_ids : []
         ETLWorker.perform_async(solr_config,
                                 etl_config,
-                                batch_size,
                                 is_recursive,
-                                ids,
+                                id,
                                 delete_ids)
         sent_deleted = true
       end
@@ -58,13 +55,11 @@ module CDMBL
     end
 
     def transformation
-      @transformation ||= etl_run.transform(extraction.set_lookup, records)
+      @transformation ||= etl_run.transform(extraction.set_lookup, [record])
     end
 
-    def records
-      identifiers.map do |identifier|
-        extraction.cdm_request(*identifier)
-      end
+    def record
+      extraction.cdm_request(*identifier)
     end
 
     def extraction

data/lib/cdmbl/extractor.rb

@@ -32,10 +32,6 @@ module CDMBL
       oai_set_lookup.new(oai_sets: to_hash(sets)).keyed
     end
 
-    def ids
-      (specific_ids) ? specific_ids : local_identifiers
-    end
-
     def deletable_ids
       oai_ids.deletable_ids
     end

data/lib/cdmbl/oai_request.rb

@@ -1,14 +1,16 @@
 module CDMBL
     class OaiRequest
-      attr_reader :base_uri, :resumption_token, :client, :from
+      attr_reader :base_uri, :resumption_token, :client, :from, :set
       def initialize(base_uri: '',
                      resumption_token: false,
                      from: false,
+                     set: false,
                      client: Net::HTTP)
           @base_uri         = base_uri
           @resumption_token = resumption_token
           @client           = client
           @from             = (from) ? "&from=#{from}" : ''
+          @set              = (set) ? "&set=#{set}" : ''
       end
 
       def identifiers
@@ -22,7 +24,7 @@ module CDMBL
       private
 
       def first_batch_uri
-        "#{base_uri}?verb=ListIdentifiers&metadataPrefix=oai_dc#{from}"
+        "#{base_uri}?verb=ListIdentifiers&metadataPrefix=oai_dc#{from}#{set}"
       end
 
       def batch_uri

data/lib/cdmbl/tasks/etl.rake

@@ -4,10 +4,14 @@ namespace :cdmbl do
   desc 'Launch a background job to index metadata from CONTENTdm to Solr.'
   task :ingest, [:solr_url, :oai_endpoint, :cdm_endpoint, :minimum_date] do |t, args|
     solr_config = { url: args[solr_url] }
-    etl_config  = { oai_endpoint: args[:oai_endpoint], cdm_endpoint: args[:cdm_endpoint], minimum_date: args[:minimum_date] }
-    etl_config = (args[:resumption_token]) ? etl_cofig.merge(args[:resumption_token]) : etl_config 
-    batch_size = (args[:batch_size]) ? args[:batch_size] : 10
-    CDMBL::ETLWorker.perform_async(solr_config, etl_config, batch_size)
+    etl_config  = {
+                    oai_endpoint: args[:oai_endpoint],
+                    cdm_endpoint: args[:cdm_endpoint],
+                    minimum_date: args[:minimum_date]
+                    set_spec: args[:set_spec]
+                  }
+    etl_config = (args[:resumption_token]) ? etl_cofig.merge(args[:resumption_token]) : etl_config
+    CDMBL::ETLWorker.perform_async(solr_config, etl_config)
   end
 end
 

data/lib/cdmbl/transformer.rb

@@ -36,7 +36,7 @@ module CDMBL
       end
     end
 
-    #TODO: Make a
+    #TODO: Make a 
     def with_location(dest_record, record)
       if record['latitu'] && record['longit'] && record['latitu'] != '' && record['longit'] != ''
         dest_record.merge({'location_llsi': "#{record['latitu']}, #{record['longit']}"})

data/lib/cdmbl/version.rb

@@ -1,3 +1,3 @@
 module CDMBL
-  VERSION = "0.2.25"
+  VERSION = "0.2.26"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cdmbl
 version: !ruby/object:Gem::Version
-  version: 0.2.25
+  version: 0.2.26
 platform: ruby
 authors:
 - chadfennell