cdmbl 0.2.12 → 0.2.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 205106670da9d0eb64b39e5079b3c885f0bc4e11
-  data.tar.gz: 45310207e4bbd256823f5ad35fb94378ee70e93c
+  metadata.gz: a45fd0906d4aabd9d22383376d7bd0d241906121
+  data.tar.gz: 2ea5a558a0d703b6c146b421426e828713852392
 SHA512:
-  metadata.gz: 0d674a82a9c58a9891d90037f2c3614c93b6a8db14a2eb5f0a154ac2632823d694f9091995a1441416220f5edaae9d1c64ab892bb74e9994c5b86a2d3e928843
-  data.tar.gz: 9e4bdbe926771af0b9cac3185c6386fd2e8a4a8a7a8328d6f149c749ea7941e44a7db96bfbecebb48d1587edb5186c7829d80ba51d70b5410585aab5176cc0ae
+  metadata.gz: a158929c2288f0aa589e275cd841504d9c3143d53df2ce062d746f683d3ad3d1bd21636122011a3e893196f9564cafcc3f4b43a36db6874d352a234cfda62d17
+  data.tar.gz: d5a2daec503800ac3df7bdb9e65b7cd931942f28c69721f285e985bad3dd2ec78d6932202abc409a74bf1c2c72cb969afffbb543c275a8b1de62e28f798156c9

data/lib/cdmbl/default_oai_notification.rb

@@ -1,8 +1,8 @@
 module CDMBL
   # An example callback
   class DefaultOaiNotification
-    def self.call!(solr_client)
-      puts "An OAI callback task"
+    def self.call!(location)
+      puts "An OAI callback task for #{location}"
     end
   end
 end

data/lib/cdmbl/etl_run.rb

@@ -34,29 +34,21 @@ module CDMBL
       @solr_client      = solr_client
     end
 
-    def load!(resumption_token: false)
-      persister.load!
-    end
-
-    def next_resumption_token
-      extraction.next_resumption_token
-    end
-
-    def persister
-      loader.new(records: transformation.records,
-                 deletable_ids: extraction.deletable_ids,
-                 solr_client: solr_client)
+    def extract
+      @extraction ||= extractor.new(oai_request: oai_request,
+                                    cdm_endpoint: cdm_endpoint)
     end
 
-    def transformation
-      @transformation ||= transformer.new(cdm_records: extraction.records,
-                                          oai_sets: extraction.set_lookup,
+    def transform(sets, records)
+      @transformation ||= transformer.new(cdm_records: records,
+                                          oai_sets: sets,
                                           field_mappings: field_mappings)
     end
 
-    def extraction
-      @extraction ||= extractor.new(oai_request: oai_request,
-                                    cdm_endpoint: cdm_endpoint)
+    def load!(deletables, records)
+      loader.new(records: records,
+                 deletable_ids: deletables,
+                 solr_client: solr_client).load!
     end
 
     def oai_request

data/lib/cdmbl/etl_worker.rb

@@ -1,20 +1,71 @@
 require 'sidekiq'
 module CDMBL
   class ETLWorker
-    attr_reader :solr_config, :etl_config
+    attr_reader :solr_config,
+                :etl_config,
+                :is_recursive,
+                :identifiers,
+                :deletables,
+                :sets
+
     include Sidekiq::Worker
-    def perform(solr_config, etl_config, recursive = true)
-      @etl_config  = etl_config.symbolize_keys
-      @solr_config = solr_config.symbolize_keys
-      puts "Ingesting resumptionToken batch: #{etl_config['resumption_token']}"
-      etl_run.load!
-      if etl_run.next_resumption_token && recursive
-        ETLWorker.perform_async(solr_config, next_etl_config)
+    def perform(solr_config,
+                etl_config,
+                is_recursive = true,
+                identifiers = [],
+                deletables = [],
+                sets = [])
+  
+      @etl_config   = etl_config.symbolize_keys
+      @solr_config  = solr_config.symbolize_keys
+      @is_recursive = is_recursive
+      @identifiers  = identifiers
+      @deletables   = deletables
+      @sets         = sets
+  
+      if !identifiers.empty?
+        load!
       else
-        CDMBL::CompletedCallback.call!(solr_client)
+        ingest_batches!
+        if extraction.next_resumption_token && is_recursive
+          ETLWorker.perform_async(solr_config, next_etl_config)
+        else
+          CDMBL::CompletedCallback.call!(solr_client)
+        end
+      end
+    end
+
+    private
+
+    def ingest_batches!
+      extraction.local_identifiers.each_slice(10) do |ids|
+        ETLWorker.perform_async(solr_config,
+                                etl_config,
+                                is_recursive,
+                                ids,
+                                extraction.deletable_ids,
+                                extraction.set_lookup)
+      end
+    end
+
+    def load!
+      etl_run.load!(deletables, transformation.records)
+    end
+
+    def transformation
+      etl_run.transform(sets, records)
+    end
+
+    def records
+      identifiers.map do |identifier|
+        extraction.cdm_request(*identifier)
       end
     end
 
+    def extraction
+      @extraction ||= etl_run.extract
+    end
+
     def etl_run
       ETLRun.new(etl_config.merge(solr_client: solr_client))
     end
@@ -24,7 +75,7 @@ module CDMBL
     end
 
     def next_etl_config
-      etl_config.merge(resumption_token: etl_run.next_resumption_token)
+      etl_config.merge(resumption_token: extraction.next_resumption_token)
     end
   end
 end

data/lib/cdmbl/extractor.rb

@@ -32,10 +32,6 @@ module CDMBL
       oai_set_lookup.new(oai_sets: to_hash(sets)).keyed
     end
 
-    def records
-      local_identifiers.map { |identifier| cdm_request(*identifier) }
-    end
-
     def ids
       (specific_ids) ? specific_ids : local_identifiers
     end
@@ -52,6 +48,12 @@ module CDMBL
       oai_identifiers.at_path('OAI_PMH/ListIdentifiers/resumptionToken')
     end
 
+    # e.g. local_identifiers.map { |identifier| extractor.cdm_request(*identifier) }
+    def cdm_request(collection, id)
+      CDMBL::CdmNotification.call!(collection, id, cdm_endpoint)
+      cdm_item.new(base_url: cdm_endpoint, collection: collection, id: id).metadata
+    end
+
     private
 
     def oai_ids
@@ -72,11 +74,6 @@ module CDMBL
       to_hash(identifiers)
     end
 
-    def cdm_request(collection, id)
-      CDMBL::CdmNotification.call!(collection, id, cdm_endpoint)
-      cdm_item.new(base_url: cdm_endpoint, collection: collection, id: id).metadata
-    end
-
     def to_hash(xml)
       Hash.from_xml(xml)
     end

data/lib/cdmbl/version.rb

@@ -1,3 +1,3 @@
 module CDMBL
-  VERSION = "0.2.12"
+  VERSION = "0.2.13"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cdmbl
 version: !ruby/object:Gem::Version
-  version: 0.2.12
+  version: 0.2.13
 platform: ruby
 authors:
 - chadfennell
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-12-02 00:00:00.000000000 Z
+date: 2016-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: hash_at_path