cdmbl 0.2.27 → 0.2.28

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fbacbfdb0861491fb0f6fc15a120716a46c280a7
-  data.tar.gz: 0eda27fe1e6f94db141ab7484c926f8ab387ea41
+  metadata.gz: 10286777e3b6a73bff82b645d9793478bef136d0
+  data.tar.gz: 0342f14ccb141499d3b9a72f4a1a3e7c2215652d
 SHA512:
-  metadata.gz: de81249496cc904c2077f18454de06e429dfb1d80ea05775c794e452070c22ea3794054e3cb12cd327f111b935df08d89545d24761fa8cbd5f10f961181cef54
-  data.tar.gz: ff87f9363619ffcf91219415ceacf49630bdc12ac5bcdc67fd8a3232f1586862bee90b854c94d30731b9b1b516f4936d761a413e7ac70f944baa585a1691d491
+  metadata.gz: 05227bcaab3a297715c4076345109770a6c701aae1923186c9043d391b9bbf252ce9e5ad7536e85e6220e8a3e1cd1a602f325d5c65e2aa1641ae1cfb458a0f1c
+  data.tar.gz: dc004423977fbf3917f03af1e07a5092142a23c3be50c91913e2bad3f2bec322acace8c13e931a3b552b02a715b87b6b45a9798d18064b348e01d347a45fc3da

data/cdmbl.gemspec

@@ -29,7 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'activesupport', '~> 4.2'
 
   spec.add_development_dependency 'bundler', '~> 1.12'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake', '~> 12.0'
   spec.add_development_dependency 'minitest', '~> 5.0'
   spec.add_development_dependency 'yard', '~> 0.9.0'
   spec.add_development_dependency 'webmock', '~> 1.24', '>= 1.24.0'

data/lib/cdmbl/etl_worker.rb

@@ -5,24 +5,27 @@ module CDMBL
 
     attr_reader :solr_config,
                 :etl_config,
+                :batch_size,
                 :is_recursive,
-                :identifier,
+                :identifiers,
                 :deletables
 
 
     def perform(solr_config,
                 etl_config,
+                batch_size = 5,
                 is_recursive = true,
-                identifier = false,
+                identifiers = [],
                 deletables = [])
 
       @etl_config   = etl_config.symbolize_keys
       @solr_config  = solr_config.symbolize_keys
+      @batch_size   = batch_size.to_i
       @is_recursive = is_recursive
-      @identifier   = identifier
+      @identifiers  = identifiers
       @deletables   = deletables
 
-      if identifier
+      if !identifiers.empty?
         load!
       else
         ingest_batches!
@@ -38,12 +41,13 @@ module CDMBL
 
     def ingest_batches!
       sent_deleted = false
-      extraction.local_identifiers.each do |id|
+      extraction.local_identifiers.each_slice(batch_size) do |ids|
         delete_ids = (sent_deleted == false) ? extraction.deletable_ids : []
         ETLWorker.perform_async(solr_config,
                                 etl_config,
+                                batch_size,
                                 is_recursive,
-                                id,
+                                ids,
                                 delete_ids)
         sent_deleted = true
       end
@@ -55,11 +59,13 @@ module CDMBL
     end
 
     def transformation
-      @transformation ||= etl_run.transform(extraction.set_lookup, [record])
+      @transformation ||= etl_run.transform(extraction.set_lookup, records)
     end
 
-    def record
-      extraction.cdm_request(*identifier)
+    def records
+      identifiers.map do |identifier|
+        extraction.cdm_request(*identifier)
+      end
     end
 
     def extraction

data/lib/cdmbl/tasks/etl.rake

@@ -2,7 +2,7 @@ require 'cdmbl'
 
 namespace :cdmbl do
   desc 'Launch a background job to index metadata from CONTENTdm to Solr.'
-  task :ingest, [:solr_url, :oai_endpoint, :cdm_endpoint, :minimum_date, :set_spec] do |t, args|
+  task :ingest, [:solr_url, :oai_endpoint, :cdm_endpoint, :minimum_date, :batch_size, :set_spec] do |t, args|
     solr_config = { url: args[solr_url] }
     etl_config  = {
                     oai_endpoint: args[:oai_endpoint],
@@ -11,7 +11,8 @@ namespace :cdmbl do
                     set_spec: args[:set_spec]
                   }
     etl_config = (args[:resumption_token]) ? etl_cofig.merge(args[:resumption_token]) : etl_config
-    CDMBL::ETLWorker.perform_async(solr_config, etl_config)
+    batch_size = (args[:batch_size]) ? args[:batch_size] : 10
+    CDMBL::ETLWorker.perform_async(solr_config, etl_config, batch_size, true)
   end
 end
 

data/lib/cdmbl/version.rb

@@ -1,3 +1,3 @@
 module CDMBL
-  VERSION = "0.2.27"
+  VERSION = "0.2.28"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cdmbl
 version: !ruby/object:Gem::Version
-  version: 0.2.27
+  version: 0.2.28
 platform: ruby
 authors:
 - chadfennell
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-03-02 00:00:00.000000000 Z
+date: 2017-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: hash_at_path
@@ -114,14 +114,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement