cdmbl 0.10.1 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3b42c6c8e77e084d0ef7859aa3e46c347f83d658
-  data.tar.gz: a2a93fba6b9a2e25aebbebaa75b21bbfab980a0a
+  metadata.gz: cb68955dc77e2f7a1ceb938a49d7fa4aa471cd50
+  data.tar.gz: 66525455ae0dca29cbac2c175d43811bf781143e
 SHA512:
-  metadata.gz: b820e473a137ef4179af8dd9e1bbeae4e35ea83fcc2dace62cbdb0c1df0eccc31b0821531bfcb1896f88075637f3b9ca8bf8e9cf62437bd9550d160e7251741c
-  data.tar.gz: 8e0a7cf8762801a5e6cd9b59d63c74d4a38794db339f75fa3b9c9ce24947eda56e63ee0b97a04ee17ed1f62929aed365cb03f5be487a93aa5e1b0e09530f4e12
+  metadata.gz: 18c172df0c8265395e1316c73b229673a03a53d54449e819b002a1a710b0238487b8906084fb232596093a6a42cfba9e5797f58e15a1cb6dedb04482c55e953d
+  data.tar.gz: 97e3dc6c0c05f68ab04bb0d686623e092665ef022bf597b1a778df9105b85cd68f350887e068beb76534d234201c57f3664f9dad1fa7a284b14ea372302cadc5

data/lib/cdmbl/etl_by_set_specs.rb

@@ -0,0 +1,18 @@
+module CDMBL
+  class ETLBySetSpecs
+    attr_reader :set_specs, :etl_config, :etl_worker_klass
+    def initialize(set_specs: [:missing_setspec],
+                   etl_config: :missing_etl_config,
+                   etl_worker_klass: ETLWorker)
+      @set_specs        = set_specs
+      @etl_config       = etl_config
+      @etl_worker_klass = etl_worker_klass
+    end
+
+    def run!
+      set_specs.map do |set_spec|
+        etl_worker_klass.perform_async(etl_config.merge(set_spec: set_spec))
+      end
+    end
+  end
+end

data/lib/cdmbl/filtered_set_specs.rb

@@ -0,0 +1,41 @@
+module CDMBL
+  module DefaultFilterSetCallback
+    def valid?(set: {})
+      true
+    end
+  end
+
+  class FilteredSetSpecs
+    attr_reader :oai_base_url,
+                :oai_client,
+                :callback
+
+    def initialize(oai_base_url: :missing_oai_base_url,
+                   oai_client: OaiClient,
+                   callback: DefaultSetFilterCallback.new)
+      @oai_base_url = oai_base_url
+      @oai_client   = oai_client
+      @callback     = callback
+    end
+
+    def set_specs
+      filtered.map {|set| set['setSpec']}
+    end
+
+    private
+
+    def filtered
+      sets.select do |set|
+        callback.valid?(set: set)
+      end
+    end
+
+    def sets
+      @sets ||= list_sets['OAI_PMH']['ListSets']['set']
+    end
+
+    def list_sets
+      @list_sets ||= oai_client.new(base_url: oai_base_url).request('verb=ListSets')
+    end
+  end
+end

data/lib/cdmbl/set_spec_filter_callback.rb

@@ -0,0 +1,17 @@
+module CDMBL
+  class SetSpecFilterCallback
+    attr_reader :pattern, :inclusive
+    def initialize(pattern: /.*/, inclusive: true)
+      @pattern   = pattern
+      @inclusive = inclusive
+    end
+
+    def valid?(set: {})
+      (inclusive) ? matches?(set) : !matches?(set)
+    end
+
+    def matches?(set)
+      pattern.match?(set['setSpec'])
+    end
+  end
+end

data/lib/cdmbl/tasks/etl.rake

@@ -20,6 +20,49 @@ namespace :cdmbl do
     )
   end
 
+  desc 'Launch an indexing worker for each collection with an optional regex
+        pattern to match setSpec. Patterns can be inclusive or exclusive.'
+  task :by_collections, [
+    :solr_url,
+    :oai_endpoint,
+    :cdm_endpoint,
+    :set_spec_pattern,
+    :inclusive,
+    :batch_size
+  ] do |t, args|
+    # Required args
+    oai_endpoint = args.fetch(:oai_endpoint)
+    solr_url = args.fetch(:solr_url)
+    cdm_endpoint = args.fetch(:cdm_endpoint)
+    # Optional args
+    pattern    = args.fetch(:set_spec_pattern, false)
+    inclusive  = args.fetch(:inclusive, true)
+    batch_size = args.fetch(:batch_size, 5)
+
+    # Define your own callback if you want to use other set related fields
+    # Use the SetSpecFilterCallback as an example of how to build your own filter
+    set_specs =
+      if pattern
+        filter = CDMBL::SetSpecFilterCallback.new(pattern: Regexp.new(pattern))
+        CDMBL::FilteredSetSpecs.new(oai_base_url: oai_endpoint,
+                                    callback: filter).set_specs
+      else
+        CDMBL::FilteredSetSpecs.new(oai_base_url: oai_endpoint).set_specs
+      end
+
+    puts "Indexing Sets: '#{set_specs.join(', ')}'"
+
+    etl_config = {
+      solr_config: { url: args.fetch(:solr_url) },
+      oai_endpoint: args.fetch(:oai_endpoint),
+      cdm_endpoint: args.fetch(:cdm_endpoint),
+      batch_size: args.fetch(:batch_size, 10),
+      max_compounds: args.fetch(:max_compounds, 10)
+    }
+
+    CDMBL::ETLBySetSpecs.new(set_specs: set_specs, etl_config: etl_config).run!
+  end
+
   desc 'Launch a background job to index a single record.'
   task :record, [
     :collection,

data/lib/cdmbl/version.rb

@@ -1,3 +1,3 @@
 module CDMBL
-  VERSION = "0.10.1"
-end
+  VERSION = "0.11.0"
+end

data/lib/cdmbl.rb

@@ -25,4 +25,7 @@ require 'cdmbl/batch_deleter_worker'
 require 'cdmbl/compound_lookup'
 require 'cdmbl/compound_filter'
 require 'cdmbl/load_worker'
-require 'cdmbl/transform_worker'
+require 'cdmbl/transform_worker'
+require 'cdmbl/filtered_set_specs'
+require 'cdmbl/etl_by_set_specs'
+require 'cdmbl/set_spec_filter_callback'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cdmbl
 version: !ruby/object:Gem::Version
-  version: 0.10.1
+  version: 0.11.0
 platform: ruby
 authors:
 - chadfennell
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-03-26 00:00:00.000000000 Z
+date: 2018-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: hash_at_path
@@ -218,12 +218,14 @@ files:
 - lib/cdmbl/default_loader_notification.rb
 - lib/cdmbl/default_oai_notification.rb
 - lib/cdmbl/default_solr.rb
+- lib/cdmbl/etl_by_set_specs.rb
 - lib/cdmbl/etl_run.rb
 - lib/cdmbl/etl_worker.rb
 - lib/cdmbl/extract_worker.rb
 - lib/cdmbl/extractor.rb
 - lib/cdmbl/field_formatter.rb
 - lib/cdmbl/field_transformer.rb
+- lib/cdmbl/filtered_set_specs.rb
 - lib/cdmbl/formatters.rb
 - lib/cdmbl/hooks.rb
 - lib/cdmbl/load_worker.rb
@@ -236,6 +238,7 @@ files:
 - lib/cdmbl/oai_set_lookup.rb
 - lib/cdmbl/rake_task.rb
 - lib/cdmbl/record_transformer.rb
+- lib/cdmbl/set_spec_filter_callback.rb
 - lib/cdmbl/tasks/delete.rake
 - lib/cdmbl/tasks/etl.rake
 - lib/cdmbl/transform_worker.rb