cdmbl 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 992abfe17a0e27829cf5d531cac21938997e301e
-  data.tar.gz: 9ef14dcdefb7fbdd428bdf5427af74ff5e30d9c5
+  metadata.gz: 5c42251d98410bacf0c610a536e9514e0e4a6662
+  data.tar.gz: 77fb5f56972cf1bb3b7e31ad8be7188cdf4c5fda
 SHA512:
-  metadata.gz: f6849f88abd6eb0ffd2d5e2606bc5c9e85a583e3f5e2f3502799288455749f47c5a70afd2a4ebb2ca9941913ac460149e04930b78916c36332f81f8b3b6056ba
-  data.tar.gz: 5ec0e748e237dfca5f576472eb24a96bda23447b937a7f130683783c9311762b8b5d8868bfff7d71971475d20d453c8eaa5a3d96496b57fe0f645252b8ff8e61
+  metadata.gz: 49d38e10d6871c4d4e4e249624b543787f6476aaebb680748babf822932127eac82bdb81259177b7b9e1e543b816ac012e4d08735bf882a57e3829c1175efb13
+  data.tar.gz: 8d99175077e75f997efb3c0233eeed32d1be2142403d3a566452471faa3b13926573a2c4cba19540dfab2328455175e8c19369ce61d5db6fe2becf5df1de3995

data/lib/cdmbl/batch_deleter.rb

@@ -0,0 +1,51 @@
+module CDMBL
+  class BatchDeleter
+    attr_reader :prefix,
+                :start,
+                :batch_size,
+                :oai_client,
+                :solr_client,
+                :oai_deletables_klass
+    def initialize(prefix: '',
+                   start: 0,
+                   batch_size: 200,
+                   oai_client: :missing_oai_client,
+                   solr_client: :missing_solr_client,
+                   oai_deletables_klass: OaiDeletables)
+      @prefix               = prefix
+      @start                = start
+      @batch_size           = batch_size
+      @oai_client           = oai_client
+      @solr_client          = solr_client
+      @oai_deletables_klass = oai_deletables_klass
+    end
+
+    def delete!
+      solr_client.delete deletables
+    end
+
+    def last_batch?
+      start + batch_size >= num_found
+    end
+
+    private
+
+    def deletables
+      oai_deletables_klass.new(identifiers: docs,
+                               prefix: prefix,
+                               oai_client: oai_client).deletables
+    end
+
+    def docs
+      ids.fetch('response', {}).fetch('docs', {})
+    end
+
+    def num_found
+      ids.fetch('response', {}).fetch('numFound', 0)
+    end
+
+    def ids
+      @ids ||= solr_client.ids(start: start)
+    end
+  end
+end

data/lib/cdmbl/batch_deleter_worker.rb

@@ -0,0 +1,41 @@
+require 'sidekiq'
+module CDMBL
+  class BatchDeleterWorker
+    include Sidekiq::Worker
+    attr_reader :start, :prefix, :oai_url, :solr_url
+    def perform(start = 0, prefix = '', oai_url = '', solr_url = '')
+      @start    = start
+      @prefix   = prefix
+      @oai_url  = oai_url
+      @solr_url = solr_url
+      delete!
+    end
+
+    private
+
+    def delete!
+      batch_deleter.delete!
+      unless batch_deleter.last_batch?
+        BatchDeleteWorker.perform_async(start: start + 1,
+                                        prefix: prefix,
+                                        oai_url: oai_url,
+                                        solr_url: solr_url)
+      end
+    end
+
+    def batch_deleter
+      @deleter ||= BatchDeleter.new(start: start,
+                                    prefix: prefix,
+                                    solr_client: solr_client,
+                                    oai_client: oai_client)
+    end
+
+    def solr_client
+      @solr_client ||= CDMBL::Solr.new(url: solr_url)
+    end
+
+    def oai_client
+      @oai_client ||= OaiClient.new base_url: oai_url
+    end
+  end
+end

data/lib/cdmbl/default_solr.rb

@@ -4,11 +4,22 @@ module CDMBL
   # Commnicate with Solr: add / delete stuff
   class DefaultSolr
     attr_reader :url, :client
-    def initialize(url: 'http://localhost:8983', client: RSolr)
+    def initialize(url: 'http://localhost:8983/solr/core-here', client: RSolr)
       @url    = url
       @client = client
     end
 
+    def ids(start: 0)
+      connection.get('select',
+        :params => { :q => '*:*',
+          :defType => 'edismax',
+          :fl => '',
+          :rows => 200,
+          :start => start
+        }
+      )
+    end
+
     def connection
       @connection ||= client.connect url: url
     end

data/lib/cdmbl/extractor.rb

@@ -29,7 +29,7 @@ module CDMBL
     end
 
     def set_lookup
-      oai_set_lookup.new(oai_sets: to_hash(sets)).keyed
+      oai_set_lookup.new(oai_sets: sets).keyed
     end
 
     def deletable_ids
@@ -67,11 +67,7 @@ module CDMBL
     end
 
     def oai_identifiers
-      to_hash(identifiers)
-    end
-
-    def to_hash(xml)
-      Hash.from_xml(xml)
+      identifiers
     end
   end
 end

data/lib/cdmbl/oai_client.rb

@@ -0,0 +1,24 @@
+require 'json'
+module CDMBL
+    class OaiClient
+      attr_reader :base_url, :http_client
+      def initialize(base_url: '', http_client: Net::HTTP)
+        @base_url    = base_url
+        @http_client = http_client
+      end
+
+      def request(query)
+        hashify get("#{base_url}?#{query}")
+      end
+
+      private
+
+      def get(url)
+        http_client.get_response(URI(url)).body
+      end
+
+      def hashify(xml)
+        Hash.from_xml(xml)
+      end
+    end
+end

data/lib/cdmbl/oai_deletables.rb

@@ -0,0 +1,44 @@
+require 'sidekiq'
+module CDMBL
+  class OaiDeletables
+    attr_reader :identifiers, :oai_record_klass, :oai_client, :prefix
+    def initialize(identifiers: [],
+                   prefix: '',
+                   oai_client: OaiClient.new,
+                   oai_record_klass: OaiGetRecord)
+      @identifiers       = identifiers
+      @prefix            = prefix
+      @oai_client        = oai_client
+      @oai_record_klass  = oai_record_klass
+    end
+
+    def deletables
+      identifiers.select do |id|
+        record_exists? to_oai_id(id)
+      end
+    end
+
+    private
+
+    def to_oai_id(id)
+      "#{prefix}#{collection(id)}/#{id(id)}"
+    end
+
+    def id(id)
+      id_parts(id).last
+    end
+
+    def collection(id)
+      id_parts(id).first
+    end
+
+    def id_parts(id)
+      id.split(':')
+    end
+
+    def record_exists?(identifier)
+      oai_record_klass.new(oai_client: oai_client,
+                           identifier: identifier).record_exists?
+    end
+  end
+end

data/lib/cdmbl/oai_get_record.rb

@@ -0,0 +1,30 @@
+require 'json'
+module CDMBL
+    # Request a single item from an OAI endpoint
+    # identifier should be forward slash delimited: colllection/identifier
+    class OaiGetRecord
+      attr_reader :identifier, :oai_client
+      def initialize(identifier: '', oai_client: OaiClient.new)
+        @identifier = identifier
+        @oai_client = oai_client
+      end
+
+      def record_exists?
+        (/The value of the identifier argument is unknown/ =~ record_errors) == nil
+      end
+
+      def record
+        @record ||= oai_client.request query
+      end
+
+      private
+
+      def record_errors
+        record.fetch('OAI_PMH', {}).fetch('error', '')
+      end
+
+      def query
+        "verb=GetRecord&identifier=#{identifier}&metadataPrefix=oai_dc"
+      end
+    end
+end

data/lib/cdmbl/oai_request.rb

@@ -1,16 +1,24 @@
+require 'json'
 module CDMBL
     class OaiRequest
-      attr_reader :base_uri, :resumption_token, :client, :from, :set
+      attr_reader :base_uri,
+                  :resumption_token,
+                  :client,
+                  :from,
+                  :set,
+                  :identifier
       def initialize(base_uri: '',
                      resumption_token: false,
                      from: false,
                      set: false,
+                     identifier: '',
                      client: Net::HTTP)
           @base_uri         = base_uri
           @resumption_token = resumption_token
           @client           = client
           @from             = (from) ? "&from=#{from}" : ''
           @set              = (set) ? "&set=#{set}" : ''
+          @identifier       = identifier
       end
 
       def identifiers
@@ -37,7 +45,7 @@ module CDMBL
 
       def request(location)
         CDMBL::OaiNotification.call!(location)
-        client.get_response(URI(location)).body
+        Hash.from_xml(client.get_response(URI(location)).body)
       end
     end
 end

data/lib/cdmbl/version.rb

@@ -1,3 +1,3 @@
 module CDMBL
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/lib/cdmbl.rb

@@ -16,4 +16,9 @@ require 'cdmbl/default_cdm_notification'
 require 'cdmbl/default_oai_notification'
 require 'cdmbl/default_loader_notification'
 require 'cdmbl/hooks'
-require 'cdmbl/oai_filter'
+require 'cdmbl/oai_filter'
+require 'cdmbl/oai_client'
+require 'cdmbl/oai_get_record'
+require 'cdmbl/oai_deletables'
+require 'cdmbl/batch_deleter'
+require 'cdmbl/batch_deleter_worker'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cdmbl
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - chadfennell
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-06-05 00:00:00.000000000 Z
+date: 2017-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: hash_at_path
@@ -208,6 +208,8 @@ files:
 - bin/setup
 - cdmbl.gemspec
 - lib/cdmbl.rb
+- lib/cdmbl/batch_deleter.rb
+- lib/cdmbl/batch_deleter_worker.rb
 - lib/cdmbl/default_cdm_notification.rb
 - lib/cdmbl/default_completed_callback.rb
 - lib/cdmbl/default_loader_notification.rb
@@ -221,7 +223,10 @@ files:
 - lib/cdmbl/formatters.rb
 - lib/cdmbl/hooks.rb
 - lib/cdmbl/loader.rb
+- lib/cdmbl/oai_client.rb
+- lib/cdmbl/oai_deletables.rb
 - lib/cdmbl/oai_filter.rb
+- lib/cdmbl/oai_get_record.rb
 - lib/cdmbl/oai_request.rb
 - lib/cdmbl/oai_set_lookup.rb
 - lib/cdmbl/rake_task.rb