research_metadata_batch 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4dd55837b2cfc04e261c6b4ab714a63b0767839dd07daa4b4d0f5b6f16c4f8aa
-  data.tar.gz: f9e675684cd2ba0fd35e7c78f4f72451ed4fd9c26e9394c81e4b636450f506fe
+  metadata.gz: 9d27b60c2067c5d5faae6b2680f50186e4087c146bdb076f65955dfd05172397
+  data.tar.gz: 25c9789f3452dc091c2b180eac05f9038ff63f10152eaee9cc4a4fd68be2e862
 SHA512:
-  metadata.gz: 188bd5524e03206bd00f84b33a94f56559f0898f95e02e23df2f44ebf4fba5443a07e3e2b2885038d8800d038bf8d0eb1439ac777ef1d4e552fee1ac6fce3e2a
-  data.tar.gz: f1936694098d4543dd6d310e7ed0ba6ff1317b827824a7c3edff650dfa9d2754810eeef03686dc9928d82085282deab935656f0cf61f1dc2117eb70f44497e26
+  metadata.gz: 103ffbdd89ab5f0056608f07c3f30fb388f294db3ec882cb5e3b4e729c83b0bfedac16ef5047b17b785c7988f065cab55fb32ca4311064bf50879fc1321dc70e
+  data.tar.gz: 15e0e3496b5194d4a15b1f07a9c8e69d64412e41622a352d62fc5f82dbc16d8dad621494c81e09b01b5c8213a19dbeb53221621e802a4616ecb1ad7b1df19ecf

data/CHANGELOG.md

@@ -2,9 +2,17 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## 0.3.0 - 2018-10-12
+### Added
+- Support for GET and POST parameters.
+- Simplified logging.
+
+### Changed
+- Record values matching rather than validation.
+
 ## 0.2.0 - 2018-10-05
 ### Changed
-- Record validation
+- Record validation.
 
 ## 0.1.0 - 2018-09-27
 ### Added

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    research_metadata_batch (0.2.0)
-      puree (~> 2.2)
+    research_metadata_batch (0.3.0)
+      puree (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
@@ -24,7 +24,7 @@ GEM
     nokogiri (1.8.4)
       mini_portile2 (~> 2.3.0)
     public_suffix (3.0.3)
-    puree (2.2.0)
+    puree (2.3.0)
       http (~> 2.0)
       nokogiri (~> 1.6)
     unf (0.1.4)

data/README.md

@@ -1,6 +1,5 @@
 # Research Metadata Batch
-For the batch processing of Pure records. Custom actions and log messages can be 
-defined in user-defined applications.
+Batch processing for the Pure Research Information System.
 
 ## Status
 
@@ -36,21 +35,16 @@ pure_config = {
 ResearchMetadataBatch::Dataset.new(pure_config: pure_config).process
 ```
 
-## Making an application
-Require this gem, then open up the base class {ResearchMetadataBatch::Base} as below. Implement methods from 
-{ResearchMetadataBatch::Custom} as inherited methods, including any secondary initialisation using the 
-``init`` method.
- 
- 
-For resource-specific customisation, open up a resource class e.g. {ResearchMetadataBatch::Dataset}. Implement methods from 
-{ResearchMetadataBatch::Custom} as resource-specific methods.
-
+## Example application
 This example uses Amazon Web Services.
 
-### Base class 
+### shared.rb
+Implement methods from {ResearchMetadataBatch::Shared}.
 ```ruby
-module ResearchMetadataBatch
-  class Base
+require 'aws-sdk-s3'
+
+module App
+  module Shared
     def init(aws_config:)
       aws_credentials = Aws::Credentials.new aws_config[:access_key_id],
                                              aws_config[:secret_access_key]
@@ -66,19 +60,21 @@ module ResearchMetadataBatch
 end
 ```
 
-### Resource class
+### research_output.rb
 ```ruby
-module ResearchMetadataBatch   
-  class Dataset    
-    # Implement methods from ResearchMetadataBatch::Custom
+require_relative 'shared'
+
+module App
+  class ResearchOutput < ResearchMetadataBatch::ResearchOutput
+    include App::Shared
   end  
 end
 ```
 
-### Running a batch process
+### script.rb
 ```ruby
 require 'research_metadata_batch'
-# require your opened classes
+require_relative 'research_output'
 
 pure_config = {
   url:      ENV['PURE_URL'],
@@ -101,7 +97,15 @@ config = {
   log_file: log_file
 }
 
-batch = ResearchMetadataBatch::Dataset.new config
+batch = App::ResearchOutput.new config
 batch.init aws_config: aws_config
-batch.process
+params = {
+  size: 50,
+  typeUri: [
+    '/dk/atira/pure/researchoutput/researchoutputtypes/contributiontojournal/article',
+    '/dk/atira/pure/researchoutput/researchoutputtypes/contributiontoconference/paper'
+  ]
+}
+batch.process params: params
+
 ```

data/lib/research_metadata_batch/base.rb

@@ -1,12 +1,12 @@
 require 'logger'
 require 'puree'
-require_relative 'custom'
+require_relative 'shared'
 
 module ResearchMetadataBatch
   # @note Not to be used directly
   class Base
-    include ResearchMetadataBatch::Custom
-   # @param pure_config [Hash]
+    include ResearchMetadataBatch::Shared
+    # @param pure_config [Hash]
     # @option config [String] :url
     # @option config [String] :username
     # @option config [String] :password
@@ -21,22 +21,18 @@ module ResearchMetadataBatch
       end
     end
 
+    # @param params [Hash] Combined GET and POST parameters for all records
     # @param max [Fixnum] Number of records to act upon. Omit to act upon as many as possible.
-    # @param limit [Fixnum] Pure records limit.
-    # @param offset [Fixnum] Pure records offset.
-    # @param action [Boolean] Set to false to mock an action.
-    # @param delay [Fixnum] Delay in seconds between limit-sized batches.
-    def process(max: nil, limit: 20, offset: 0, action: true, delay: 0)
-      records_available = resource_count
-
-      @logger.info "#{records_available} records in Pure before processing"
-      if action
-        begin
-          preflight
-          @logger.info preflight_success_log_message
-        rescue => error
-          @logger.info preflight_error_log_message(error)
-        end
+    # @param delay [Fixnum] Delay in seconds between batches.
+    def process(params: {}, max: nil, delay: 0)
+      offset = params[:offset]
+      records_available = resource_count params
+      @logger.info "PURE_RECORDS_AVAILABLE=#{records_available}"
+      begin
+        preflight_msg = preflight
+        @logger.info "PREFLIGHT=#{preflight_msg}" if preflight_msg
+      rescue => error
+        @logger.error "PREFLIGHT=#{error}"
       end
 
       if max
@@ -47,7 +43,7 @@ module ResearchMetadataBatch
         qty_to_find = records_available
       end
 
-      if offset < 0 || offset > records_available - 1
+      if !offset || offset < 0 || offset > records_available - 1
         offset = 0
       end
 
@@ -57,35 +53,27 @@ module ResearchMetadataBatch
       while position < records_available
         # extract from Pure
         begin
-          result = resource_batch limit, position
-        rescue => e
-          @logger.error e
+          params[:offset] = position
+          result = resource_batch params
+        rescue => error
+          @logger.error "METADATA_EXTRACTION=#{error}"
           sleep 10
           redo
         end
 
         result.each do |i|
 
-          record_validation_error = validate_record i
-          if record_validation_error
-            @logger.warn "#{log_message_prefix(position, i.uuid)} - VALIDATION_ERROR=#{record_validation_error}"
+          unless valid? i
+            @logger.info "#{log_message_prefix(position, i.uuid)} : VALID=false"
             position += 1
             next
           end
 
           begin
-            if action
-              act_msg = act i
-            else
-              act_msg = mock_act i
-            end
-            if act_msg
-              @logger.info "#{log_message_prefix(position, i.uuid)} - #{act_success_log_message(i, act_msg)}"
-            else
-              @logger.info "#{log_message_prefix(position, i.uuid)}"
-            end
+            act_msg = act i
+            @logger.info "#{log_message_prefix(position, i.uuid)} : #{act_msg}"
           rescue => error
-            @logger.error "#{log_message_prefix(position, i.uuid)} - ERROR=#{error}"
+            @logger.error "#{log_message_prefix(position, i.uuid)} : #{error}"
           end
 
           position += 1
@@ -98,37 +86,35 @@ module ResearchMetadataBatch
 
         # handle error response
         if result.empty?
-          @logger.error "PURE_RECORD=#{position} - ERROR=No data"
+          @logger.error "PURE_RECORD=#{position} : METADATA_EXTRACTION=No data"
           position += 1
         end
 
         sleep delay
       end
 
-      @logger.info "#{records_available} records in Pure after processing"
+      @logger.info "PURE_RECORDS_AVAILABLE=#{records_available}"
 
     end
 
     private
 
-    def act(model)
-      puts model.inspect
-    end
-
     # @return [String]
     def log_message_prefix(pure_record, pure_uuid)
-      "PURE_RECORD=#{pure_record} - PURE_UUID=#{pure_uuid}"
+      "PURE_RECORD=#{pure_record} : PURE_UUID=#{pure_uuid}"
     end
 
-    def resource_count
+    def resource_count(params)
+      params = params.dup
       resource_class = "Puree::Extractor::#{Puree::Util::String.titleize(@resource_type)}"
-      Object.const_get(resource_class).new(@pure_config).count
+      Object.const_get(resource_class).new(@pure_config).count(params)
     end
 
-    def resource_batch(limit, offset)
+    def resource_batch(params)
+      params = params.dup
       resource_method = "#{@resource_type}s".to_sym
       client = Puree::REST::Client.new(@pure_config).send resource_method
-      response = client.all params: {size: limit, offset: offset}
+      response = client.all_complex params: params
       Puree::XMLExtractor::Collection.send resource_method, response.to_s
     end
   end

data/lib/research_metadata_batch/research_output.rb

@@ -11,7 +11,7 @@ module ResearchMetadataBatch
 
     private
 
-    def resource_batch(limit, offset)
+    def resource_batch(params)
       research_outputs_hash = super
       research_outputs_array = []
       research_outputs_hash.each do |k, v|

data/lib/research_metadata_batch/{custom.rb → shared.rb}

@@ -1,7 +1,7 @@
 module ResearchMetadataBatch
 
   # @note These methods (except init) are used internally by {ResearchMetadataBatch::Base#process} and have been left public for documentation purposes only
-  module Custom
+  module Shared
 
     # Second stage initialisation, perhaps third party services.
     # @param args [Hash]
@@ -9,37 +9,20 @@ module ResearchMetadataBatch
     end
 
     # Anything to be done at the start of a batch run
+    # @return [String, nil] Optionally, return something to indicate what has been done.
     def preflight
     end
 
-    # Message when preflight method completes
-    # @return [String]
-    def preflight_success_log_message
-    end
-
-    # Message when preflight method does not complete
-    # @return [String]
-    def preflight_error_log_message(error)
-    end
-
     # Do something with model metadata
     # @return [String, nil] Optionally, return something transaction-specific, such as a code/ID from an external service.
     def act(model)
+      puts model.inspect
     end
 
-    # Message when act/mock_act completes
-    # @return [String]
-    def act_success_log_message(model, act_msg)
-    end
-
-    # Fake doing something with model metadata
-    # @return [String, nil]
-    def mock_act(model)
-    end
-
-    # Check for values in metadata
-    # @return [String, nil]
-    def validate_record(model)
+    # Check for values in metadata.
+    # @return [Boolean]
+    def valid?(model)
+      true
     end
 
   end

data/lib/research_metadata_batch/version.rb

@@ -1,4 +1,4 @@
 module ResearchMetadataBatch
   # Semantic version number
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/research_metadata_batch.gemspec

@@ -14,7 +14,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
   spec.required_ruby_version = '~> 2.1'
-  spec.add_dependency 'puree', '~> 2.2'
+  spec.add_dependency 'puree', '~> 2.3'
   spec.metadata = {
     "source_code_uri" => "https://github.com/lulibrary/research_metadata_batch"
   }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: research_metadata_batch
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Adrian Albin-Clark
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-05 00:00:00.000000000 Z
+date: 2018-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: puree
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '2.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '2.3'
 description: 
 email:
 - a.albin-clark@lancaster.ac.uk
@@ -39,7 +39,6 @@ files:
 - README.md
 - lib/research_metadata_batch.rb
 - lib/research_metadata_batch/base.rb
-- lib/research_metadata_batch/custom.rb
 - lib/research_metadata_batch/dataset.rb
 - lib/research_metadata_batch/event.rb
 - lib/research_metadata_batch/external_organisation.rb
@@ -50,6 +49,7 @@ files:
 - lib/research_metadata_batch/publisher.rb
 - lib/research_metadata_batch/research_metadata_batch.rb
 - lib/research_metadata_batch/research_output.rb
+- lib/research_metadata_batch/shared.rb
 - lib/research_metadata_batch/version.rb
 - research_metadata_batch.gemspec
 homepage: