research_metadata_batch 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: bc61ec93f5d5bffe445daa23c318a5ddd2a829b01a799a5d46866e9094f9d594
+  data.tar.gz: b0b0a8d00afb219911466efb16b3ec024a670af3e307a86426ed0c13de84e1fd
+SHA512:
+  metadata.gz: 412d324f9e61ed97b51f3b8a8c1fbb42d9f93f41dd12d98c4487922816cfee9d9387c7e06cc6a0041fc8ccf7096eb352ded2345e8208d6f70a7f8cf45a6ca063
+  data.tar.gz: 3235149ee8993d812b876165efa311017ea42ef80747c5c6730b0da3bc27c0208884fbabaf02676bd1da253756e25c1690bb00b76a32137e6dce28c321480f08

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.idea

data/CHANGELOG.md

@@ -0,0 +1,17 @@
+# Change Log
+All notable changes to this project will be documented in this file.
+This project adheres to [Semantic Versioning](http://semver.org/).
+
+## 0.1.0 - 2018-09-27
+### Added
+- Support for resource types: 
+    - dataset 
+    - event 
+    - external_organisation
+    - journal
+    - person
+    - organisational_unit
+    - project
+    - publisher
+    - research_output 
+- For Pure API 59.

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in research_metadata_batch.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,41 @@
+PATH
+  remote: .
+  specs:
+    research_metadata_batch (0.1.0)
+      puree (~> 2.2)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    domain_name (0.5.20180417)
+      unf (>= 0.0.5, < 1.0.0)
+    http (2.2.2)
+      addressable (~> 2.3)
+      http-cookie (~> 1.0)
+      http-form_data (~> 1.0.1)
+      http_parser.rb (~> 0.6.0)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    http-form_data (1.0.3)
+    http_parser.rb (0.6.0)
+    mini_portile2 (2.3.0)
+    nokogiri (1.8.4)
+      mini_portile2 (~> 2.3.0)
+    public_suffix (3.0.3)
+    puree (2.2.0)
+      http (~> 2.0)
+      nokogiri (~> 1.6)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.5)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  research_metadata_batch!
+
+BUNDLED WITH
+   1.16.1

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Adrian Albin-Clark
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,106 @@
+# Research Metadata Batch
+For the batch processing of Pure records. Custom actions and log messages can be 
+defined in user-defined applications.
+
+## Status
+
+[![Gem Version](https://badge.fury.io/rb/research_metadata_batch.svg)](https://badge.fury.io/rb/research_metadata_batch)
+[![Maintainability](https://api.codeclimate.com/v1/badges/d3d1723f2900c3e4774a/maintainability)](https://codeclimate.com/github/lulibrary/research-metadata-batch/maintainability)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'research_metadata_batch'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install research_metadata_batch
+
+## Basic usage
+Uses the default gem behaviour which merely inspects the metadata models using STDOUT.
+
+```ruby 
+pure_config = {
+  url:      ENV['PURE_URL'],
+  username: ENV['PURE_USERNAME'],
+  password: ENV['PURE_PASSWORD'],
+  api_key:  ENV['PURE_API_KEY']
+}
+ResearchMetadataBatch::Dataset.new(pure_config: pure_config).process
+```
+
+## Making an application
+Require this gem, then open up the base class {ResearchMetadataBatch::Base} as below. Implement methods from 
+{ResearchMetadataBatch::Custom} as inherited methods, including any secondary initialisation using the 
+``init`` method.
+ 
+ 
+For resource-specific customisation, open up a resource class e.g. {ResearchMetadataBatch::Dataset}. Implement methods from 
+{ResearchMetadataBatch::Custom} as resource-specific methods.
+
+This example uses Amazon Web Services.
+
+### Base class 
+```ruby
+module ResearchMetadataBatch
+  class Base
+    def init(aws_config:)
+      aws_credentials = Aws::Credentials.new aws_config[:access_key_id],
+                                             aws_config[:secret_access_key]
+      @s3_client = Aws::S3::Client.new region: aws_config[:region],
+                                       credentials: aws_credentials
+      @s3_bucket = aws_config[:s3_bucket]
+    end
+  
+    def act(model)
+      # Do something involving Amazon Web Services 
+    end
+  end
+end
+```
+
+### Resource class
+```ruby
+module ResearchMetadataBatch   
+  class Dataset    
+    # Implement methods from ResearchMetadataBatch::Custom
+  end  
+end
+```
+
+### Running a batch process
+```ruby
+require_relative '/path/to/your/opened/class'
+
+pure_config = {
+  url:      ENV['PURE_URL'],
+  username: ENV['PURE_USERNAME'],
+  password: ENV['PURE_PASSWORD'],
+  api_key:  ENV['PURE_API_KEY']
+}
+
+aws_config = {
+  access_key_id: ENV['AWS_ACCESS_KEY_ID'],
+  secret_access_key: ENV['AWS_SECRET_ACCESS_KEY'],
+  region: ENV['AWS_REGION'],
+  s3_bucket: 'YOUR_S3_BUCKET'
+}
+
+log_file = '/path/to/your/log/file'
+
+config = {
+  pure_config: pure_config,
+  log_file: log_file
+}
+
+batch = ResearchMetadataBatch::Dataset.new config
+batch.init aws_config: aws_config
+batch.process
+```

data/lib/research_metadata_batch.rb

@@ -0,0 +1,6 @@
+require 'research_metadata_batch/version'
+require 'research_metadata_batch/research_metadata_batch'
+
+# Batch processing for the Pure Research Information System.
+module ResearchMetadataBatch
+end

data/lib/research_metadata_batch/base.rb

@@ -0,0 +1,134 @@
+require 'logger'
+require 'puree'
+require_relative 'custom'
+
+module ResearchMetadataBatch
+  # @note Not to be used directly
+  class Base
+    include ResearchMetadataBatch::Custom
+   # @param pure_config [Hash]
+    # @option config [String] :url
+    # @option config [String] :username
+    # @option config [String] :password
+    # @option config [String] :api_key
+   # @param log_file [String]
+    def initialize(pure_config:, log_file: nil)
+      @pure_config = pure_config
+      if log_file
+        @logger = Logger.new File.new(log_file, 'a'), 20, 'daily'
+      else
+        @logger = Logger.new(STDOUT)
+      end
+    end
+
+    # @param max [Fixnum] Number of records to act upon. Omit to act upon as many as possible.
+    # @param limit [Fixnum] Pure records limit.
+    # @param offset [Fixnum] Pure records offset.
+    # @param action [Boolean] Set to false to mock an action.
+    # @param delay [Fixnum] Delay in seconds between limit-sized batches.
+    def process(max: nil, limit: 20, offset: 0, action: true, delay: 0)
+      records_available = resource_count
+
+      @logger.info "#{records_available} records in Pure before processing"
+      if action
+        begin
+          preflight
+          @logger.info preflight_success_log_message
+        rescue => error
+          @logger.info preflight_error_log_message(error)
+        end
+      end
+
+      if max
+        if max >= 0 && max <= records_available
+          qty_to_find = max
+        end
+      else
+        qty_to_find = records_available
+      end
+
+      if offset < 0 || offset > records_available - 1
+        offset = 0
+      end
+
+      qty_obtained = 0
+      position = offset
+
+      while position < records_available
+        # extract from Pure
+        begin
+          result = resource_batch limit, position
+        rescue => e
+          @logger.error e
+          sleep 10
+          redo
+        end
+
+        result.each do |i|
+
+          if !record_valid? i
+            @logger.warn "#{log_message_prefix(position, i.uuid)} - record invalid"
+            position += 1
+            next
+          end
+
+          begin
+            if action
+              act_msg = act i
+            else
+              act_msg = mock_act i
+            end
+            @logger.info "#{log_message_prefix(position, i.uuid)} - #{act_success_log_message(i, act_msg)}" if act_msg
+          rescue => error
+            @logger.error "#{log_message_prefix(position, i.uuid)} - ERROR=#{error}"
+          end
+
+          position += 1
+          qty_obtained += 1
+
+          break if qty_obtained == qty_to_find
+        end
+
+        break if qty_obtained == qty_to_find
+
+        # handle error response
+        if result.empty?
+          @logger.error "#{log_message_prefix(position, nil)} - ERROR=system"
+          position += 1
+        end
+
+        sleep delay
+      end
+
+      @logger.info "#{records_available} records in Pure after processing"
+
+    end
+
+    private
+
+    def act(model)
+      puts model.inspect
+    end
+
+    def record_valid?(model)
+      true
+    end
+
+    # @return [String]
+    def log_message_prefix(pure_record, pure_uuid)
+      "PURE_RECORD=#{pure_record} - PURE_UUID=#{pure_uuid}"
+    end
+
+    def resource_count
+      resource_class = "Puree::Extractor::#{Puree::Util::String.titleize(@resource_type)}"
+      Object.const_get(resource_class).new(@pure_config).count
+    end
+
+    def resource_batch(limit, offset)
+      resource_method = "#{@resource_type}s".to_sym
+      client = Puree::REST::Client.new(@pure_config).send resource_method
+      response = client.all params: {size: limit, offset: offset}
+      Puree::XMLExtractor::Collection.send resource_method, response.to_s
+    end
+  end
+end

data/lib/research_metadata_batch/custom.rb

@@ -0,0 +1,47 @@
+module ResearchMetadataBatch
+
+  # @note These methods (except init) are used internally by {ResearchMetadataBatch::Base#process} and have been left public for documentation purposes only
+  module Custom
+
+    # Second stage initialisation, perhaps third party services.
+    # @param args [Hash]
+    def init(**args)
+    end
+
+    # Anything to be done at the start of a batch run
+    def preflight
+    end
+
+    # Message when preflight method completes
+    # @return [String]
+    def preflight_success_log_message
+    end
+
+    # Message when preflight method does not complete
+    # @return [String]
+    def preflight_error_log_message(error)
+    end
+
+    # Do something with model metadata
+    # @return [String, nil] Optionally, return something transaction-specific, such as a code/ID from an external service.
+    def act(model)
+    end
+
+    # Message when act/mock_act completes
+    # @return [String]
+    def act_success_log_message(model, act_msg)
+    end
+
+    # Fake doing something with model metadata
+    # @return [String, nil]
+    def mock_act(model)
+    end
+
+    # Check for values in metadata
+    # @return [Boolean]
+    def record_valid?(model)
+    end
+
+  end
+
+end

data/lib/research_metadata_batch/dataset.rb

@@ -0,0 +1,14 @@
+require_relative 'base'
+
+module ResearchMetadataBatch
+
+  class Dataset < ResearchMetadataBatch::Base
+    # (see ResearchMetadataBatch::Base#initialize)
+    def initialize(pure_config:, log_file: nil)
+      super
+      @resource_type = :dataset
+    end
+
+  end
+
+end

data/lib/research_metadata_batch/event.rb

@@ -0,0 +1,14 @@
+require_relative 'base'
+
+module ResearchMetadataBatch
+
+  class Event < ResearchMetadataBatch::Base
+    # (see ResearchMetadataBatch::Base#initialize)
+    def initialize(pure_config:, log_file: nil)
+      super
+      @resource_type = :event
+    end
+
+  end
+
+end

data/lib/research_metadata_batch/external_organisation.rb

@@ -0,0 +1,14 @@
+require_relative 'base'
+
+module ResearchMetadataBatch
+
+  class ExternalOrganisation < ResearchMetadataBatch::Base
+    # (see ResearchMetadataBatch::Base#initialize)
+    def initialize(pure_config:, log_file: nil)
+      super
+      @resource_type = :external_organisation
+    end
+
+  end
+
+end

data/lib/research_metadata_batch/journal.rb

@@ -0,0 +1,14 @@
+require_relative 'base'
+
+module ResearchMetadataBatch
+
+  class Journal < ResearchMetadataBatch::Base
+    # (see ResearchMetadataBatch::Base#initialize)
+    def initialize(pure_config:, log_file: nil)
+      super
+      @resource_type = :journal
+    end
+
+  end
+
+end

data/lib/research_metadata_batch/organisational_unit.rb

@@ -0,0 +1,14 @@
+require_relative 'base'
+
+module ResearchMetadataBatch
+
+  class OrganisationalUnit < ResearchMetadataBatch::Base
+    # (see ResearchMetadataBatch::Base#initialize)
+    def initialize(pure_config:, log_file: nil)
+      super
+      @resource_type = :organisational_unit
+    end
+
+  end
+
+end

data/lib/research_metadata_batch/person.rb

@@ -0,0 +1,14 @@
+require_relative 'base'
+
+module ResearchMetadataBatch
+
+  class Person < ResearchMetadataBatch::Base
+    # (see ResearchMetadataBatch::Base#initialize)
+    def initialize(pure_config:, log_file: nil)
+      super
+      @resource_type = :person
+    end
+
+  end
+
+end

data/lib/research_metadata_batch/project.rb

@@ -0,0 +1,14 @@
+require_relative 'base'
+
+module ResearchMetadataBatch
+
+  class Project < ResearchMetadataBatch::Base
+    # (see ResearchMetadataBatch::Base#initialize)
+    def initialize(pure_config:, log_file: nil)
+      super
+      @resource_type = :project
+    end
+
+  end
+
+end

data/lib/research_metadata_batch/publisher.rb

@@ -0,0 +1,14 @@
+require_relative 'base'
+
+module ResearchMetadataBatch
+
+  class Publisher < ResearchMetadataBatch::Base
+    # (see ResearchMetadataBatch::Base#initialize)
+    def initialize(pure_config:, log_file: nil)
+      super
+      @resource_type = :publisher
+    end
+
+  end
+
+end

data/lib/research_metadata_batch/research_metadata_batch.rb

@@ -0,0 +1,9 @@
+require_relative 'dataset'
+require_relative 'event'
+require_relative 'external_organisation'
+require_relative 'journal'
+require_relative 'organisational_unit'
+require_relative 'person'
+require_relative 'project'
+require_relative 'publisher'
+require_relative 'research_output'

data/lib/research_metadata_batch/research_output.rb

@@ -0,0 +1,25 @@
+require_relative 'base'
+
+module ResearchMetadataBatch
+
+  class ResearchOutput < ResearchMetadataBatch::Base
+    # (see ResearchMetadataBatch::Base#initialize)
+    def initialize(pure_config:, log_file: nil)
+      super
+      @resource_type = :research_output
+    end
+
+    private
+
+    def resource_batch(limit, offset)
+      research_outputs_hash = super
+      research_outputs_array = []
+      research_outputs_hash.each do |k, v|
+        research_outputs_array += v
+      end
+      research_outputs_array
+    end
+
+  end
+
+end

data/lib/research_metadata_batch/version.rb

@@ -0,0 +1,4 @@
+module ResearchMetadataBatch
+  # Semantic version number
+  VERSION = "0.1.0"
+end

data/research_metadata_batch.gemspec

@@ -0,0 +1,19 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "research_metadata_batch/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "research_metadata_batch"
+  spec.version       = ResearchMetadataBatch::VERSION
+  spec.authors       = ["Adrian Albin-Clark"]
+  spec.email         = ["a.albin-clark@lancaster.ac.uk"]
+  spec.summary       = %q{Batch processing for the Pure Research Information System.}
+  spec.homepage      = 'https://github.com/lulibrary/research_metadata_batch'
+  spec.license       = "MIT"
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+  spec.required_ruby_version = '~> 2.1'
+  spec.add_dependency 'puree', '~> 2.2'
+end

metadata

@@ -0,0 +1,79 @@
+--- !ruby/object:Gem::Specification
+name: research_metadata_batch
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Adrian Albin-Clark
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-10-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: puree
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+description: 
+email:
+- a.albin-clark@lancaster.ac.uk
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- CHANGELOG.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- lib/research_metadata_batch.rb
+- lib/research_metadata_batch/base.rb
+- lib/research_metadata_batch/custom.rb
+- lib/research_metadata_batch/dataset.rb
+- lib/research_metadata_batch/event.rb
+- lib/research_metadata_batch/external_organisation.rb
+- lib/research_metadata_batch/journal.rb
+- lib/research_metadata_batch/organisational_unit.rb
+- lib/research_metadata_batch/person.rb
+- lib/research_metadata_batch/project.rb
+- lib/research_metadata_batch/publisher.rb
+- lib/research_metadata_batch/research_metadata_batch.rb
+- lib/research_metadata_batch/research_output.rb
+- lib/research_metadata_batch/version.rb
+- research_metadata_batch.gemspec
+homepage: https://github.com/lulibrary/research_metadata_batch
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '2.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.3
+signing_key: 
+specification_version: 4
+summary: Batch processing for the Pure Research Information System.
+test_files: []