dmao-json-ingesters 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c22b68788e8a965c942d77d4bacf57cebfe02f04
+  data.tar.gz: ad8adcd8d0381e56ffe5a956d17fd203e1399091
+SHA512:
+  metadata.gz: e83095aa70991c0e88d9ac7bd2ce6fe10d3740dba9557f775a8d369e22a1523fc48e416a42eff6f32817a4c0e06cbd4c015d441fd9114179d64009bb6598d225
+  data.tar.gz: 8b8b64670231c9ca01822bd4e468a4ca07470a3c21bd565b803c6257d96b5b09faefe60cdc58d64f713aeb2e77ab13e084b694ea8a58da9896bfc33d2904ca0d

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.rbenv-gemsets

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in dmao-json-ingesters.gemspec
+gemspec

data/README.md

@@ -0,0 +1,36 @@
+# DMAO::Ingesters::JSON::OrganisationIngester
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/json/organisation/ingester`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'dmao-json-organisation-ingester'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install dmao-json-organisation-ingester
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/lulibrary/DMAO-JSON-Organisation-Ingester.
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "dmao/ingesters/json/ingesters"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/dmao-json-ingesters.gemspec

@@ -0,0 +1,42 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'dmao/ingesters/json/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "dmao-json-ingesters"
+  spec.version       = DMAO::Ingesters::JSON::VERSION
+  spec.authors       = ["Stephen Robinson"]
+  spec.email         = ["library.dit@lancaster.ac.uk"]
+
+  spec.summary       = %q{JSON ingesters for DMA Online}
+  spec.description   = %q{}
+  spec.homepage      = "https://github.com/lulibrary/dmao-json-ingesters"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = "https://rubygems.org"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against " \
+      "public gem pushes."
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "dmao-generic-ingesters", "~> 0.2.1"
+  spec.add_dependency "rest-client"
+
+  spec.add_development_dependency "bundler", "~> 1.13"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "mocha", "~> 1.1"
+  spec.add_development_dependency "codeclimate-test-reporter", "~> 1.0"
+  spec.add_development_dependency "vcr", "~> 3.0.3"
+  spec.add_development_dependency "webmock", "~> 2.1"
+end

data/lib/dmao/ingesters/json/errors/ingest_file_error.rb

@@ -0,0 +1,17 @@
+module DMAO
+  module Ingesters
+    module JSON
+      module Errors
+
+        class IngestFileError < StandardError
+
+          def initialize(msg="Error with specified ingest file.")
+            super(msg)
+          end
+
+        end
+
+      end
+    end
+  end
+end

data/lib/dmao/ingesters/json/ingesters.rb

@@ -0,0 +1,12 @@
+require "dmao/ingesters/json/version"
+require "dmao/ingesters/json/organisation_ingester"
+
+module DMAO
+  module Ingesters
+    module JSON
+      module Ingesters
+
+      end
+    end
+  end
+end

data/lib/dmao/ingesters/json/organisation_ingester.rb

@@ -0,0 +1,126 @@
+require 'dmao/ingesters/generic/organisation_ingester'
+require 'dmao/ingesters/json/errors/ingest_file_error'
+require 'dmao/ingesters/logging/ingest_error_log'
+require 'json'
+require 'securerandom'
+
+module DMAO
+  module Ingesters
+    module JSON
+
+      class OrganisationIngester < DMAO::Ingesters::Generic::OrganisationIngester
+
+        include DMAO::Ingesters::Logging::IngestErrorLog
+
+        VALID_INGESTER_OPTIONS = [:file]
+
+        def initialize(api_url=nil, api_token=nil, institution_id=nil)
+          super(api_url, api_token, institution_id)
+          @ignore_for_linking_uuids = []
+          create_logger
+        end
+
+        def ingest options={}
+
+          options.keep_if { |k, _v| VALID_INGESTER_OPTIONS.include? k }
+
+          raise ArgumentError.new("No ingest options passed in, file option is required") if options.nil? || options.empty?
+          raise ArgumentError.new("No ingest options passed in, file option is required") if options[:file].nil? || options[:file].empty?
+
+          organisation_units = parse_json_file options[:file]
+
+          ingest_organisation_units organisation_units
+          link_organisation_units organisation_units
+
+          close_log_file
+
+          ingest_successful?
+
+        end
+
+        private
+
+        def ingest_successful?
+          logged_errors? ? false : true
+        end
+
+        def parse_json_file filepath
+
+          begin
+
+            file = File.read(filepath)
+
+            organisation_units = ::JSON.parse(file)["organisation_units"]
+
+          rescue Errno::ENOENT
+            raise DMAO::Ingesters::JSON::Errors::IngestFileError.new "Error reading file no such file or directory @ #{filepath}"
+          rescue ::JSON::ParserError
+            raise DMAO::Ingesters::JSON::Errors::IngestFileError.new "Error reading file, invalid json data in #{filepath}"
+          end
+
+          if organisation_units.nil? || organisation_units.empty?
+            raise DMAO::Ingesters::JSON::Errors::IngestFileError.new "Error with JSON data, no organisation units array defined in #{filepath}"
+          end
+
+          organisation_units
+
+        end
+
+        def ingest_organisation_units organisation_units
+
+          organisation_units.each do |unit|
+
+            details = {
+                name: unit["details"]["name"],
+                description: unit["details"]["description"],
+                url: unit["details"]["url"],
+                system_uuid: unit["system"]["uuid"],
+                system_modified_at: unit["system"]["modified_at"],
+                isni: unit["details"]["isni"],
+                unit_type: unit["details"]["type"]
+            }
+
+            begin
+              ingest_organisation_unit details
+            rescue ::DMAO::Ingesters::Errors::EmptyAttributes, ::DMAO::Ingesters::Errors::IngestOrganisationUnitError => e
+              log_ingest_error details[:system_uuid], e.message
+              ignore_for_linking details[:system_uuid]
+            end
+
+          end
+
+        end
+
+        def link_organisation_units organisation_units
+
+          organisation_units.each do |unit|
+
+            next if ignore_for_linking? unit["system"]["uuid"]
+
+            next if unit["parent"]["uuid"].nil? || unit["parent"]["uuid"].empty?
+
+            next if ignore_for_linking? unit["parent"]["uuid"]
+
+            begin
+              link_child_to_parent unit["system"]["uuid"], unit["parent"]["uuid"]
+            rescue ::DMAO::Ingesters::Errors::LinkOrganisationUnitError, ::DMAO::Ingesters::Errors::IngestOrganisationUnitError => e
+              log_ingest_error unit["system"]["uuid"], e.message
+            end
+
+          end
+
+        end
+
+        def ignore_for_linking system_uuid
+          @ignore_for_linking_uuids << system_uuid
+        end
+
+        def ignore_for_linking? system_uuid
+          @ignore_for_linking_uuids.include? system_uuid
+        end
+
+      end
+
+    end
+  end
+end

data/lib/dmao/ingesters/json/version.rb

@@ -0,0 +1,7 @@
+module DMAO
+  module Ingesters
+    module JSON
+      VERSION = "0.1.0"
+    end
+  end
+end

data/lib/dmao/ingesters/jwt_errors/invalid_jwt_api_base_url.rb

@@ -0,0 +1,15 @@
+module DMAO
+  module Ingesters
+    module JWTErrors
+
+      class InvalidJWTApiBaseUrl < StandardError
+
+        def initialize(msg="JWT Api base url is not set please check the config file.")
+          super(msg)
+        end
+
+      end
+
+    end
+  end
+end

data/lib/dmao/ingesters/jwt_errors/invalid_jwt_api_token.rb

@@ -0,0 +1,15 @@
+module DMAO
+  module Ingesters
+    module JWTErrors
+
+      class InvalidJWTApiToken < StandardError
+
+        def initialize(msg="JWT Api token is not set please check the config file.")
+          super(msg)
+        end
+
+      end
+
+    end
+  end
+end

data/lib/dmao/ingesters/jwt_errors/invalid_jwt_issuer.rb

@@ -0,0 +1,15 @@
+module DMAO
+  module Ingesters
+    module JWTErrors
+
+      class InvalidJWTIssuer < StandardError
+
+        def initialize(msg="JWT Issuer is not set please check the config file.")
+          super(msg)
+        end
+
+      end
+
+    end
+  end
+end

data/lib/dmao/ingesters/jwt_errors/unable_to_issue_token.rb

@@ -0,0 +1,15 @@
+module DMAO
+  module Ingesters
+    module JWTErrors
+
+      class UnableToIssueToken < StandardError
+
+        def initialize()
+          super
+        end
+
+      end
+
+    end
+  end
+end

data/lib/dmao/ingesters/jwt_service.rb

@@ -0,0 +1,51 @@
+require 'json'
+require 'rest-client'
+require 'dmao/ingesters/jwt_errors/invalid_jwt_api_base_url'
+require 'dmao/ingesters/jwt_errors/invalid_jwt_api_token'
+require 'dmao/ingesters/jwt_errors/invalid_jwt_issuer'
+require 'dmao/ingesters/jwt_errors/unable_to_issue_token'
+
+module DMAO
+  module Ingesters
+
+    class JWTService
+
+      def self.request_jwt_for_institution_ingest institution_id, ingest_id
+
+        api_base_url = ENV['DMAO_JWT_API_URL']
+        api_token = ENV['DMAO_JWT_API_TOKEN']
+        jwt_issuer = ENV['DMAO_JWT_ISSUER']
+
+        raise DMAO::Ingesters::JWTErrors::InvalidJWTApiBaseUrl if api_base_url.nil? || api_base_url.empty?
+        raise DMAO::Ingesters::JWTErrors::InvalidJWTApiToken if api_token.nil? || api_token.empty?
+        raise DMAO::Ingesters::JWTErrors::InvalidJWTIssuer if jwt_issuer.nil? || jwt_issuer.empty?
+
+        subject = "dmao_ingest_#{ingest_id}"
+        custom_claims = {
+            institution_id: institution_id,
+            roles: ['institution_admin']
+        }
+
+        request_attributes = {
+            subject: subject,
+            custom_claims: custom_claims
+        }
+
+        rest_client = RestClient::Resource.new(api_base_url, headers: { content_type: :json, authorization: "Bearer #{api_token}" })
+
+        begin
+          response = rest_client["#{jwt_issuer}/tokens"].post request_attributes.to_json
+        rescue RestClient::NotFound
+          raise DMAO::Ingesters::JWTErrors::InvalidJWTIssuer.new('JWT Issuer not found at JWT Service')
+        rescue RestClient::UnprocessableEntity, RestClient::InternalServerError
+          raise DMAO::Ingesters::JWTErrors::UnableToIssueToken
+        end
+
+        ::JSON.parse(response.body)["token"]
+
+      end
+
+    end
+
+  end
+end

data/lib/dmao/ingesters/logging/errors/bad_file_name.rb

@@ -0,0 +1,17 @@
+module DMAO
+  module Ingesters
+    module Logging
+      module Errors
+
+        class BadFileName < StandardError
+
+          def initialize(msg="Filename not specified")
+            super(msg)
+          end
+
+        end
+
+      end
+    end
+  end
+end

data/lib/dmao/ingesters/logging/errors/logger_not_initialized.rb

@@ -0,0 +1,17 @@
+module DMAO
+  module Ingesters
+    module Logging
+      module Errors
+
+        class LoggerNotInitialized < StandardError
+
+          def initialize(msg="Logger is not initialized")
+            super(msg)
+          end
+
+        end
+
+      end
+    end
+  end
+end

data/lib/dmao/ingesters/logging/ingest_error_log.rb

@@ -0,0 +1,46 @@
+require 'dmao/ingesters/logging/ingest_logger'
+require 'dmao/ingesters/logging/errors/logger_not_initialized'
+
+module DMAO
+  module Ingesters
+    module Logging
+
+      module IngestErrorLog
+
+        def create_logger
+          time = Time.now
+          time_formatted = time.strftime("%Y-%m-%d_%H:%M:%S")
+          filename = "ingest_#{time_formatted}"
+          @logged_errors = false
+          @logger = DMAO::Ingesters::Logging::IngestLogger.new filename
+        end
+
+        def log_ingest_error system_uuid, error_message
+
+          raise DMAO::Ingesters::Logging::Errors::LoggerNotInitialized.new if @logger.nil?
+
+          @logged_errors = true
+
+          @logger.error("#{system_uuid} - #{error_message}")
+
+        end
+
+        def close_log_file
+          raise DMAO::Ingesters::Logging::Errors::LoggerNotInitialized.new if @logger.nil?
+          @logger.close
+        end
+
+        def get_log_file_path
+          raise DMAO::Ingesters::Logging::Errors::LoggerNotInitialized.new if @logger.nil?
+          @logger.log_file_path
+        end
+
+        def logged_errors?
+          @logged_errors
+        end
+
+      end
+
+    end
+  end
+end

data/lib/dmao/ingesters/logging/ingest_logger.rb

@@ -0,0 +1,40 @@
+require 'logger'
+require 'dmao/ingesters/logging/errors/bad_file_name'
+
+module DMAO
+  module Ingesters
+    module Logging
+
+      class IngestLogger < Logger
+
+        attr_reader :log_file_path
+
+        def initialize filename=nil
+
+          raise DMAO::Ingesters::Logging::Errors::BadFileName.new("Filename needs to be specified for ingest logger") if filename.nil? || filename.empty?
+
+          log_directory = ENV['TEMP_INGEST_LOG_DIRECTORY'] || '/tmp/'
+
+          @log_file_path = log_directory + filename + '.log'
+
+          super(@log_file_path)
+
+          self.formatter = formatter
+
+          self
+
+        end
+
+        def formatter
+          Proc.new{|severity, time, _progname, msg|
+            formatted_severity = sprintf("%-5s", severity.to_s)
+            formatted_time = time.strftime("%Y-%m-%d %H:%M:%S")
+            "[#{formatted_severity} #{formatted_time}] #{msg.to_s.strip}\n"
+          }
+        end
+
+      end
+
+    end
+  end
+end

metadata

@@ -0,0 +1,190 @@
+--- !ruby/object:Gem::Specification
+name: dmao-json-ingesters
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Stephen Robinson
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-03-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: dmao-generic-ingesters
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: codeclimate-test-reporter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.3
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+description: ''
+email:
+- library.dit@lancaster.ac.uk
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- dmao-json-ingesters.gemspec
+- lib/dmao/ingesters/json/errors/ingest_file_error.rb
+- lib/dmao/ingesters/json/ingesters.rb
+- lib/dmao/ingesters/json/organisation_ingester.rb
+- lib/dmao/ingesters/json/version.rb
+- lib/dmao/ingesters/jwt_errors/invalid_jwt_api_base_url.rb
+- lib/dmao/ingesters/jwt_errors/invalid_jwt_api_token.rb
+- lib/dmao/ingesters/jwt_errors/invalid_jwt_issuer.rb
+- lib/dmao/ingesters/jwt_errors/unable_to_issue_token.rb
+- lib/dmao/ingesters/jwt_service.rb
+- lib/dmao/ingesters/logging/errors/bad_file_name.rb
+- lib/dmao/ingesters/logging/errors/logger_not_initialized.rb
+- lib/dmao/ingesters/logging/ingest_error_log.rb
+- lib/dmao/ingesters/logging/ingest_logger.rb
+homepage: https://github.com/lulibrary/dmao-json-ingesters
+licenses: []
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: JSON ingesters for DMA Online
+test_files: []