research_metadata_batch 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9d27b60c2067c5d5faae6b2680f50186e4087c146bdb076f65955dfd05172397
-  data.tar.gz: 25c9789f3452dc091c2b180eac05f9038ff63f10152eaee9cc4a4fd68be2e862
+  metadata.gz: 6dfb82b4d3d35c9e38a757050f207c3adee991510369c98704ec77dafda3dff2
+  data.tar.gz: 1d4fecd07215e5e928bd76289d740f09c29a9c3144d566a03daf44d2086024c1
 SHA512:
-  metadata.gz: 103ffbdd89ab5f0056608f07c3f30fb388f294db3ec882cb5e3b4e729c83b0bfedac16ef5047b17b785c7988f065cab55fb32ca4311064bf50879fc1321dc70e
-  data.tar.gz: 15e0e3496b5194d4a15b1f07a9c8e69d64412e41622a352d62fc5f82dbc16d8dad621494c81e09b01b5c8213a19dbeb53221621e802a4616ecb1ad7b1df19ecf
+  metadata.gz: 75fc086bc6591d49058e2402293f964a6b0b00698e6236b0fdd50887b538951f585d5cd6a6e67a88b8d990b8f60b6a1fa37d4210077c647e2920f7065873c28e
+  data.tar.gz: c6363b1d95e72f6aeb9c332649d2c74fb2582bd100ce3ebcb2c7d701ea23ace6712a508678d721e587a4263fb456e2925afb2c5a0301e795930dc0c60ecc3386

data/CHANGELOG.md

@@ -2,6 +2,10 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## 0.4.0 - 2018-11-02
+### Changed
+- Logging as JSON.
+
 ## 0.3.0 - 2018-10-12
 ### Added
 - Support for GET and POST parameters.

data/Gemfile.lock

@@ -1,7 +1,8 @@
 PATH
   remote: .
   specs:
-    research_metadata_batch (0.3.0)
+    research_metadata_batch (0.4.0)
+      ougai (~> 1.7)
       puree (~> 2.3)
 
 GEM
@@ -21,10 +22,13 @@ GEM
     http-form_data (1.0.3)
     http_parser.rb (0.6.0)
     mini_portile2 (2.3.0)
-    nokogiri (1.8.4)
+    nokogiri (1.8.5)
       mini_portile2 (~> 2.3.0)
+    oj (3.6.10)
+    ougai (1.7.0)
+      oj (~> 3.4)
     public_suffix (3.0.3)
-    puree (2.3.0)
+    puree (2.4.0)
       http (~> 2.0)
       nokogiri (~> 1.6)
     unf (0.1.4)

data/README.md

@@ -35,26 +35,25 @@ pure_config = {
 ResearchMetadataBatch::Dataset.new(pure_config: pure_config).process
 ```
 
-## Example application
-This example uses Amazon Web Services.
+## Creating an application
+Either open up classes or create subclasses to implement application-specific behaviour.
+
+This example creates subclasses and uses Amazon Web Services.
 
 ### shared.rb
 Implement methods from {ResearchMetadataBatch::Shared}.
 ```ruby
-require 'aws-sdk-s3'
+# require aws sdk
 
 module App
   module Shared
     def init(aws_config:)
-      aws_credentials = Aws::Credentials.new aws_config[:access_key_id],
-                                             aws_config[:secret_access_key]
-      @s3_client = Aws::S3::Client.new region: aws_config[:region],
-                                       credentials: aws_credentials
-      @s3_bucket = aws_config[:s3_bucket]
+      # Do something with :aws_config
     end
   
     def act(model)
-      # Do something involving Amazon Web Services 
+      # Do something with Amazon Web Services
+      return {key1: 'some_value', key2: 'another_value', msg: 'what_happened'} 
     end
   end
 end
@@ -84,10 +83,7 @@ pure_config = {
 }
 
 aws_config = {
-  access_key_id: ENV['AWS_ACCESS_KEY_ID'],
-  secret_access_key: ENV['AWS_SECRET_ACCESS_KEY'],
-  region: ENV['AWS_REGION'],
-  s3_bucket: 'YOUR_S3_BUCKET'
+  # details
 }
 
 log_file = '/path/to/your/log/file'
@@ -108,4 +104,4 @@ params = {
 }
 batch.process params: params
 
-```
+```

data/lib/research_metadata_batch/base.rb

@@ -1,4 +1,4 @@
-require 'logger'
+require 'ougai'
 require 'puree'
 require_relative 'shared'
 
@@ -15,9 +15,9 @@ module ResearchMetadataBatch
     def initialize(pure_config:, log_file: nil)
       @pure_config = pure_config
       if log_file
-        @logger = Logger.new File.new(log_file, 'a'), 20, 'daily'
+        @logger = Ougai::Logger.new File.new(log_file, 'a'), 20, 'daily'
       else
-        @logger = Logger.new(STDOUT)
+        @logger = Ougai::Logger.new(STDOUT)
       end
     end
 
@@ -27,12 +27,12 @@ module ResearchMetadataBatch
     def process(params: {}, max: nil, delay: 0)
       offset = params[:offset]
       records_available = resource_count params
-      @logger.info "PURE_RECORDS_AVAILABLE=#{records_available}"
+      log_records_available records_available
       begin
-        preflight_msg = preflight
-        @logger.info "PREFLIGHT=#{preflight_msg}" if preflight_msg
+        preflight_h = preflight
+        @logger.info({preflight: preflight_h}) if preflight_h.is_a?(Hash) && !preflight_h.empty?
       rescue => error
-        @logger.error "PREFLIGHT=#{error}"
+        @logger.error({preflight: error})
       end
 
       if max
@@ -56,7 +56,7 @@ module ResearchMetadataBatch
           params[:offset] = position
           result = resource_batch params
         rescue => error
-          @logger.error "METADATA_EXTRACTION=#{error}"
+          @logger.error({metadata_extraction: error})
           sleep 10
           redo
         end
@@ -64,16 +64,20 @@ module ResearchMetadataBatch
         result.each do |i|
 
           unless valid? i
-            @logger.info "#{log_message_prefix(position, i.uuid)} : VALID=false"
+            @logger.info log_entry_core(position, i.uuid).merge({valid: false})
             position += 1
             next
           end
 
           begin
-            act_msg = act i
-            @logger.info "#{log_message_prefix(position, i.uuid)} : #{act_msg}"
+            act_h = act i
+            act_log_entry = log_entry_core(position, i.uuid)
+            act_log_entry.merge!(act_h) if act_h.is_a?(Hash) && !act_h.empty?
+            @logger.info act_log_entry
           rescue => error
-            @logger.error "#{log_message_prefix(position, i.uuid)} : #{error}"
+            act_error_log_entry = log_entry_core(position, i.uuid)
+            act_error_log_entry.merge!({error: error})
+            @logger.error act_error_log_entry
           end
 
           position += 1
@@ -86,22 +90,26 @@ module ResearchMetadataBatch
 
         # handle error response
         if result.empty?
-          @logger.error "PURE_RECORD=#{position} : METADATA_EXTRACTION=No data"
+          @logger.error({pure_record: position, metadata_extraction: 'No data'})
           position += 1
         end
 
         sleep delay
       end
 
-      @logger.info "PURE_RECORDS_AVAILABLE=#{records_available}"
+      log_records_available records_available
 
     end
 
     private
 
-    # @return [String]
-    def log_message_prefix(pure_record, pure_uuid)
-      "PURE_RECORD=#{pure_record} : PURE_UUID=#{pure_uuid}"
+    def log_records_available(count)
+      @logger.info({pure_records_available: count})
+    end
+
+    # @return [Hash]
+    def log_entry_core(pure_record, pure_uuid)
+      {pure_record: pure_record, pure_uuid: pure_uuid}
     end
 
     def resource_count(params)

data/lib/research_metadata_batch/shared.rb

@@ -9,12 +9,12 @@ module ResearchMetadataBatch
     end
 
     # Anything to be done at the start of a batch run
-    # @return [String, nil] Optionally, return something to indicate what has been done.
+    # @return [Hash, nil] Optionally, return something to indicate what has been done.
     def preflight
     end
 
     # Do something with model metadata
-    # @return [String, nil] Optionally, return something transaction-specific, such as a code/ID from an external service.
+    # @return [Hash, nil] Optionally, return something transaction-specific, such as a code/ID from an external service.
     def act(model)
       puts model.inspect
     end

data/lib/research_metadata_batch/version.rb

@@ -1,4 +1,4 @@
 module ResearchMetadataBatch
   # Semantic version number
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/research_metadata_batch.gemspec

@@ -15,6 +15,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.required_ruby_version = '~> 2.1'
   spec.add_dependency 'puree', '~> 2.3'
+  spec.add_dependency 'ougai', '~> 1.7'
   spec.metadata = {
     "source_code_uri" => "https://github.com/lulibrary/research_metadata_batch"
   }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: research_metadata_batch
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Adrian Albin-Clark
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-12 00:00:00.000000000 Z
+date: 2018-11-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: puree
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.3'
+- !ruby/object:Gem::Dependency
+  name: ougai
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
 description: 
 email:
 - a.albin-clark@lancaster.ac.uk