pushmi_pullyu 2.0.7 → 2.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de3e5873be562beedd8ebf27f364d50e2ac993f6331cfd27480614de061a01ab
-  data.tar.gz: 5e07018702ff3f8660db82ee6e2cb481c650cb10413172eb36d4791e2ec97d3a
+  metadata.gz: b4380518d529c79ac7972e3ab29dee3524082df9e66a3f0f1e3eefbb5c08855d
+  data.tar.gz: c43b7f5a944ceb4fafa298ebbb7ec47725a8bbff04a8b04ea51874b6bacc2ba6
 SHA512:
-  metadata.gz: 4ac86a98c6c63c3d8fe32c7f8238d676fcfd5193b291bf115af9999351d0ac36a23e84e4d75f4443bea2b7f6c73cfcc309117a8d02aba1292d9044014229f5c2
-  data.tar.gz: a01c230a0ac56783ae878bdc2fd83ac351706623c612d61d270ad005b47d550d28d4fc4ef1a0c2c7f09c1caa17c837157b72e8233f4a64d7c033418546a72fbb
+  metadata.gz: 0c7e9b00f0e8a6727393da41e9060dff06a6b8aba0baf9f7ed84bbc89e17e0988e1858a8c0bd925983ccda379e249374af0c738be976ec21708a3cf6be35b20e
+  data.tar.gz: 6ed33393e60e371ea74b9eba143ff3ce3f6ddac81954ddf944d68f3d75087f2adc777ac4f8a8db4037c20131201ec0f1325a90d465b3cd06be6fa04a878a728d

data/CHANGELOG.md

@@ -6,7 +6,14 @@ PushmiPullyu is a Ruby application, whose primary job is to manage the flow of c
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and releases in PushmiPullyu adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [2.1.1]
+ - Increase clarity of log files [#433](https://github.com/ualbertalib/pushmi_pullyu/issues/433)
+
+## [2.1.0]
+ - Add more logging information [#433](https://github.com/ualbertalib/pushmi_pullyu/issues/433)
+ - Add V3 authentication [#349](https://github.com/ualbertalib/pushmi_pullyu/issues/349)
+
+- Add logic to perform authentication against the V3 Auth protocol
 
 ## [2.0.7] - 2023-09-13
 

data/Gemfile

@@ -9,8 +9,8 @@ group :development, :test do
   gem 'pry', '~> 0.10', '>= 0.10.4'
   gem 'pry-byebug', '~> 3.6'
   gem 'rake', '~> 13.0'
-  gem 'rspec', '~> 3.0'
-  gem 'rubocop', '~> 1.57'
+  gem 'rspec', '~> 3.13'
+  gem 'rubocop', '~> 1.64'
   gem 'rubocop-rspec', '~> 2.24'
   gem 'timecop', '~> 0.9'
   gem 'vcr', '~> 5.0'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pushmi_pullyu (2.0.7)
+    pushmi_pullyu (2.1.1)
       activesupport (>= 5, < 8)
       bagit (~> 0.4)
       connection_pool (~> 2.2)
@@ -34,7 +34,7 @@ GEM
     bagit (0.4.6)
       docopt (~> 0.5.0)
       validatable (~> 1.6)
-    base64 (0.1.1)
+    base64 (0.2.0)
     bigdecimal (3.1.4)
     builder (3.2.4)
     byebug (11.1.3)
@@ -65,7 +65,7 @@ GEM
       no_proxy_fix
       octokit (~> 6.0)
       terminal-table (>= 1, < 4)
-    diff-lcs (1.5.0)
+    diff-lcs (1.5.1)
     docopt (0.5.0)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
@@ -94,7 +94,7 @@ GEM
       domain_name (~> 0.5)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    json (2.6.3)
+    json (2.7.2)
     kramdown (2.4.0)
       rexml
     kramdown-parser-gfm (1.1.0)
@@ -123,8 +123,8 @@ GEM
     open4 (1.3.4)
     openstack (3.3.21)
       json
-    parallel (1.23.0)
-    parser (3.2.2.4)
+    parallel (1.24.0)
+    parser (3.3.1.0)
       ast (~> 2.4.1)
       racc
     pry (0.14.2)
@@ -134,7 +134,7 @@ GEM
       byebug (~> 11.0)
       pry (>= 0.13, < 0.15)
     public_suffix (5.0.3)
-    racc (1.7.1)
+    racc (1.8.0)
     rainbow (3.1.1)
     rake (13.0.6)
     rchardet (1.8.0)
@@ -150,45 +150,45 @@ GEM
     rdf-xsd (3.2.1)
       rdf (~> 3.2)
       rexml (~> 3.2)
-    redis (5.0.7)
-      redis-client (>= 0.9.0)
-    redis-client (0.17.0)
+    redis (5.0.8)
+      redis-client (>= 0.17.0)
+    redis-client (0.17.1)
       connection_pool
-    regexp_parser (2.8.2)
+    regexp_parser (2.9.2)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
-    rexml (3.2.6)
+    rexml (3.2.8)
+      strscan (>= 3.0.9)
     rollbar (3.4.1)
-    rspec (3.12.0)
-      rspec-core (~> 3.12.0)
-      rspec-expectations (~> 3.12.0)
-      rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.0)
-      rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.0)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.0)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.0)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-support (3.12.0)
-    rubocop (1.57.1)
-      base64 (~> 0.1.1)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.0)
+    rubocop (1.64.0)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.2.2.4)
+      parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.28.1, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.29.0)
-      parser (>= 3.2.1.0)
+    rubocop-ast (1.31.3)
+      parser (>= 3.3.1.0)
     rubocop-capybara (2.19.0)
       rubocop (~> 1.41)
     rubocop-factory_bot (2.24.0)
@@ -215,6 +215,7 @@ GEM
     sparql-client (3.2.2)
       net-http-persistent (~> 4.0, >= 4.0.2)
       rdf (~> 3.2, >= 3.2.11)
+    strscan (3.1.0)
     sxp (1.2.4)
       matrix (~> 0.4)
       rdf (~> 3.2)
@@ -248,8 +249,8 @@ DEPENDENCIES
   pry-byebug (~> 3.6)
   pushmi_pullyu!
   rake (~> 13.0)
-  rspec (~> 3.0)
-  rubocop (~> 1.57)
+  rspec (~> 3.13)
+  rubocop (~> 1.64)
   rubocop-rspec (~> 2.24)
   timecop (~> 0.9)
   vcr (~> 5.0)

data/examples/pushmi_pullyu.yml

@@ -32,7 +32,10 @@ swift:
   auth_url: http://localhost:8080/auth/v1.0
   project_name: demo
   project_domain_name: default
-  container: ERA
+  container: era
+  # These 2 extra parameters are now required for keystone v3 authentication
+  auth_version: v3
+  user_domain: default
 
 rollbar:
   token: 'abc123xyz'

data/lib/pushmi_pullyu/cli.rb

@@ -199,6 +199,8 @@ class PushmiPullyu::CLI
   def run_preservation_cycle
     begin
       entity = queue.wait_next_item
+      PushmiPullyu::Logging.log_preservation_attempt(entity,
+                                                     queue.get_entity_ingestion_attempt(entity))
       return unless entity && entity[:type].present? && entity[:uuid].present?
     rescue StandardError => e
       log_exception(e)
@@ -212,7 +214,7 @@ class PushmiPullyu::CLI
         # Push tarred AIP to swift API
         deposited_file = swift.deposit_file(aip_filename, options[:swift][:container])
         # Log successful preservation event to the log files
-        PushmiPullyu::Logging.log_preservation_event(deposited_file, aip_directory)
+        PushmiPullyu::Logging.log_preservation_success(entity, deposited_file, aip_directory)
       end
     # An EntityInvalid expection means there is a problem with the entity information format so there is no point in
     # readding it to the queue as it will always fail
@@ -221,7 +223,9 @@ class PushmiPullyu::CLI
       log_exception(e)
       begin
         queue.add_entity_in_timeframe(entity)
+        PushmiPullyu::Logging.log_preservation_fail_and_retry(entity, queue.get_entity_ingestion_attempt(entity), e)
       rescue PushmiPullyu::PreservationQueue::MaxDepositAttemptsReached => e
+        PushmiPullyu::Logging.log_preservation_failure(entity, queue.get_entity_ingestion_attempt(entity), e)
         log_exception(e)
       end
 
@@ -244,7 +248,11 @@ class PushmiPullyu::CLI
 
   def setup_log
     if options[:daemonize]
-      PushmiPullyu::Logging.initialize_logger(PushmiPullyu.application_log_file)
+      PushmiPullyu::Logging.initialize_loggers(
+        log_target: PushmiPullyu.application_log_file,
+        events_target: "#{PushmiPullyu.options[:logdir]}/preservation_events.log",
+        json_target: "#{PushmiPullyu.options[:logdir]}/preservation_events.json"
+      )
     else
       logger.formatter = PushmiPullyu::Logging::SimpleFormatter.new
     end
@@ -266,10 +274,12 @@ class PushmiPullyu::CLI
   def swift
     @swift ||= PushmiPullyu::SwiftDepositer.new(username: options[:swift][:username],
                                                 password: options[:swift][:password],
-                                                tenant: options[:swift][:tenant],
                                                 project_name: options[:swift][:project_name],
+                                                tenant: options[:swift][:tenant],
                                                 project_domain_name: options[:swift][:project_domain_name],
-                                                auth_url: options[:swift][:auth_url])
+                                                user_domain: options[:swift][:user_domain],
+                                                auth_url: options[:swift][:auth_url],
+                                                auth_version: options[:swift][:auth_version])
   end
 
   # On first call of shutdown, this will gracefully close the main run loop

data/lib/pushmi_pullyu/logging.rb

@@ -21,14 +21,17 @@ module PushmiPullyu::Logging
 
     attr_writer :logger
 
-    def initialize_logger(log_target = $stdout)
+    def initialize_loggers(log_target: $stdout, events_target: $stdout, json_target: $stdout)
+      @preservation_logger = Logger.new(events_target)
+      @preservation_json_logger = Logger.new(json_target)
+
       @logger = Logger.new(log_target)
       @logger.level = Logger::INFO
       @logger
     end
 
     def logger
-      @logger ||= initialize_logger
+      @logger ||= initialize_loggers
     end
 
     def log_aip_activity(aip_directory, message)
@@ -43,10 +46,13 @@ module PushmiPullyu::Logging
       aip_logger.close
     end
 
-    def log_preservation_event(deposited_file, aip_directory)
-      preservation_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.log")
-      preservation_json_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.json")
+    def log_preservation_event(message, message_json)
+      logger.info(message)
+      @preservation_logger.info(message)
+      @preservation_json_logger.info("#{message_json},")
+    end
 
+    def log_preservation_success(entity, deposited_file, aip_directory)
       message = "#{deposited_file.name} was successfully deposited into Swift Storage!\n" \
                 "Here are the details of this preservation event:\n" \
                 "\tUUID: '#{deposited_file.name}'\n" \
@@ -68,15 +74,76 @@ module PushmiPullyu::Logging
         end
       end
 
-      # Log to both the application log, and the preservation log file
-      logger.info(message)
-      preservation_logger.info(message)
+      message_information = {
+        event_type: :success,
+        event_time: Time.now.to_s,
+        entity_type: entity[:type],
+        entity_uuid: entity[:uuid],
+        event_details: preservation_success_to_json(deposited_file, aip_directory)
+      }
+
+      log_preservation_event(message, message_information.to_json)
+    end
 
-      preservation_logger.close
+    def log_preservation_fail_and_retry(entity, try_attempt, exception)
+      # We add + 1 to try_attempt because humans like to start counting from 1
+      try_attempt += 1
+      message = "#{entity[:type]} failed to be deposited and will try again.\n" \
+                "Here are the details of this preservation event:\n" \
+                "\t#{entity[:type]} uuid: #{entity[:uuid]}" \
+                "\tReadding to preservation queue with try attempt: #{try_attempt}\n" \
+                "\tError of type: #{exception.class.name}\n" \
+                "\tError message: #{exception.message}\n"
+
+      message_information = {
+        event_type: :fail_and_retry,
+        event_time: Time.now.to_s,
+        entity_type: entity[:type],
+        entity_uuid: entity[:uuid],
+        try_attempt: try_attempt,
+        error_message: exception.message
+      }
+
+      log_preservation_event(message, message_information.to_json)
+    end
+
+    def log_preservation_failure(entity, try_attempt, exception)
+      # We add + 1 to try_attempt because humans like to start counting from 1
+      try_attempt += 1
+      message = "#{entity[:type]} failed to be deposited.\n" \
+                "Here are the details of this preservation event:\n" \
+                "\t#{entity[:type]} uuid: #{entity[:uuid]}" \
+                "\tTry attempt: #{try_attempt}\n"
+
+      message_information = {
+        event_type: :failure,
+        event_time: Time.now.to_s,
+        entity_type: entity[:type],
+        entity_uuid: entity[:uuid],
+        try_attempt: try_attempt,
+        error_message: exception.message
+      }
+
+      log_preservation_event(message, message_information.to_json)
+    end
 
-      message_json_str = preservation_event_to_json(deposited_file, aip_directory)
-      preservation_json_logger.info("#{message_json_str},")
-      preservation_json_logger.close
+    def log_preservation_attempt(entity, try_attempt)
+      # We add + 1 to try_attempt because humans like to start counting from 1
+      try_attempt += 1
+      message = "#{entity[:type]} will attempt to be deposited.\n" \
+                "Here are the details of this preservation event:\n" \
+                "\t#{entity[:type]} uuid: #{entity[:uuid]}" \
+                "\tTry attempt: #{try_attempt}\n"
+
+      message_information = {
+        event_type: :attempt,
+        event_time: Time.now.to_s,
+        entity_type: entity[:type],
+        entity_uuid: entity[:uuid],
+        try_attempt: try_attempt
+      }
+
+      log_preservation_event(message, message_information.to_json)
     end
 
     ###
@@ -107,14 +174,14 @@ module PushmiPullyu::Logging
     # note:
     #   to parse, the prefix "I, ... INFO --:" in each line needs to be
     #   stripped using a bash command such as "sed"
-    def preservation_event_to_json(deposited_file, aip_directory)
+    def preservation_success_to_json(deposited_file, aip_directory)
       message = {}
 
       message['do_uuid'] = deposited_file.name.to_s
       message['aip_deposited_at'] = deposited_file.last_modified.to_s
       message['aip_md5sum'] = deposited_file.etag.to_s
       message['aip_sha256'] = ''
-      message['aip_metadata'] = deposited_file.metadata.to_json.to_s
+      message['aip_metadata'] = deposited_file.metadata
 
       file_details = file_log_details(aip_directory)
 
@@ -131,14 +198,14 @@ module PushmiPullyu::Logging
       end
 
       message['aip_file_details'] = tmp_details
-      message.to_json
+      message
     end
 
     def reopen
       if @logger
         @logger.reopen
       else
-        @logger = initialize_logger
+        @logger = initialize_loggers
       end
     end
 

data/lib/pushmi_pullyu/preservation_queue.rb

@@ -70,6 +70,13 @@ class PushmiPullyu::PreservationQueue
     end
   end
 
+  def get_entity_ingestion_attempt(entity)
+    entity_attempts_key = "#{PushmiPullyu.options[:ingestion_prefix]}#{entity[:uuid]}"
+    @redis.with do |connection|
+      return connection.get(entity_attempts_key).to_i
+    end
+  end
+
   def add_entity_in_timeframe(entity)
     entity_attempts_key = "#{PushmiPullyu.options[:ingestion_prefix]}#{entity[:uuid]}"
 

data/lib/pushmi_pullyu/swift_depositer.rb

@@ -6,16 +6,24 @@ class PushmiPullyu::SwiftDepositer
   attr_reader :swift_connection
 
   def initialize(connection)
-    @swift_connection = OpenStack::Connection.create(
+    # Generic authentication parameters
+    swift_connection_parameters = {
       username: connection[:username],
       api_key: connection[:password],
-      auth_method: 'password',
       auth_url: connection[:auth_url],
       project_name: connection[:project_name],
-      project_domain_name: connection[:project_domain_name],
-      authtenant_name: connection[:tenant],
+      auth_method: 'password',
       service_type: 'object-store'
-    )
+    }
+
+    if connection[:auth_version] == 'v3'
+      swift_connection_parameters[:user_domain] = connection[:user_domain]
+    elsif connection[:auth_version] == 'v1'
+      swift_connection_parameters[:project_domain_name] = connection[:project_domain_name]
+      swift_connection_parameters[:authtenant_name] = connection[:tenant]
+    end
+
+    @swift_connection = OpenStack::Connection.create(swift_connection_parameters)
   end
 
   def deposit_file(file_name, swift_container)

data/lib/pushmi_pullyu/version.rb

@@ -1,3 +1,3 @@
 module PushmiPullyu
-  VERSION = '2.0.7'.freeze
+  VERSION = '2.1.1'.freeze
 end

data/lib/pushmi_pullyu.rb

@@ -39,7 +39,9 @@ module PushmiPullyu
       auth_url: 'http://localhost:8080/auth/v1.0',
       project_name: 'demo',
       project_domain_name: 'default',
-      container: 'ERA'
+      container: 'era',
+      auth_version: 'v3',
+      user_domain: 'default'
     },
     rollbar: {},
     # rubocop disable: Style/FetchEnvVar

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pushmi_pullyu
 version: !ruby/object:Gem::Version
-  version: 2.0.7
+  version: 2.1.1
 platform: ruby
 authors:
 - Shane Murnaghan
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-10-16 00:00:00.000000000 Z
+date: 2024-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -294,7 +294,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Ruby application to manage flow of content from Jupiter into Swift for preservation