pushmi_pullyu 2.0.4 → 2.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aabca31fdc4036e26f6fbd682b2e9aece8179973fc1509e7eada08b7bcdaf796
-  data.tar.gz: '0678d634ac98e9a4aeefa0bfb5d6df5a4869c0db3ba1b784be38a65caf916d75'
+  metadata.gz: 128bfb644445954d7b9c5eb3700dd1f3e4d3c92ea1e8448ab98054d750b2d53d
+  data.tar.gz: 2fc44c64692367c9c4b254ec32b8241c7ecdc8fb1a33384561cbfcd34096a085
 SHA512:
-  metadata.gz: fb769198d6bf7e609fbbe7eec02498756f185e338439983a1ad14391392e9d3f66dd8368438791e237066137edd9bbf0afe355d6ef84cb61a11b5a00acc3e48a
-  data.tar.gz: b921457537461cd5512fec7ac072f301c701a98a40ee210236c39cb098b04b9c1206ca2b2a0c759ab6e9fce4f2a4768329ded84a104c1fd8ec51e74e4b7b395b
+  metadata.gz: daaaf67bda17bdea14310b41a87a1c51190f41337d8f0d467887dea5ed9cc700b2b05822bddd6038a4c7746927439d2576fd8a7c3fb2a878d7f59a8ac84e45be
+  data.tar.gz: 33197ef8eb83869cea35ea5af258d83c16c56e77cbb0716fc61a6fa344b3d5cdf75e25a0fd72d6e2ab71148ddae9648351d29c5f2b53c9569af9f04869538ba0

data/CHANGELOG.md

@@ -8,6 +8,17 @@ and releases in PushmiPullyu adheres to [Semantic Versioning](https://semver.org
 
 ## [Unreleased]
 
+## [2.0.6] - 2023-03-17
+
+- Fix URI concatenation for jupiter's base url. [#309](https://github.com/ualbertalib/pushmi_pullyu/issues/309)
+
+## [2.0.5] - 2023-02-17
+
+- Add rescue block to catch exceptions while waiting for next item [#280](https://github.com/ualbertalib/pushmi_pullyu/issues/280)
+- Add logic to fetch new community and collection information from jupiter and create their AIPS. [#255](https://github.com/ualbertalib/pushmi_pullyu/issues/255)
+- Add delay to re-ingestion attempts to allow for problems to be fixed [#297](https://github.com/ualbertalib/pushmi_pullyu/issues/297)
+- Bump git from 1.9.1 to 1.13.0
+
 ## [2.0.4] - 2022-11-22
 
 - Fix issue with temporary work files not being deleted after a failed swift deposit [#242](https://github.com/ualbertalib/pushmi_pullyu/issues/242)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pushmi_pullyu (2.0.4)
+    pushmi_pullyu (2.0.6)
       activesupport (>= 5, < 8)
       bagit (~> 0.4)
       connection_pool (~> 2.2)
@@ -18,7 +18,7 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.0.2.4)
+    activesupport (7.0.4.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -39,8 +39,8 @@ GEM
       open4 (~> 1.3)
     coderay (1.1.3)
     colored2 (3.1.2)
-    concurrent-ruby (1.1.10)
-    connection_pool (2.2.5)
+    concurrent-ruby (1.2.0)
+    connection_pool (2.3.0)
     cork (0.3.0)
       colored2 (~> 3.1)
     crack (0.4.5)
@@ -91,22 +91,23 @@ GEM
     faraday-net_http_persistent (1.2.0)
     faraday-patron (1.0.0)
     faraday-rack (1.0.0)
-    git (1.9.1)
+    git (1.13.0)
+      addressable (~> 2.8)
       rchardet (~> 1.8)
     hashdiff (1.0.1)
     htmlentities (4.3.4)
     http-accept (1.7.0)
-    http-cookie (1.0.4)
+    http-cookie (1.0.5)
       domain_name (~> 0.5)
-    i18n (1.10.0)
+    i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    json (2.6.1)
+    json (2.6.3)
     kramdown (2.3.1)
       rexml
     kramdown-parser-gfm (1.1.0)
       kramdown (~> 2.0)
     link_header (0.0.8)
-    logger (1.5.1)
+    logger (1.5.3)
     macaddr (1.7.2)
       systemu (~> 2.6.5)
     matrix (0.4.2)
@@ -115,7 +116,7 @@ GEM
       mime-types-data (~> 3.2015)
     mime-types-data (3.2022.0105)
     minitar (0.9)
-    minitest (5.15.0)
+    minitest (5.17.0)
     multipart-post (2.1.1)
     nap (1.1.0)
     net-http-persistent (4.0.1)
@@ -141,9 +142,9 @@ GEM
     rainbow (3.1.1)
     rake (13.0.6)
     rchardet (1.8.0)
-    rdf (3.2.7)
+    rdf (3.2.9)
       link_header (~> 0.0, >= 0.0.8)
-    rdf-aggregate-repo (3.2.0)
+    rdf-aggregate-repo (3.2.1)
       rdf (~> 3.2)
     rdf-n3 (3.2.1)
       ebnf (~> 2.2)
@@ -153,7 +154,7 @@ GEM
     rdf-xsd (3.2.1)
       rdf (~> 3.2)
       rexml (~> 3.2)
-    redis (4.6.0)
+    redis (4.8.1)
     regexp_parser (2.6.0)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
@@ -161,7 +162,7 @@ GEM
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
     rexml (3.2.5)
-    rollbar (3.3.0)
+    rollbar (3.4.0)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
       rspec-expectations (~> 3.12.0)
@@ -194,32 +195,32 @@ GEM
       addressable (>= 2.3.5)
       faraday (> 0.8, < 2.0)
     scanf (1.0.0)
-    sparql (3.2.1)
+    sparql (3.2.5)
       builder (~> 3.2)
-      ebnf (~> 2.2)
-      logger (~> 1.4)
-      rdf (~> 3.2, >= 3.2.3)
+      ebnf (~> 2.2, >= 2.3.1)
+      logger (~> 1.5)
+      rdf (~> 3.2, >= 3.2.8)
       rdf-aggregate-repo (~> 3.2)
       rdf-xsd (~> 3.2)
-      sparql-client (~> 3.2)
-      sxp (~> 1.2, >= 1.2.1)
+      sparql-client (~> 3.2, >= 3.2.1)
+      sxp (~> 1.2, >= 1.2.2)
     sparql-client (3.2.1)
       net-http-persistent (~> 4.0, >= 4.0.1)
       rdf (~> 3.2, >= 3.2.6)
-    sxp (1.2.2)
-      matrix
+    sxp (1.2.3)
+      matrix (~> 0.4)
       rdf (~> 3.2)
     systemu (2.6.5)
     terminal-table (3.0.2)
       unicode-display_width (>= 1.1.1, < 3)
-    timecop (0.9.5)
-    tzinfo (2.0.4)
+    timecop (0.9.6)
+    tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.8.1)
+    unf_ext (0.0.8.2)
     unicode-display_width (2.3.0)
-    unicode-types (1.7.0)
+    unicode-types (1.8.0)
     uuid (2.3.9)
       macaddr (~> 1.0)
     validatable (1.6.7)

data/README.md

@@ -66,7 +66,10 @@ Specific options:
   -W, --workdir PATH               Path for directory where AIP creation work takes place in
   -N, --process_name NAME          Name of the application process
   -m, --monitor                    Start monitor process for a deamon
-  -q, --queue NAME                 Name of the queue to read from
+  -q, --queue NAME                 Name of the queue to read from    
+  -i, --ingestion_prefix PREFIX    Prefix for keys used in counting the number of failed ingestion attempts
+  -x, --ingestion_attempts NUMBER  Max number of attempts to try ingesting an entity
+  -f, --first_failed_wait NUMBER   Time in seconds to wait after first failed entity deposit. This time will double every failed attempt
 
 Common options:
   -v, --version                    Show version
@@ -101,6 +104,13 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
+You'll need to set these two env vars otherwise you'll likely run into an error (Errno::ECONNRESET: Connection reset by peer):
+
+```bash
+export JUPITER_API_KEY=3eeb395e-63b7-11ea-bc55-0242ac130003
+export JUPITER_USER=ditech@ualberta.ca
+```
+
 ## Testing
 
 Pre-requisites:
@@ -144,7 +154,7 @@ This will cut a tag version, builds the gem, and pushes the gem up to Rubygems
 
 Note: You may need permission to push a gem up to Rubygems!
 You will first need to create an account on rubygems.org.
-Once you have an account, bug Shane or Matt to [add you as an owner](http://guides.rubygems.org/command-reference/#gem-owner) to pushmi_pullyu Rubygem. Once you are an owner you should be able to push new versions of pushmi_pullyu up to Rubygems.
+Once you have an account, bug @pgwillia (Tricia Jenkins), @lagoan (Omar Rodriguez-Arenas), or @henryzhang87 (Henry Zhang) to [add you as an owner](http://guides.rubygems.org/command-reference/#gem-owner) to pushmi_pullyu Rubygem. Once you are an owner you should be able to push new versions of pushmi_pullyu up to Rubygems
 
 ## Deployment
 

data/examples/pushmi_pullyu.yml

@@ -16,6 +16,9 @@ piddir: tmp/pids
 workdir: tmp/work
 process_name: pushmi_pullyu
 queue_name: dev:pmpy_queue
+ingestion_prefix: "'prod:pmpy_ingest_attempt:'" 
+ingestion_attempts: 15
+first_failed_wait: 10
 minimum_age: 0
 
 redis:

data/lib/pushmi_pullyu/aip/downloader.rb

@@ -35,6 +35,11 @@ class PushmiPullyu::AIP::Downloader
     # Main object metadata
     download_and_log(object_aip_paths[:main_object_remote],
                      object_aip_paths[:main_object_local])
+
+    # Communities and collections do not have their own files.
+    return unless can_have_files?
+
+    FileUtils.mkdir_p(object_aip_paths[:file_sets_directory_local])
     download_and_log(object_aip_paths[:file_sets_remote],
                      object_aip_paths[:file_sets_local])
 
@@ -81,7 +86,7 @@ class PushmiPullyu::AIP::Downloader
     @uri = URI.parse(PushmiPullyu.options[:jupiter][:jupiter_url])
     @http = Net::HTTP.new(@uri.host, @uri.port)
     @http.use_ssl = true if @uri.instance_of? URI::HTTPS
-    request = Net::HTTP::Post.new("#{@uri.request_uri}auth/system")
+    request = Net::HTTP::Post.new(URI.join(@uri, '/auth/system'))
     request.set_form_data(
       email: PushmiPullyu.options[:jupiter][:user],
       api_key: PushmiPullyu.options[:jupiter][:api_key]
@@ -97,7 +102,7 @@ class PushmiPullyu::AIP::Downloader
     log_downloading(remote, local)
 
     @uri = URI.parse(PushmiPullyu.options[:jupiter][:jupiter_url])
-    request = Net::HTTP::Get.new(@uri.request_uri + remote)
+    request = Net::HTTP::Get.new(URI.join(@uri, remote))
     # add previously stored cookies
     request['Cookie'] = @cookies
 
@@ -113,7 +118,7 @@ class PushmiPullyu::AIP::Downloader
   end
 
   def get_file_paths(url)
-    request = Net::HTTP::Get.new(@uri.request_uri + url)
+    request = Net::HTTP::Get.new(URI.join(@uri, url))
     # add previously stored cookies
     request['Cookie'] = @cookies
 
@@ -139,25 +144,26 @@ class PushmiPullyu::AIP::Downloader
     PushmiPullyu::Logging.log_aip_activity(@aip_directory, message)
   end
 
+  def can_have_files?
+    @entity[:type] == 'items' || @entity[:type] == 'theses'
+  end
+
   ### Directories
 
   def aip_dirs
     @aip_dirs ||= {
       objects: "#{@aip_directory}/data/objects",
       metadata: "#{@aip_directory}/data/objects/metadata",
-      files: "#{@aip_directory}/data/objects/files",
-      files_metadata: "#{@aip_directory}/data/objects/metadata/files_metadata",
-      logs: "#{@aip_directory}/data/logs",
-      file_logs: "#{@aip_directory}/data/logs/files_logs"
+      logs: "#{@aip_directory}/data/logs"
     }
   end
 
   def file_set_dirs(file_set_uuid)
     @file_set_dirs ||= {}
     @file_set_dirs[file_set_uuid] ||= {
-      metadata: "#{aip_dirs[:files_metadata]}/#{file_set_uuid}",
-      files: "#{aip_dirs[:files]}/#{file_set_uuid}",
-      logs: "#{aip_dirs[:file_logs]}/#{file_set_uuid}"
+      files: "#{@aip_directory}/data/objects/files/#{file_set_uuid}",
+      logs: "#{@aip_directory}/data/logs/files_logs/#{file_set_uuid}",
+      metadata: "#{@aip_directory}/data/objects/metadata/files_metadata/#{file_set_uuid}"
     }
   end
 
@@ -193,7 +199,9 @@ class PushmiPullyu::AIP::Downloader
       main_object_remote: object_uri,
       main_object_local: "#{aip_dirs[:metadata]}/object_metadata.n3",
       file_sets_remote: "#{object_uri}/filesets",
-      file_sets_local: "#{aip_dirs[:files_metadata]}/file_order.xml",
+      # This directory needs to be created before we can downloaded the file order information
+      file_sets_directory_local: "#{@aip_directory}/data/objects/metadata/files_metadata",
+      file_sets_local: "#{@aip_directory}/data/objects/metadata/files_metadata/file_order.xml",
       # This is downloaded for processing but not saved
       file_paths_remote: "#{object_uri}/file_paths"
     }.freeze

data/lib/pushmi_pullyu/cli.rb

@@ -148,6 +148,21 @@ class PushmiPullyu::CLI
         opts[:queue_name] = queue
       end
 
+      o.on('-i', '--ingestion_prefix PREFIX',
+           'Prefix for keys used in counting the number of failed ingestion attempts') do |prefix|
+        opts[:ingestion_prefix] = prefix
+      end
+
+      o.on('-x', '--ingestion_attempts NUMBER', Integer,
+           'Max number of attempts to try ingesting an entity') do |ingestion_attempts|
+        opts[:ingestion_attempts] = ingestion_attempts
+      end
+
+      o.on('-f', '--first_failed_wait NUMBER', Integer,
+           'Time in seconds to wait after first failed deposit. Time will double every failed attempt') do |failed_wait|
+        opts[:first_failed_wait] = failed_wait
+      end
+
       o.separator ''
       o.separator 'Common options:'
 
@@ -182,11 +197,12 @@ class PushmiPullyu::CLI
   end
 
   def run_preservation_cycle
-    entity_json = queue.wait_next_item
-    # jupiter is submitting the entries to reddis in a hash format using fat arrows. We need to change them to colons in
-    # order to parse them correctly from json
-    entity = JSON.parse(entity_json.gsub('=>', ':'), { symbolize_names: true })
-    return unless entity[:type].present? && entity[:uuid].present?
+    begin
+      entity = queue.wait_next_item
+      return unless entity && entity[:type].present? && entity[:uuid].present?
+    rescue StandardError => e
+      log_exception(e)
+    end
 
     # add additional information about the error context to errors that occur while processing this item.
     Rollbar.scoped(entity_uuid: entity[:uuid]) do
@@ -202,7 +218,11 @@ class PushmiPullyu::CLI
     # readding it to the queue as it will always fail
     rescue PushmiPullyu::AIP::EntityInvalid => e
     rescue StandardError => e
-      queue.add_entity_json(entity_json)
+      begin
+        queue.add_entity_in_timeframe(entity)
+      rescue PushmiPullyu::PreservationQueue::MaxDepositAttemptsReached => e
+        log_exception(e)
+      end
 
     # rubocop:disable Lint/RescueException
     # Something other than a StandardError exception means something happened which we were not expecting!
@@ -211,8 +231,7 @@ class PushmiPullyu::CLI
       raise e
     # rubocop:enable Lint/RescueException
     ensure
-      Rollbar.error(e)
-      logger.error(e)
+      log_exception(e)
     end
   end
 
@@ -287,4 +306,9 @@ class PushmiPullyu::CLI
     end
   end
 
+  def log_exception(exception)
+    Rollbar.error(exception)
+    logger.error(exception)
+  end
+
 end

data/lib/pushmi_pullyu/preservation_queue.rb

@@ -20,6 +20,7 @@ require 'connection_pool'
 class PushmiPullyu::PreservationQueue
 
   class ConnectionError < StandardError; end
+  class MaxDepositAttemptsReached < StandardError; end
 
   def initialize(redis_url: 'redis://localhost:6379',
                  pool_opts: { size: 1, timeout: 5 },
@@ -50,7 +51,8 @@ class PushmiPullyu::PreservationQueue
           rd.multi do |tx|
             tx.zrem(@queue_name, element) # remove the top element transactionally
           end
-          return element
+
+          return JSON.parse(element, { symbolize_names: true })
         else
           rd.unwatch # cancel the transaction since there was nothing in the queue
           return nil
@@ -68,12 +70,27 @@ class PushmiPullyu::PreservationQueue
     end
   end
 
-  def add_entity_json(entity_json)
+  def add_entity_in_timeframe(entity)
+    entity_attempts_key = "#{PushmiPullyu.options[:ingestion_prefix]}#{entity[:uuid]}"
+
     @redis.with do |connection|
-      connection.zadd @queue_name, Time.now.to_f, entity_json
+      # separate information for priority information and queue
+      deposit_attempt = connection.incr entity_attempts_key
+
+      if deposit_attempt <= PushmiPullyu.options[:ingestion_attempts]
+        connection.zadd @queue_name, Time.now.to_f + self.class.extra_wait_time(deposit_attempt),
+                        entity.slice(:uuid, :type).to_json
+      else
+        connection.del entity_attempts_key
+        raise MaxDepositAttemptsReached
+      end
     end
   end
 
+  def self.extra_wait_time(deposit_attempt)
+    (2**deposit_attempt) * PushmiPullyu.options[:first_failed_wait]
+  end
+
   protected
 
   def connected?

data/lib/pushmi_pullyu/version.rb

@@ -1,3 +1,3 @@
 module PushmiPullyu
-  VERSION = '2.0.4'.freeze
+  VERSION = '2.0.6'.freeze
 end

data/lib/pushmi_pullyu.rb

@@ -26,6 +26,9 @@ module PushmiPullyu
     workdir: 'tmp/work',
     process_name: 'pushmi_pullyu',
     queue_name: 'dev:pmpy_queue',
+    ingestion_prefix: 'prod:pmpy_ingest_attempt:',
+    ingestion_attempts: 15,
+    first_failed_wait: 10,
     redis: {
       url: 'redis://localhost:6379'
     },

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pushmi_pullyu
 version: !ruby/object:Gem::Version
-  version: 2.0.4
+  version: 2.0.6
 platform: ruby
 authors:
 - Shane Murnaghan
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-11-22 00:00:00.000000000 Z
+date: 2023-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport