pushmi_pullyu 2.0.3 → 2.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d64e5d619eb9dee63c358167cd6778034eb72fa2686fbe95fa187cfdfe7525b3
-  data.tar.gz: 333805bd92b1d5e308949d583e46056ffb1aeac9900d9c23ca59f2b88a7a4d01
+  metadata.gz: 25051393a56976db185a30c9cd498afd7997854d2a6931f3b6190132cec76a97
+  data.tar.gz: 731b8dbcf8a3d9124f44d57001bda6952bb2da07df874a1f97f52d7e7176c065
 SHA512:
-  metadata.gz: b8b162ca028ca37362d0bcd68737189f6af656c747bfc18a1593e35635bddf606d649e8d02299d092b1b569cab7e8a03950855fe2f7da4f5877a2714c99af8cc
-  data.tar.gz: 56c8653b1d868993e31607de7cd1d811752ff4de1ab63813dec2f745321f2b7d6a54107eb304f0f93191d6372bf22e55977512155edca08ee25ad96c23fc0099
+  metadata.gz: 82b083e9b991fbc0b7c95507415d312bc29b29a4f16fdcfe8689efc2dc4c69542a4c15010ce9b5841cebebe73920de7499512bf1eaab57bb379816bd1dba75ae
+  data.tar.gz: 2439db5f7058ddc7b042894538a592d076c467b34c7211c262d15e3a2d6d436f8a3d785f6b53c3bd68e93566832fed18bbde6b32efa4f0f4e8e08eceacd3ed57

data/.github/workflows/ruby.yml

@@ -18,9 +18,8 @@ jobs:
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:
-        ruby-version: 2.6.6
-    - name: Install dependencies
-      run: bundle install
+        ruby-version: '2.7'
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
     - name: Lint with RuboCop
       run: bundle exec rubocop --parallel
     - name: Run Danger

data/.gitignore

@@ -1,6 +1,5 @@
 /.bundle/
 /.yardoc
-/Gemfile.lock
 /_yardoc/
 /coverage/
 /doc/

data/.rubocop.yml

@@ -12,7 +12,7 @@ AllCops:
     - 'vendor/**/*'
     - 'Dangerfile'
   ExtraDetails: true
-  TargetRubyVersion: 2.5
+  TargetRubyVersion: 2.7
   NewCops: enable
 
 # readability is Actually Good

data/CHANGELOG.md

@@ -8,6 +8,24 @@ and releases in PushmiPullyu adheres to [Semantic Versioning](https://semver.org
 
 ## [Unreleased]
 
+## [2.0.5] - 2023-02-17
+
+- Add rescue block to catch exceptions while waiting for next item [#280](https://github.com/ualbertalib/pushmi_pullyu/issues/280)
+- Add logic to fetch new community and collection information from jupiter and create their AIPS. [#255](https://github.com/ualbertalib/pushmi_pullyu/issues/255)
+- Add delay to re-ingestion attempts to allow for problems to be fixed [#297](https://github.com/ualbertalib/pushmi_pullyu/issues/297)
+- Bump git from 1.9.1 to 1.13.0
+
+## [2.0.4] - 2022-11-22
+
+- Fix issue with temporary work files not being deleted after a failed swift deposit [#242](https://github.com/ualbertalib/pushmi_pullyu/issues/242)
+- Bump to Ruby 2.7
+- Fix issue with entity information consumed even after failed deposit [#232](https://github.com/ualbertalib/pushmi_pullyu/issues/232)
+- Bump rspec from 3.10.0 to 3.12.0
+- Bump rollbar from 3.3.0 to 3.3.2
+- Bump pry-byebug from 3.8.0 to 3.10.1
+- Bump webmock from 3.14.0 to 3.18.1
+- Bump rubocop-rspec from 2.6.0 to 2.11.1
+- Bump timecop from 0.9.4 to 0.9.5
 ## [2.0.3] - 2022-04-28
 
 - Changed Danger token in Github Actions

data/Gemfile.lock

@@ -0,0 +1,251 @@
+PATH
+  remote: .
+  specs:
+    pushmi_pullyu (2.0.5)
+      activesupport (>= 5, < 8)
+      bagit (~> 0.4)
+      connection_pool (~> 2.2)
+      daemons (~> 1.2, >= 1.2.4)
+      minitar (~> 0.7)
+      openstack (~> 3.3, >= 3.3.10)
+      rdf (>= 1.99, < 4.0)
+      rdf-n3 (>= 1.99, < 4.0)
+      redis (>= 3.3, < 5.0)
+      rest-client (>= 1.8, < 3.0)
+      rollbar (>= 2.18, < 4.0)
+      uuid (~> 2.3.9)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (7.0.4.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    addressable (2.8.1)
+      public_suffix (>= 2.0.2, < 6.0)
+    amazing_print (1.4.0)
+    ast (2.4.2)
+    bagit (0.4.5)
+      docopt (~> 0.5.0)
+      validatable (~> 1.6)
+    builder (3.2.4)
+    byebug (11.1.3)
+    claide (1.0.3)
+    claide-plugins (0.9.2)
+      cork
+      nap
+      open4 (~> 1.3)
+    coderay (1.1.3)
+    colored2 (3.1.2)
+    concurrent-ruby (1.2.0)
+    connection_pool (2.3.0)
+    cork (0.3.0)
+      colored2 (~> 3.1)
+    crack (0.4.5)
+      rexml
+    daemons (1.4.1)
+    danger (8.4.2)
+      claide (~> 1.0)
+      claide-plugins (>= 0.9.2)
+      colored2 (~> 3.1)
+      cork (~> 0.1)
+      faraday (>= 0.9.0, < 2.0)
+      faraday-http-cache (~> 2.0)
+      git (~> 1.7)
+      kramdown (~> 2.3)
+      kramdown-parser-gfm (~> 1.0)
+      no_proxy_fix
+      octokit (~> 4.7)
+      terminal-table (>= 1, < 4)
+    diff-lcs (1.5.0)
+    docopt (0.5.0)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    ebnf (2.3.1)
+      amazing_print (~> 1.4)
+      htmlentities (~> 4.3)
+      rdf (~> 3.2)
+      scanf (~> 1.0)
+      sxp (~> 1.2)
+      unicode-types (~> 1.7)
+    faraday (1.8.0)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0.1)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.1)
+      faraday-patron (~> 1.0)
+      faraday-rack (~> 1.0)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords (>= 0.0.4)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-excon (1.1.0)
+    faraday-http-cache (2.2.0)
+      faraday (>= 0.8)
+    faraday-httpclient (1.0.1)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.2.0)
+    faraday-patron (1.0.0)
+    faraday-rack (1.0.0)
+    git (1.13.0)
+      addressable (~> 2.8)
+      rchardet (~> 1.8)
+    hashdiff (1.0.1)
+    htmlentities (4.3.4)
+    http-accept (1.7.0)
+    http-cookie (1.0.5)
+      domain_name (~> 0.5)
+    i18n (1.12.0)
+      concurrent-ruby (~> 1.0)
+    json (2.6.3)
+    kramdown (2.3.1)
+      rexml
+    kramdown-parser-gfm (1.1.0)
+      kramdown (~> 2.0)
+    link_header (0.0.8)
+    logger (1.5.3)
+    macaddr (1.7.2)
+      systemu (~> 2.6.5)
+    matrix (0.4.2)
+    method_source (1.0.0)
+    mime-types (3.4.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2022.0105)
+    minitar (0.9)
+    minitest (5.17.0)
+    multipart-post (2.1.1)
+    nap (1.1.0)
+    net-http-persistent (4.0.1)
+      connection_pool (~> 2.2)
+    netrc (0.11.0)
+    no_proxy_fix (0.1.2)
+    octokit (4.21.0)
+      faraday (>= 0.9)
+      sawyer (~> 0.8.0, >= 0.5.3)
+    open4 (1.3.4)
+    openstack (3.3.21)
+      json
+    parallel (1.22.1)
+    parser (3.1.2.1)
+      ast (~> 2.4.1)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    pry-byebug (3.10.1)
+      byebug (~> 11.0)
+      pry (>= 0.13, < 0.15)
+    public_suffix (5.0.0)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    rchardet (1.8.0)
+    rdf (3.2.9)
+      link_header (~> 0.0, >= 0.0.8)
+    rdf-aggregate-repo (3.2.1)
+      rdf (~> 3.2)
+    rdf-n3 (3.2.1)
+      ebnf (~> 2.2)
+      rdf (~> 3.2)
+      sparql (~> 3.2)
+      sxp (~> 1.2)
+    rdf-xsd (3.2.1)
+      rdf (~> 3.2)
+      rexml (~> 3.2)
+    redis (4.8.1)
+    regexp_parser (2.6.0)
+    rest-client (2.1.0)
+      http-accept (>= 1.7.0, < 2.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
+    rexml (3.2.5)
+    rollbar (3.4.0)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.0)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
+    rubocop (1.28.1)
+      parallel (~> 1.10)
+      parser (>= 3.1.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.17.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.23.0)
+      parser (>= 3.1.1.0)
+    rubocop-rspec (2.11.1)
+      rubocop (~> 1.19)
+    ruby-progressbar (1.11.0)
+    ruby2_keywords (0.0.5)
+    sawyer (0.8.2)
+      addressable (>= 2.3.5)
+      faraday (> 0.8, < 2.0)
+    scanf (1.0.0)
+    sparql (3.2.5)
+      builder (~> 3.2)
+      ebnf (~> 2.2, >= 2.3.1)
+      logger (~> 1.5)
+      rdf (~> 3.2, >= 3.2.8)
+      rdf-aggregate-repo (~> 3.2)
+      rdf-xsd (~> 3.2)
+      sparql-client (~> 3.2, >= 3.2.1)
+      sxp (~> 1.2, >= 1.2.2)
+    sparql-client (3.2.1)
+      net-http-persistent (~> 4.0, >= 4.0.1)
+      rdf (~> 3.2, >= 3.2.6)
+    sxp (1.2.3)
+      matrix (~> 0.4)
+      rdf (~> 3.2)
+    systemu (2.6.5)
+    terminal-table (3.0.2)
+      unicode-display_width (>= 1.1.1, < 3)
+    timecop (0.9.6)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.8.2)
+    unicode-display_width (2.3.0)
+    unicode-types (1.8.0)
+    uuid (2.3.9)
+      macaddr (~> 1.0)
+    validatable (1.6.7)
+    vcr (5.1.0)
+    webmock (3.18.1)
+      addressable (>= 2.8.0)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  bundler (~> 2.0)
+  danger (~> 8.0)
+  pry (~> 0.10, >= 0.10.4)
+  pry-byebug (~> 3.6)
+  pushmi_pullyu!
+  rake (~> 13.0)
+  rspec (~> 3.0)
+  rubocop (~> 1.23)
+  rubocop-rspec (~> 2.6)
+  timecop (~> 0.8)
+  vcr (~> 5.0)
+  webmock (~> 3.3)
+
+BUNDLED WITH
+   2.3.19

data/README.md

@@ -26,7 +26,7 @@ Its primary job is to manage the flow of content from Jupiter into Swift for pre
 
 ## Requirements
 
-PushmiPullyu supports Ruby 2.3.1+
+PushmiPullyu supports Ruby 2.7
 
 ## Installation
 
@@ -66,7 +66,10 @@ Specific options:
   -W, --workdir PATH               Path for directory where AIP creation work takes place in
   -N, --process_name NAME          Name of the application process
   -m, --monitor                    Start monitor process for a deamon
-  -q, --queue NAME                 Name of the queue to read from
+  -q, --queue NAME                 Name of the queue to read from    
+  -i, --ingestion_prefix PREFIX    Prefix for keys used in counting the number of failed ingestion attempts
+  -x, --ingestion_attempts NUMBER  Max number of attempts to try ingesting an entity
+  -f, --first_failed_wait NUMBER   Time in seconds to wait after first failed entity deposit. This time will double every failed attempt
 
 Common options:
   -v, --version                    Show version
@@ -144,7 +147,7 @@ This will cut a tag version, builds the gem, and pushes the gem up to Rubygems
 
 Note: You may need permission to push a gem up to Rubygems!
 You will first need to create an account on rubygems.org.
-Once you have an account, bug Shane or Matt to [add you as an owner](http://guides.rubygems.org/command-reference/#gem-owner) to pushmi_pullyu Rubygem. Once you are an owner you should be able to push new versions of pushmi_pullyu up to Rubygems.
+Once you have an account, bug @pgwillia (Tricia Jenkins), @lagoan (Omar Rodriguez-Arenas), or @henryzhang87 (Henry Zhang) to [add you as an owner](http://guides.rubygems.org/command-reference/#gem-owner) to pushmi_pullyu Rubygem. Once you are an owner you should be able to push new versions of pushmi_pullyu up to Rubygems
 
 ## Deployment
 

data/examples/pushmi_pullyu.yml

@@ -16,6 +16,9 @@ piddir: tmp/pids
 workdir: tmp/work
 process_name: pushmi_pullyu
 queue_name: dev:pmpy_queue
+ingestion_prefix: "'prod:pmpy_ingest_attempt:'" 
+ingestion_attempts: 15
+first_failed_wait: 10
 minimum_age: 0
 
 redis:

data/lib/pushmi_pullyu/aip/downloader.rb

@@ -35,6 +35,11 @@ class PushmiPullyu::AIP::Downloader
     # Main object metadata
     download_and_log(object_aip_paths[:main_object_remote],
                      object_aip_paths[:main_object_local])
+
+    # Communities and collections do not have their own files.
+    return unless can_have_files?
+
+    FileUtils.mkdir_p(object_aip_paths[:file_sets_directory_local])
     download_and_log(object_aip_paths[:file_sets_remote],
                      object_aip_paths[:file_sets_local])
 
@@ -139,25 +144,26 @@ class PushmiPullyu::AIP::Downloader
     PushmiPullyu::Logging.log_aip_activity(@aip_directory, message)
   end
 
+  def can_have_files?
+    @entity[:type] == 'items' || @entity[:type] == 'theses'
+  end
+
   ### Directories
 
   def aip_dirs
     @aip_dirs ||= {
       objects: "#{@aip_directory}/data/objects",
       metadata: "#{@aip_directory}/data/objects/metadata",
-      files: "#{@aip_directory}/data/objects/files",
-      files_metadata: "#{@aip_directory}/data/objects/metadata/files_metadata",
-      logs: "#{@aip_directory}/data/logs",
-      file_logs: "#{@aip_directory}/data/logs/files_logs"
+      logs: "#{@aip_directory}/data/logs"
     }
   end
 
   def file_set_dirs(file_set_uuid)
     @file_set_dirs ||= {}
     @file_set_dirs[file_set_uuid] ||= {
-      metadata: "#{aip_dirs[:files_metadata]}/#{file_set_uuid}",
-      files: "#{aip_dirs[:files]}/#{file_set_uuid}",
-      logs: "#{aip_dirs[:file_logs]}/#{file_set_uuid}"
+      files: "#{@aip_directory}/data/objects/files/#{file_set_uuid}",
+      logs: "#{@aip_directory}/data/logs/files_logs/#{file_set_uuid}",
+      metadata: "#{@aip_directory}/data/objects/metadata/files_metadata/#{file_set_uuid}"
     }
   end
 
@@ -193,7 +199,9 @@ class PushmiPullyu::AIP::Downloader
       main_object_remote: object_uri,
       main_object_local: "#{aip_dirs[:metadata]}/object_metadata.n3",
       file_sets_remote: "#{object_uri}/filesets",
-      file_sets_local: "#{aip_dirs[:files_metadata]}/file_order.xml",
+      # This directory needs to be created before we can downloaded the file order information
+      file_sets_directory_local: "#{@aip_directory}/data/objects/metadata/files_metadata",
+      file_sets_local: "#{@aip_directory}/data/objects/metadata/files_metadata/file_order.xml",
       # This is downloaded for processing but not saved
       file_paths_remote: "#{object_uri}/file_paths"
     }.freeze

data/lib/pushmi_pullyu/aip.rb

@@ -6,19 +6,22 @@ module PushmiPullyu::AIP
   module_function
 
   def create(entity)
-    raise EntityInvalid if entity.nil? ||
-                           UUID.validate(entity[:uuid]) != true ||
+    raise EntityInvalid if entity.blank? ||
+                           UUID.validate(entity[:uuid]).blank? ||
                            entity[:type].blank?
 
     aip_directory = "#{PushmiPullyu.options[:workdir]}/#{entity[:uuid]}"
     aip_filename = "#{aip_directory}.tar"
+    begin
+      PushmiPullyu::AIP::Downloader.new(entity, aip_directory).run
+      PushmiPullyu::AIP::Creator.new(entity[:uuid], aip_directory, aip_filename).run
 
-    PushmiPullyu::AIP::Downloader.new(entity, aip_directory).run
-    PushmiPullyu::AIP::Creator.new(entity[:uuid], aip_directory, aip_filename).run
-
-    yield aip_filename, aip_directory
-
-    FileUtils.rm_rf(aip_filename) if File.exist?(aip_filename)
-    FileUtils.rm_rf(aip_directory) if File.exist?(aip_directory)
+      yield aip_filename, aip_directory
+    # Here we will ensure the files are removed even if an exception comes up.
+    # You will notice there is no rescue block.  We will catch exceptions in `PushmiPullyu::CLI`
+    ensure
+      FileUtils.rm_rf(aip_filename)
+      FileUtils.rm_rf(aip_directory)
+    end
   end
 end

data/lib/pushmi_pullyu/cli.rb

@@ -148,6 +148,21 @@ class PushmiPullyu::CLI
         opts[:queue_name] = queue
       end
 
+      o.on('-i', '--ingestion_prefix PREFIX',
+           'Prefix for keys used in counting the number of failed ingestion attempts') do |prefix|
+        opts[:ingestion_prefix] = prefix
+      end
+
+      o.on('-x', '--ingestion_attempts NUMBER', Integer,
+           'Max number of attempts to try ingesting an entity') do |ingestion_attempts|
+        opts[:ingestion_attempts] = ingestion_attempts
+      end
+
+      o.on('-f', '--first_failed_wait NUMBER', Integer,
+           'Time in seconds to wait after first failed deposit. Time will double every failed attempt') do |failed_wait|
+        opts[:first_failed_wait] = failed_wait
+      end
+
       o.separator ''
       o.separator 'Common options:'
 
@@ -182,12 +197,12 @@ class PushmiPullyu::CLI
   end
 
   def run_preservation_cycle
-    entity_json = JSON.parse(queue.wait_next_item)
-    entity = {
-      type: entity_json['type'],
-      uuid: entity_json['uuid']
-    }
-    return unless entity[:type].present? && entity[:uuid].present?
+    begin
+      entity = queue.wait_next_item
+      return unless entity && entity[:type].present? && entity[:uuid].present?
+    rescue StandardError => e
+      log_exception(e)
+    end
 
     # add additional information about the error context to errors that occur while processing this item.
     Rollbar.scoped(entity_uuid: entity[:uuid]) do
@@ -199,13 +214,24 @@ class PushmiPullyu::CLI
         # Log successful preservation event to the log files
         PushmiPullyu::Logging.log_preservation_event(deposited_file, aip_directory)
       end
-      # rubocop:disable Lint/RescueException
+    # An EntityInvalid expection means there is a problem with the entity information format so there is no point in
+    # readding it to the queue as it will always fail
+    rescue PushmiPullyu::AIP::EntityInvalid => e
+    rescue StandardError => e
+      begin
+        queue.add_entity_in_timeframe(entity)
+      rescue MaxDepositAttemptsReached => e
+        log_exception(e)
+      end
+
+    # rubocop:disable Lint/RescueException
+    # Something other than a StandardError exception means something happened which we were not expecting!
+    # Make sure we log the problem
     rescue Exception => e
-      Rollbar.error(e)
-      logger.error(e)
-      # TODO: we could re-raise here and let the daemon die on any preservation error, or just log the issue and
-      # move on to the next item.
-      # rubocop:enable Lint/RescueException
+      raise e
+    # rubocop:enable Lint/RescueException
+    ensure
+      log_exception(e)
     end
   end
 
@@ -280,4 +306,9 @@ class PushmiPullyu::CLI
     end
   end
 
+  def log_exception(exception)
+    Rollbar.error(exception)
+    logger.error(exception)
+  end
+
 end

data/lib/pushmi_pullyu/logging.rb

@@ -47,12 +47,12 @@ module PushmiPullyu::Logging
       preservation_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.log")
       preservation_json_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.json")
 
-      message = "#{deposited_file.name} was successfully deposited into Swift Storage!\n"\
-                "Here are the details of this preservation event:\n"\
-                "\tUUID: '#{deposited_file.name}'\n"\
-                "\tTimestamp of Completion: '#{deposited_file.last_modified}'\n"\
-                "\tAIP Checksum: '#{deposited_file.etag}'\n"\
-                "\tMetadata: #{deposited_file.metadata}\n"\
+      message = "#{deposited_file.name} was successfully deposited into Swift Storage!\n" \
+                "Here are the details of this preservation event:\n" \
+                "\tUUID: '#{deposited_file.name}'\n" \
+                "\tTimestamp of Completion: '#{deposited_file.last_modified}'\n" \
+                "\tAIP Checksum: '#{deposited_file.etag}'\n" \
+                "\tMetadata: #{deposited_file.metadata}\n" \
 
       file_details = file_log_details(aip_directory)
 
@@ -155,7 +155,7 @@ module PushmiPullyu::Logging
               fileset_name: File.dirname(file).split('/')[-1],
               file_name: File.basename(file),
               file_size: File.size(file),
-              file_extension: File.extname(file).strip.downcase[1..-1]
+              file_extension: File.extname(file).strip.downcase[1..]
             }
           end
         end

data/lib/pushmi_pullyu/preservation_queue.rb

@@ -20,6 +20,7 @@ require 'connection_pool'
 class PushmiPullyu::PreservationQueue
 
   class ConnectionError < StandardError; end
+  class MaxDepositAttemptsReached < StandardError; end
 
   def initialize(redis_url: 'redis://localhost:6379',
                  pool_opts: { size: 1, timeout: 5 },
@@ -50,7 +51,8 @@ class PushmiPullyu::PreservationQueue
           rd.multi do |tx|
             tx.zrem(@queue_name, element) # remove the top element transactionally
           end
-          return element
+
+          return JSON.parse(element, { symbolize_names: true })
         else
           rd.unwatch # cancel the transaction since there was nothing in the queue
           return nil
@@ -68,6 +70,27 @@ class PushmiPullyu::PreservationQueue
     end
   end
 
+  def add_entity_in_timeframe(entity)
+    entity_attempts_key = "#{PushmiPullyu.options[:ingestion_prefix]}#{entity[:uuid]}"
+
+    @redis.with do |connection|
+      # separate information for priority information and queue
+      deposit_attempt = connection.incr entity_attempts_key
+
+      if deposit_attempt <= PushmiPullyu.options[:ingestion_attempts]
+        connection.zadd @queue_name, Time.now.to_f + self.class.extra_wait_time(deposit_attempt),
+                        entity.slice(:uuid, :type).to_json
+      else
+        connection.del entity_attempts_key
+        raise MaxDepositAttemptsReached
+      end
+    end
+  end
+
+  def self.extra_wait_time(deposit_attempt)
+    (2**deposit_attempt) * PushmiPullyu.options[:first_failed_wait]
+  end
+
   protected
 
   def connected?

data/lib/pushmi_pullyu/version.rb

@@ -1,3 +1,3 @@
 module PushmiPullyu
-  VERSION = '2.0.3'.freeze
+  VERSION = '2.0.5'.freeze
 end

data/lib/pushmi_pullyu.rb

@@ -26,6 +26,9 @@ module PushmiPullyu
     workdir: 'tmp/work',
     process_name: 'pushmi_pullyu',
     queue_name: 'dev:pmpy_queue',
+    ingestion_prefix: 'prod:pmpy_ingest_attempt:',
+    ingestion_attempts: 15,
+    first_failed_wait: 10,
     redis: {
       url: 'redis://localhost:6379'
     },

data/pushmi_pullyu.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.required_ruby_version = '>= 2.5'
+  spec.required_ruby_version = '>= 2.7'
 
   spec.add_runtime_dependency 'activesupport', '>= 5', '< 8'
   spec.add_runtime_dependency 'bagit', '~> 0.4'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pushmi_pullyu
 version: !ruby/object:Gem::Version
-  version: 2.0.3
+  version: 2.0.5
 platform: ruby
 authors:
 - Shane Murnaghan
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-04-28 00:00:00.000000000 Z
+date: 2023-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -409,6 +409,7 @@ files:
 - CHANGELOG.md
 - Dangerfile
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -446,7 +447,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.5'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="