pushmi_pullyu 2.0.3 → 2.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d64e5d619eb9dee63c358167cd6778034eb72fa2686fbe95fa187cfdfe7525b3
-  data.tar.gz: 333805bd92b1d5e308949d583e46056ffb1aeac9900d9c23ca59f2b88a7a4d01
+  metadata.gz: aabca31fdc4036e26f6fbd682b2e9aece8179973fc1509e7eada08b7bcdaf796
+  data.tar.gz: '0678d634ac98e9a4aeefa0bfb5d6df5a4869c0db3ba1b784be38a65caf916d75'
 SHA512:
-  metadata.gz: b8b162ca028ca37362d0bcd68737189f6af656c747bfc18a1593e35635bddf606d649e8d02299d092b1b569cab7e8a03950855fe2f7da4f5877a2714c99af8cc
-  data.tar.gz: 56c8653b1d868993e31607de7cd1d811752ff4de1ab63813dec2f745321f2b7d6a54107eb304f0f93191d6372bf22e55977512155edca08ee25ad96c23fc0099
+  metadata.gz: fb769198d6bf7e609fbbe7eec02498756f185e338439983a1ad14391392e9d3f66dd8368438791e237066137edd9bbf0afe355d6ef84cb61a11b5a00acc3e48a
+  data.tar.gz: b921457537461cd5512fec7ac072f301c701a98a40ee210236c39cb098b04b9c1206ca2b2a0c759ab6e9fce4f2a4768329ded84a104c1fd8ec51e74e4b7b395b

data/.github/workflows/ruby.yml

@@ -18,9 +18,8 @@ jobs:
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:
-        ruby-version: 2.6.6
-    - name: Install dependencies
-      run: bundle install
+        ruby-version: '2.7'
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
     - name: Lint with RuboCop
       run: bundle exec rubocop --parallel
     - name: Run Danger

data/.gitignore

@@ -1,6 +1,5 @@
 /.bundle/
 /.yardoc
-/Gemfile.lock
 /_yardoc/
 /coverage/
 /doc/

data/.rubocop.yml

@@ -12,7 +12,7 @@ AllCops:
     - 'vendor/**/*'
     - 'Dangerfile'
   ExtraDetails: true
-  TargetRubyVersion: 2.5
+  TargetRubyVersion: 2.7
   NewCops: enable
 
 # readability is Actually Good

data/CHANGELOG.md

@@ -8,6 +8,17 @@ and releases in PushmiPullyu adheres to [Semantic Versioning](https://semver.org
 
 ## [Unreleased]
 
+## [2.0.4] - 2022-11-22
+
+- Fix issue with temporary work files not being deleted after a failed swift deposit [#242](https://github.com/ualbertalib/pushmi_pullyu/issues/242)
+- Bump to Ruby 2.7
+- Fix issue with entity information consumed even after failed deposit [#232](https://github.com/ualbertalib/pushmi_pullyu/issues/232)
+- Bump rspec from 3.10.0 to 3.12.0
+- Bump rollbar from 3.3.0 to 3.3.2
+- Bump pry-byebug from 3.8.0 to 3.10.1
+- Bump webmock from 3.14.0 to 3.18.1
+- Bump rubocop-rspec from 2.6.0 to 2.11.1
+- Bump timecop from 0.9.4 to 0.9.5
 ## [2.0.3] - 2022-04-28
 
 - Changed Danger token in Github Actions

data/Gemfile.lock

@@ -0,0 +1,250 @@
+PATH
+  remote: .
+  specs:
+    pushmi_pullyu (2.0.4)
+      activesupport (>= 5, < 8)
+      bagit (~> 0.4)
+      connection_pool (~> 2.2)
+      daemons (~> 1.2, >= 1.2.4)
+      minitar (~> 0.7)
+      openstack (~> 3.3, >= 3.3.10)
+      rdf (>= 1.99, < 4.0)
+      rdf-n3 (>= 1.99, < 4.0)
+      redis (>= 3.3, < 5.0)
+      rest-client (>= 1.8, < 3.0)
+      rollbar (>= 2.18, < 4.0)
+      uuid (~> 2.3.9)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (7.0.2.4)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    addressable (2.8.1)
+      public_suffix (>= 2.0.2, < 6.0)
+    amazing_print (1.4.0)
+    ast (2.4.2)
+    bagit (0.4.5)
+      docopt (~> 0.5.0)
+      validatable (~> 1.6)
+    builder (3.2.4)
+    byebug (11.1.3)
+    claide (1.0.3)
+    claide-plugins (0.9.2)
+      cork
+      nap
+      open4 (~> 1.3)
+    coderay (1.1.3)
+    colored2 (3.1.2)
+    concurrent-ruby (1.1.10)
+    connection_pool (2.2.5)
+    cork (0.3.0)
+      colored2 (~> 3.1)
+    crack (0.4.5)
+      rexml
+    daemons (1.4.1)
+    danger (8.4.2)
+      claide (~> 1.0)
+      claide-plugins (>= 0.9.2)
+      colored2 (~> 3.1)
+      cork (~> 0.1)
+      faraday (>= 0.9.0, < 2.0)
+      faraday-http-cache (~> 2.0)
+      git (~> 1.7)
+      kramdown (~> 2.3)
+      kramdown-parser-gfm (~> 1.0)
+      no_proxy_fix
+      octokit (~> 4.7)
+      terminal-table (>= 1, < 4)
+    diff-lcs (1.5.0)
+    docopt (0.5.0)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    ebnf (2.3.1)
+      amazing_print (~> 1.4)
+      htmlentities (~> 4.3)
+      rdf (~> 3.2)
+      scanf (~> 1.0)
+      sxp (~> 1.2)
+      unicode-types (~> 1.7)
+    faraday (1.8.0)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0.1)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.1)
+      faraday-patron (~> 1.0)
+      faraday-rack (~> 1.0)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords (>= 0.0.4)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-excon (1.1.0)
+    faraday-http-cache (2.2.0)
+      faraday (>= 0.8)
+    faraday-httpclient (1.0.1)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.2.0)
+    faraday-patron (1.0.0)
+    faraday-rack (1.0.0)
+    git (1.9.1)
+      rchardet (~> 1.8)
+    hashdiff (1.0.1)
+    htmlentities (4.3.4)
+    http-accept (1.7.0)
+    http-cookie (1.0.4)
+      domain_name (~> 0.5)
+    i18n (1.10.0)
+      concurrent-ruby (~> 1.0)
+    json (2.6.1)
+    kramdown (2.3.1)
+      rexml
+    kramdown-parser-gfm (1.1.0)
+      kramdown (~> 2.0)
+    link_header (0.0.8)
+    logger (1.5.1)
+    macaddr (1.7.2)
+      systemu (~> 2.6.5)
+    matrix (0.4.2)
+    method_source (1.0.0)
+    mime-types (3.4.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2022.0105)
+    minitar (0.9)
+    minitest (5.15.0)
+    multipart-post (2.1.1)
+    nap (1.1.0)
+    net-http-persistent (4.0.1)
+      connection_pool (~> 2.2)
+    netrc (0.11.0)
+    no_proxy_fix (0.1.2)
+    octokit (4.21.0)
+      faraday (>= 0.9)
+      sawyer (~> 0.8.0, >= 0.5.3)
+    open4 (1.3.4)
+    openstack (3.3.21)
+      json
+    parallel (1.22.1)
+    parser (3.1.2.1)
+      ast (~> 2.4.1)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    pry-byebug (3.10.1)
+      byebug (~> 11.0)
+      pry (>= 0.13, < 0.15)
+    public_suffix (5.0.0)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    rchardet (1.8.0)
+    rdf (3.2.7)
+      link_header (~> 0.0, >= 0.0.8)
+    rdf-aggregate-repo (3.2.0)
+      rdf (~> 3.2)
+    rdf-n3 (3.2.1)
+      ebnf (~> 2.2)
+      rdf (~> 3.2)
+      sparql (~> 3.2)
+      sxp (~> 1.2)
+    rdf-xsd (3.2.1)
+      rdf (~> 3.2)
+      rexml (~> 3.2)
+    redis (4.6.0)
+    regexp_parser (2.6.0)
+    rest-client (2.1.0)
+      http-accept (>= 1.7.0, < 2.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
+    rexml (3.2.5)
+    rollbar (3.3.0)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.0)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
+    rubocop (1.28.1)
+      parallel (~> 1.10)
+      parser (>= 3.1.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.17.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.23.0)
+      parser (>= 3.1.1.0)
+    rubocop-rspec (2.11.1)
+      rubocop (~> 1.19)
+    ruby-progressbar (1.11.0)
+    ruby2_keywords (0.0.5)
+    sawyer (0.8.2)
+      addressable (>= 2.3.5)
+      faraday (> 0.8, < 2.0)
+    scanf (1.0.0)
+    sparql (3.2.1)
+      builder (~> 3.2)
+      ebnf (~> 2.2)
+      logger (~> 1.4)
+      rdf (~> 3.2, >= 3.2.3)
+      rdf-aggregate-repo (~> 3.2)
+      rdf-xsd (~> 3.2)
+      sparql-client (~> 3.2)
+      sxp (~> 1.2, >= 1.2.1)
+    sparql-client (3.2.1)
+      net-http-persistent (~> 4.0, >= 4.0.1)
+      rdf (~> 3.2, >= 3.2.6)
+    sxp (1.2.2)
+      matrix
+      rdf (~> 3.2)
+    systemu (2.6.5)
+    terminal-table (3.0.2)
+      unicode-display_width (>= 1.1.1, < 3)
+    timecop (0.9.5)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.8.1)
+    unicode-display_width (2.3.0)
+    unicode-types (1.7.0)
+    uuid (2.3.9)
+      macaddr (~> 1.0)
+    validatable (1.6.7)
+    vcr (5.1.0)
+    webmock (3.18.1)
+      addressable (>= 2.8.0)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  bundler (~> 2.0)
+  danger (~> 8.0)
+  pry (~> 0.10, >= 0.10.4)
+  pry-byebug (~> 3.6)
+  pushmi_pullyu!
+  rake (~> 13.0)
+  rspec (~> 3.0)
+  rubocop (~> 1.23)
+  rubocop-rspec (~> 2.6)
+  timecop (~> 0.8)
+  vcr (~> 5.0)
+  webmock (~> 3.3)
+
+BUNDLED WITH
+   2.3.12

data/README.md

@@ -26,7 +26,7 @@ Its primary job is to manage the flow of content from Jupiter into Swift for pre
 
 ## Requirements
 
-PushmiPullyu supports Ruby 2.3.1+
+PushmiPullyu supports Ruby 2.7
 
 ## Installation
 

data/lib/pushmi_pullyu/aip.rb

@@ -6,19 +6,22 @@ module PushmiPullyu::AIP
   module_function
 
   def create(entity)
-    raise EntityInvalid if entity.nil? ||
-                           UUID.validate(entity[:uuid]) != true ||
+    raise EntityInvalid if entity.blank? ||
+                           UUID.validate(entity[:uuid]).blank? ||
                            entity[:type].blank?
 
     aip_directory = "#{PushmiPullyu.options[:workdir]}/#{entity[:uuid]}"
     aip_filename = "#{aip_directory}.tar"
+    begin
+      PushmiPullyu::AIP::Downloader.new(entity, aip_directory).run
+      PushmiPullyu::AIP::Creator.new(entity[:uuid], aip_directory, aip_filename).run
 
-    PushmiPullyu::AIP::Downloader.new(entity, aip_directory).run
-    PushmiPullyu::AIP::Creator.new(entity[:uuid], aip_directory, aip_filename).run
-
-    yield aip_filename, aip_directory
-
-    FileUtils.rm_rf(aip_filename) if File.exist?(aip_filename)
-    FileUtils.rm_rf(aip_directory) if File.exist?(aip_directory)
+      yield aip_filename, aip_directory
+    # Here we will ensure the files are removed even if an exception comes up.
+    # You will notice there is no rescue block.  We will catch exceptions in `PushmiPullyu::CLI`
+    ensure
+      FileUtils.rm_rf(aip_filename)
+      FileUtils.rm_rf(aip_directory)
+    end
   end
 end

data/lib/pushmi_pullyu/cli.rb

@@ -182,11 +182,10 @@ class PushmiPullyu::CLI
   end
 
   def run_preservation_cycle
-    entity_json = JSON.parse(queue.wait_next_item)
-    entity = {
-      type: entity_json['type'],
-      uuid: entity_json['uuid']
-    }
+    entity_json = queue.wait_next_item
+    # jupiter is submitting the entries to reddis in a hash format using fat arrows. We need to change them to colons in
+    # order to parse them correctly from json
+    entity = JSON.parse(entity_json.gsub('=>', ':'), { symbolize_names: true })
     return unless entity[:type].present? && entity[:uuid].present?
 
     # add additional information about the error context to errors that occur while processing this item.
@@ -199,13 +198,21 @@ class PushmiPullyu::CLI
         # Log successful preservation event to the log files
         PushmiPullyu::Logging.log_preservation_event(deposited_file, aip_directory)
       end
-      # rubocop:disable Lint/RescueException
+    # An EntityInvalid expection means there is a problem with the entity information format so there is no point in
+    # readding it to the queue as it will always fail
+    rescue PushmiPullyu::AIP::EntityInvalid => e
+    rescue StandardError => e
+      queue.add_entity_json(entity_json)
+
+    # rubocop:disable Lint/RescueException
+    # Something other than a StandardError exception means something happened which we were not expecting!
+    # Make sure we log the problem
     rescue Exception => e
+      raise e
+    # rubocop:enable Lint/RescueException
+    ensure
       Rollbar.error(e)
       logger.error(e)
-      # TODO: we could re-raise here and let the daemon die on any preservation error, or just log the issue and
-      # move on to the next item.
-      # rubocop:enable Lint/RescueException
     end
   end
 

data/lib/pushmi_pullyu/logging.rb

@@ -47,12 +47,12 @@ module PushmiPullyu::Logging
       preservation_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.log")
       preservation_json_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.json")
 
-      message = "#{deposited_file.name} was successfully deposited into Swift Storage!\n"\
-                "Here are the details of this preservation event:\n"\
-                "\tUUID: '#{deposited_file.name}'\n"\
-                "\tTimestamp of Completion: '#{deposited_file.last_modified}'\n"\
-                "\tAIP Checksum: '#{deposited_file.etag}'\n"\
-                "\tMetadata: #{deposited_file.metadata}\n"\
+      message = "#{deposited_file.name} was successfully deposited into Swift Storage!\n" \
+                "Here are the details of this preservation event:\n" \
+                "\tUUID: '#{deposited_file.name}'\n" \
+                "\tTimestamp of Completion: '#{deposited_file.last_modified}'\n" \
+                "\tAIP Checksum: '#{deposited_file.etag}'\n" \
+                "\tMetadata: #{deposited_file.metadata}\n" \
 
       file_details = file_log_details(aip_directory)
 
@@ -155,7 +155,7 @@ module PushmiPullyu::Logging
               fileset_name: File.dirname(file).split('/')[-1],
               file_name: File.basename(file),
               file_size: File.size(file),
-              file_extension: File.extname(file).strip.downcase[1..-1]
+              file_extension: File.extname(file).strip.downcase[1..]
             }
           end
         end

data/lib/pushmi_pullyu/preservation_queue.rb

@@ -68,6 +68,12 @@ class PushmiPullyu::PreservationQueue
     end
   end
 
+  def add_entity_json(entity_json)
+    @redis.with do |connection|
+      connection.zadd @queue_name, Time.now.to_f, entity_json
+    end
+  end
+
   protected
 
   def connected?

data/lib/pushmi_pullyu/version.rb

@@ -1,3 +1,3 @@
 module PushmiPullyu
-  VERSION = '2.0.3'.freeze
+  VERSION = '2.0.4'.freeze
 end

data/pushmi_pullyu.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.required_ruby_version = '>= 2.5'
+  spec.required_ruby_version = '>= 2.7'
 
   spec.add_runtime_dependency 'activesupport', '>= 5', '< 8'
   spec.add_runtime_dependency 'bagit', '~> 0.4'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pushmi_pullyu
 version: !ruby/object:Gem::Version
-  version: 2.0.3
+  version: 2.0.4
 platform: ruby
 authors:
 - Shane Murnaghan
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-04-28 00:00:00.000000000 Z
+date: 2022-11-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -409,6 +409,7 @@ files:
 - CHANGELOG.md
 - Dangerfile
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -446,7 +447,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.5'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="