preservation 0.2.2 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8a7fb8d9bbb3ee395f23b470f0fba14641bba7d5
-  data.tar.gz: c2452352ba383321304b6a8c030bf514d8956113
+  metadata.gz: 922ab4347457976c6f59c174922cec3699ec186b
+  data.tar.gz: 52fcc64ecab218ffa17f270d69d954d81c779142
 SHA512:
-  metadata.gz: 33dddbb76619f3b188644c1d7c0fba431e3559967fdc307cc363e6f8fe7054c4fb363f2604980bced5ee9477f4d7fb1511ce2e5a8d46ec17d788d8bbde27e6e7
-  data.tar.gz: a268d12ff6c6e6955ebd678c479ece0433ba341032480ff87993ec35c549ba2af4fdb9a5b3171468bb3dd3306d0d71b5cfda0ab1067100c20577085e4ab37366
+  metadata.gz: 9dba19729c2c899a7e9e1399f66e95eaaaf3ede7d15adfce8d3f7b8c43d0ff932e014e261bcd3aec961335f8b41ee96f3f942a1e7a131b953ccd2662a1b44029
+  data.tar.gz: 42fd26b22096093d0f199b69622f8d4e92f100f7ad2d259cd43b27eead7357ca5fa4054204568439211f7ee3674f8af7e43ea07f6663cf365f9223ca0240e2f2

data/CHANGELOG.md

@@ -4,6 +4,11 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
+## 0.3.0 - 2016-09-30
+### Added
+- Transfer - prepare batches of datasets.
+- Reporting - pending transfers.
+
 ## 0.2.2 - 2016-09-28
 ### Fixed
 - Transfer - related work as simple array in metadata.

data/PITCHME.md

@@ -84,17 +84,9 @@ Preservation::Storage.cleanup
       "positive selection"
     ],
     "dcterms.license": "CC BY",
-    "related": [
-      {
-        "dc.title": "The unprecedented scale of the West African Ebola virus disease outbreak is due to environmental and sociological factors, not special attributes of the currently circulating strain of the virus",
-        "type": "Journal article",
-        "dc.identifier": "http://dx.doi.org/10.1136/ebmed-2014-110127"
-      },
-      {
-        "dc.title": "The 2014 Ebola virus disease outbreak in West Africa",
-        "type": "Journal article",
-        "dc.identifier": "http://dx.doi.org/10.1099/vir.0.067199-0"
-      }
+    "dc.relation": [
+      "http://dx.doi.org/10.1136/ebmed-2014-110127",
+      "http://dx.doi.org/10.1099/vir.0.067199-0"
     ]
   }
 ]

data/lib/preservation/builder.rb

@@ -31,7 +31,7 @@ module Preservation
 
     # Build directory name
     #
-    # @param metadata record [Hash]
+    # @param metadata_record [Hash]
     # @param directory_name_scheme [Symbol]
     # @return [String]
     def self.build_directory_name(metadata_record, directory_name_scheme)

data/lib/preservation/conversion.rb

@@ -5,7 +5,7 @@ module Preservation
   module Conversion
     # Binary to hexadecimal
     #
-    # @param [Binary String]
+    # @param s [Binary String]
     # @return [Hexadecimal String]
     def self.bin_to_hex(s)
       s.each_byte.map { |b| b.to_s(16) }.join
@@ -13,7 +13,7 @@ module Preservation
 
     # Hexadecimal to binary
     #
-    # @param [Hexadecimal String]
+    # @param s [Hexadecimal String]
     # @return [Binary String]
     def self.hex_to_bin(s)
       s.scan(/../).map { |x| x.hex.chr }.join

data/lib/preservation/report/transfer.rb

@@ -10,6 +10,7 @@ module Preservation
       #
       # @param status_to_find [String]
       # @param status_presence [Boolean]
+      # @return [Array<Hash>]
       def self.status(status_to_find: nil, status_presence: true)
         if status_presence === true
           status_presence = '='
@@ -22,7 +23,10 @@ module Preservation
         records = []
         db.results_as_hash = true
         db.execute( query, [ status_to_find ] ) do |row|
-          records << row_to_hash(row)
+          bin_path = Preservation::Conversion.hex_to_bin row['hex_path']
+          if !bin_path.nil? && !bin_path.empty?
+            records << row_to_hash(row)
+          end
         end
 
         records
@@ -53,22 +57,61 @@ module Preservation
         db.get_first_value( query, [status_to_find] )
       end
 
+      # Pending transfers
+      #
+      # @return [Hash]
+      def self.pending
+        # Get the directories
+        dirs = Dir.entries Preservation.ingest_path
+        a = []
+        # For each directory, if it isn't in the db, add it to list
+        dirs.each do |dir|
+          next if !in_db?(dir)
+          o = {}
+          o['path'] = dir
+          o['path_timestamp'] = File.mtime "#{Preservation.ingest_path}/#{dir}"
+          a << o
+        end
+        a
+      end
+
+      # Is there a pending transfer with this path?
+      #
+      # @return [Boolean]
+      def self.pending?(path_to_find)
+        is_pending = false
+        pending.each do |i|
+          if i['path'] == path_to_find
+            is_pending = true
+            break
+          end
+        end
+        is_pending
+      end
+
+
       # Compilation of statistics and data, with focus on exceptions
       #
       # @return [Hash]
       def self.exception
-        incomplete = status(status_to_find: 'COMPLETE', status_presence: false)
-        failed = status(status_to_find: 'FAILED', status_presence: true)
+        incomplete_result = status(status_to_find: 'COMPLETE', status_presence: false)
+        failed_result = status(status_to_find: 'FAILED', status_presence: true)
+        pending_result = pending
+        current_result = current
+        complete_count_result = complete_count
         report = {}
-        report['current'] = current if !current.empty?
+        report['pending'] = {}
+        report['pending']['count'] = pending_result.count
+        report['pending']['data'] = pending_result if !pending_result.empty?
+        report['current'] = current_result if !current_result.empty?
         report['failed'] = {}
-        report['failed']['count'] = failed.count
-        report['failed']['data'] = failed if !failed.empty?
+        report['failed']['count'] = failed_result.count
+        report['failed']['data'] = failed_result if !failed_result.empty?
         report['incomplete'] = {}
-        report['incomplete']['count'] = incomplete.count
-        report['incomplete']['data'] = incomplete if !incomplete.empty?
+        report['incomplete']['count'] = incomplete_result.count
+        report['incomplete']['data'] = incomplete_result if !incomplete_result.empty?
         report['complete'] = {}
-        report['complete']['count'] = complete_count if complete_count
+        report['complete']['count'] = complete_count_result if complete_count_result
         report
       end
 
@@ -119,6 +162,8 @@ module Preservation
         preserved
       end
 
+      private
+
       # Db
       #
       # @return [SQLite3::Database]
@@ -144,6 +189,10 @@ module Preservation
         o['current'] = current if current
         o['id'] = id if id
         o['uuid'] = uuid if !uuid.nil? && !uuid.empty?
+        path = "#{Preservation.ingest_path}/#{bin_path}"
+        if File.exist? path
+          o['path_timestamp'] = File.mtime path
+        end
         o
       end
 

data/lib/preservation/transfer/pure.rb

@@ -29,9 +29,12 @@ module Preservation
       # @param uuid [String] uuid to preserve
       # @param dir_scheme [Symbol] how to make directory name
       # @param delay [Integer] days to wait (after modification date) before preserving
+      # @return [Boolean] indicates presence of metadata description file
       def prepare_dataset(uuid: nil,
                           dir_scheme: :uuid,
                           delay: 0)
+        success = false
+
         if uuid.nil?
           @logger.error 'Missing ' + uuid
           exit
@@ -49,6 +52,7 @@ module Preservation
           @logger.error 'No metadata for ' + uuid
           exit
         end
+
         # configurable to become more human-readable
         dir_name = Preservation::Builder.build_directory_name(d, dir_scheme)
 
@@ -115,6 +119,7 @@ module Preservation
             # puts pretty
             File.write(metadata_filename,pretty)
             @logger.info 'Created ' + metadata_filename
+            success = true
           else
             @logger.info 'Skipping ' + dir_name + ', Pure UUID ' + d['uuid'] +
                          ' because ' + metadata_filename + ' exists'
@@ -122,6 +127,49 @@ module Preservation
         else
           @logger.info 'Skipping ' + dir_name + ', Pure UUID ' + d['uuid']
         end
+        success
+      end
+
+      # For multiple datasets, if necessary, fetch the metadata,
+      # prepare a directory in the ingest path and populate it with the files and
+      # JSON description file.
+      #
+      # @param max [Integer] maximum to prepare, omit to set no maximum
+      # @param dir_scheme [Symbol] how to make directory name
+      # @param delay [Integer] days to wait (after modification date) before preserving
+      def prepare_dataset_batch(max: nil,
+                                dir_scheme: :uuid,
+                                delay: 30)
+        collection = Puree::Collection.new resource:  :dataset,
+                                           base_url:   @base_url,
+                                           username:   @username,
+                                           password:   @password,
+                                           basic_auth: @basic_auth
+        count = collection.count
+
+        max = count if max.nil?
+
+        batch_size = 10
+        num_prepared = 0
+        0.step(count, batch_size) do |n|
+
+          minimal_metadata = collection.find limit:  batch_size,
+                                             offset: n,
+                                             full:   false
+          uuids = []
+          minimal_metadata.each do |i|
+            uuids << i['uuid']
+          end
+
+          uuids.each do |uuid|
+            success = prepare_dataset uuid:       uuid,
+                                      dir_scheme: dir_scheme.to_sym,
+                                      delay:      delay
+
+            num_prepared += 1 if success
+            exit if num_prepared == max
+          end
+        end
       end
 
       private

data/lib/preservation/version.rb

@@ -1,5 +1,5 @@
 module Preservation
   # Semantic version number
   #
-  VERSION = "0.2.2"
+  VERSION = "0.4.0"
 end

data/preservation.gemspec

@@ -21,6 +21,6 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '~> 2.1'
 
   spec.add_runtime_dependency 'free_disk_space', '~> 1.0'
-  spec.add_runtime_dependency 'puree', '~> 0.17'
+  spec.add_runtime_dependency 'puree', '~> 0.19'
   spec.add_runtime_dependency 'sqlite3', '~> 1.3'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: preservation
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.4.0
 platform: ruby
 authors:
 - Adrian Albin-Clark
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-28 00:00:00.000000000 Z
+date: 2016-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: free_disk_space
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.17'
+        version: '0.19'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.17'
+        version: '0.19'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement