preservation 0.1.0 → 0.2.0

data/lib/preservation/report/database.rb

@@ -0,0 +1,26 @@
+module Preservation
+
+  # Reporting
+  #
+  module Report
+
+    # Database
+    #
+    module Database
+
+      # Database connection
+      #
+      # @return [SQLite3::Database]
+       def self.db_connection(db_path)
+        if db_path.nil?
+          puts 'Missing db_path'
+          exit
+        end
+        @db ||= SQLite3::Database.new db_path
+      end
+
+    end
+
+  end
+
+end

data/lib/preservation/report/transfer.rb

@@ -0,0 +1,166 @@
+module Preservation
+
+  module Report
+
+    # Transfer reporting
+    #
+    module Transfer
+
+      # Transfers based on presence (or not) of a particular status
+      #
+      # @param status_to_find [String]
+      # @param status_presence [Boolean]
+      def self.status(status_to_find: nil, status_presence: true)
+        if status_presence === true
+          status_presence = '='
+        else
+          status_presence = '<>'
+        end
+
+        query = "SELECT id, uuid, hex(path) as hex_path, unit_type, status, microservice, current FROM unit WHERE status #{status_presence} ?"
+
+        # Archivematica stores path as BLOB, so need to convert path to Hex, to search for it
+        # and use hex function in DB query
+        records = []
+        db.results_as_hash = true
+        db.execute( query, [ status_to_find ] ) do |row|
+          id = row['id']
+          uuid = row['uuid']
+          bin_path = Preservation::Conversion.hex_to_bin row['hex_path']
+          unit_type = row['unit_type']
+          status = row['status']
+          microservice = row['microservice']
+          current = row['current']
+          o = {}
+          o['path'] = bin_path if !bin_path.empty?
+          o['unit_type'] = unit_type if !unit_type.empty?
+          o['status'] = status if !status.empty?
+          o['microservice'] = microservice if !microservice.empty?
+          o['current'] = current if current
+          o['id'] = id if id
+          o['uuid'] = uuid if !uuid.empty?
+
+          records << o
+        end
+
+        records
+      end
+
+      # Current transfer
+      #
+      # @return [Hash]
+      def self.current
+        query = "SELECT id, uuid, hex(path) as hex_path, unit_type, status, microservice, current FROM unit WHERE current = 1"
+
+        # Archivematica stores path as BLOB, so need to convert path to Hex, to search for it
+        # and use hex function in DB query
+        o = {}
+        db.results_as_hash = true
+        db.execute( query ) do |row|
+          id = row['id']
+          uuid = row['uuid']
+          bin_path = hex_to_bin row['hex_path']
+          unit_type = row['unit_type']
+          status = row['status']
+          microservice = row['microservice']
+          current = row['current']
+          o['path'] = bin_path if !bin_path.empty?
+          o['unit_type'] = unit_type if !unit_type.empty?
+          o['status'] = status if !status.empty?
+          o['microservice'] = microservice if !microservice.empty?
+          o['current'] = current if current
+          o['id'] = id if id
+          o['uuid'] = uuid if !uuid.empty?
+        end
+        o
+      end
+
+      # Count of complete transfers
+      #
+      # @return [Integer]
+      def self.complete_count
+        query = 'SELECT count(*) FROM unit WHERE status = ?'
+
+        status_to_find = 'COMPLETE'
+        db.results_as_hash = true
+        db.get_first_value( query, [status_to_find] )
+      end
+
+      # Compilation of statistics and data, with focus on exceptions
+      #
+      # @return [Hash]
+      def self.exception
+        incomplete = status(status_to_find: 'COMPLETE', status_presence: false)
+        failed = status(status_to_find: 'FAILED', status_presence: true)
+        report = {}
+        report['current'] = current if !current.empty?
+        report['failed'] = {}
+        report['failed']['count'] = failed.count
+        report['failed']['data'] = failed if !failed.empty?
+        report['incomplete'] = {}
+        report['incomplete']['count'] = incomplete.count
+        report['incomplete']['data'] = incomplete if !incomplete.empty?
+        report['complete'] = {}
+        report['complete']['count'] = complete_count if complete_count
+        report
+      end
+
+      # Is it in database?
+      # @param path_to_find [String] directory name within ingest path
+      # @return [Boolean]
+      def self.in_db?(path_to_find)
+        in_db = false
+
+        # Get path out of DB as a hex string
+        query = 'SELECT hex(path) FROM unit'
+
+        # Archivematica stores path as BLOB, so need to convert path to Hex, to search for it
+        # and use hex function in DB query
+        db.execute( query ) do |row|
+          bin_path = Preservation::Conversion.hex_to_bin row[0]
+          if bin_path === path_to_find
+            in_db = true
+          end
+        end
+
+        in_db
+      end
+
+      # Has preservation been done?
+      # @param path_to_find [String] directory name within ingest path
+      # @return [Boolean]
+      def self.preserved?(path_to_find)
+        preserved = false
+
+        # 'ingest' value in unit_type and 'COMPLETE' value in status DB fields
+        # indicates completed
+        unit_type_to_find = 'ingest'
+        status_to_find = 'COMPLETE'
+
+        # Get path out of DB as a hex string for completed ingests
+        query = 'SELECT hex(path) FROM unit WHERE unit_type = ? AND status = ?'
+
+        # Archivematica stores path as BLOB, so need to convert path to Hex, to search for it
+        # and use hex function in DB query
+        db.execute( query, [ unit_type_to_find, status_to_find ] ) do |row|
+          bin_path = Preservation::Conversion.hex_to_bin row[0]
+          if bin_path === path_to_find
+            preserved = true
+          end
+        end
+
+        preserved
+      end
+
+      # Db
+      #
+      # @return [SQLite3::Database]
+      def self.db
+        Preservation::Report::Database.db_connection Preservation.db_path
+      end
+
+    end
+
+  end
+
+end

data/lib/preservation/storage.rb

@@ -0,0 +1,50 @@
+module Preservation
+
+  # Storage
+  #
+  module Storage
+
+    # Free up disk space for completed transfers
+    #
+    def self.cleanup
+      preserved = get_preserved
+      if !preserved.nil? && !preserved.empty?
+        preserved.each do |i|
+          # skip anything that has a different owner to script
+          if File.stat(i).grpowned?
+            FileUtils.remove_dir i
+            # @logger.info 'Deleted ' + i
+          end
+        end
+      end
+    end
+
+    # Enough storage for download?
+    #
+    # @return [Boolean]
+    def self.enough_storage_for_download?(required_bytes)
+      # scale up the required space using a multiplier
+      multiplier = 2
+      available = FreeDiskSpace.bytes('/')
+      required_bytes * multiplier < available ? true : false
+    end
+
+    # Collect all paths from DB where preservation has been done
+    # @return [Array<String>]
+    def self.get_preserved
+      ingest_complete = Preservation::Report::Transfer.status(status_to_find: 'COMPLETE',
+                                                              status_presence: true)
+      preserved = []
+      ingest_complete.each do |i|
+        dir_path = Preservation.ingest_path + '/' + i['path']
+        if File.exists?(dir_path)
+          preserved << dir_path
+        end
+      end
+
+      preserved
+    end
+
+  end
+
+end

data/lib/preservation/temporal.rb

@@ -0,0 +1,21 @@
+module Preservation
+
+  # Temporal
+  #
+  module Temporal
+
+    # time_to_preserve?
+    #
+    # @param start_utc [String]
+    # @param delay [Integer] days to wait (after start date) before preserving
+    # @return [Boolean]
+    def self.time_to_preserve?(start_utc, delay)
+      now = DateTime.now
+      start_datetime = DateTime.parse(start_utc)
+      days_since_start = (now - start_datetime).to_i # result in days
+      days_since_start >= delay ? true : false
+    end
+
+  end
+
+end

data/lib/preservation/transfer/pure.rb

@@ -0,0 +1,215 @@
+module Preservation
+
+  # Transfer preparation
+  #
+  module Transfer
+
+    # Transfer preparation for Pure
+    #
+    class Pure < Ingest
+
+      # @param base_url [String]
+      # @param username [String]
+      # @param password [String]
+      # @param basic_auth [Boolean]
+      def initialize(base_url: nil, username: nil, password: nil, basic_auth: nil)
+        super()
+        @base_url = base_url
+        @basic_auth = basic_auth
+        if basic_auth === true
+          @username = username
+          @password = password
+        end
+      end
+
+      # For given uuid, if necessary, fetch the metadata,
+      # prepare a directory in the ingest path and populate it with the files and
+      # JSON description file.
+      #
+      # @param uuid [String] uuid to preserve
+      # @param dir_scheme [Symbol] how to make directory name
+      # @param delay [Integer] days to wait (after modification date) before preserving
+      def prepare_dataset(uuid: nil,
+                          dir_scheme: :uuid,
+                          delay: 0)
+        if uuid.nil?
+          @logger.error 'Missing ' + uuid
+          exit
+        end
+        dir_base_path = Preservation.ingest_path
+
+        dataset = Puree::Dataset.new base_url: @base_url,
+                                     username: @username,
+                                     password: @password,
+                                     basic_auth: @basic_auth
+
+        dataset.find uuid: uuid
+        d = dataset.metadata
+        if d.empty?
+          @logger.error 'No metadata for ' + uuid
+          exit
+        end
+        # configurable to become more human-readable
+        dir_name = Preservation::Builder.build_directory_name(d, dir_scheme)
+
+        # continue only if dir_name is not empty (e.g. because there was no DOI)
+        # continue only if there is no DB entry
+        # continue only if the dataset has a DOI
+        # continue only if there are files for this resource
+        # continue only if it is time to preserve
+        if !dir_name.nil? &&
+           !dir_name.empty? &&
+           !Preservation::Report::Transfer.in_db?(dir_name) &&
+           !d['doi'].empty? &&
+           !d['file'].empty? &&
+           Preservation::Temporal.time_to_preserve?(d['modified'], delay)
+
+          dir_file_path = dir_base_path + '/' + dir_name
+          dir_metadata_path = dir_file_path + '/metadata/'
+          metadata_filename = dir_metadata_path + 'metadata.json'
+
+          # calculate total size of data files
+          download_storage_required = 0
+          d['file'].each { |i| download_storage_required += i['size'].to_i }
+
+          # do we have enough space in filesystem to fetch data files?
+          if Preservation::Storage.enough_storage_for_download? download_storage_required
+            # @logger.info 'Sufficient disk space for ' + dir_file_path
+          else
+            @logger.error 'Insufficient disk space to store files fetched from Pure. Skipping ' + dir_file_path
+          end
+
+          # has metadata file been created? if so, files and metadata are in place
+          # continue only if files not present in ingest location
+          if !File.size? metadata_filename
+
+            @logger.info 'Preparing ' + dir_name + ', Pure UUID ' + d['uuid']
+
+            data = []
+            d['file'].each do |f|
+              o = package_dataset_metadata d, f
+              data << o
+              wget_str = Preservation::Builder.build_wget @username,
+                                                          @password,
+                                                          f['url']
+
+              Dir.mkdir(dir_file_path) if !Dir.exists?(dir_file_path)
+
+              # fetch the file
+              Dir.chdir(dir_file_path) do
+                # puts 'Changing dir to ' + Dir.pwd
+                # puts 'Size of ' + f['name'] + ' is ' + File.size(f['name']).to_s
+                if File.size?(f['name'])
+                  # puts 'Should be deleting ' + f['name']
+                  File.delete(f['name'])
+                end
+                # puts f['name'] + ' missing or empty'
+                # puts wget_str
+                `#{wget_str}`
+              end
+            end
+
+            Dir.mkdir(dir_metadata_path) if !Dir.exists?(dir_metadata_path)
+
+            pretty = JSON.pretty_generate( data, :indent => '  ')
+            # puts pretty
+            File.write(metadata_filename,pretty)
+            @logger.info 'Created ' + metadata_filename
+          else
+            @logger.info 'Skipping ' + dir_name + ', Pure UUID ' + d['uuid'] +
+                         ' because ' + metadata_filename + ' exists'
+          end
+        else
+          @logger.info 'Skipping ' + dir_name + ', Pure UUID ' + d['uuid']
+        end
+      end
+
+      private
+
+      def package_dataset_metadata(d, f)
+          o = {}
+          o['filename'] = 'objects/' + f['name']
+          o['dc.title'] = d['title']
+          if !d['description'].empty?
+            o['dc.description'] = d['description']
+          end
+          o['dcterms.created'] = d['created']
+          if !d['available']['year'].empty?
+            o['dcterms.available'] = Puree::Date.iso(d['available'])
+          end
+          o['dc.publisher'] = d['publisher']
+          if !d['doi'].empty?
+            o['dc.identifier'] = d['doi']
+          end
+          if !d['spatial'].empty?
+            o['dcterms.spatial'] = d['spatial']
+          end
+          if !d['temporal']['start']['year'].empty?
+            temporal_range = ''
+            temporal_range << Puree::Date.iso(d['temporal']['start'])
+            if !d['temporal']['end']['year'].empty?
+              temporal_range << '/'
+              temporal_range << Puree::Date.iso(d['temporal']['end'])
+            end
+            o['dcterms.temporal'] = temporal_range
+          end
+          creators = []
+          contributors = []
+          person_types = %w(internal external other)
+          person_types.each do |person_type|
+            d['person'][person_type].each do |i|
+              if i['role'] == 'Creator'
+                creator = i['name']['last'] + ', ' + i['name']['first']
+                creators << creator
+              end
+              if i['role'] == 'Contributor'
+                contributor = i['name']['last'] + ', ' + i['name']['first']
+                contributors << contributor
+              end
+            end
+          end
+          o['dc.creator'] = creators
+          if !contributors.empty?
+            o['dc.contributor'] = contributors
+          end
+          keywords = []
+          d['keyword'].each { |i|
+            keywords << i
+          }
+          if !keywords.empty?
+            o['dc.subject'] = keywords
+          end
+          if !f['license']['name'].empty?
+            o['dcterms.license'] = f['license']['name']
+          end
+          # o['dc.format'] = f['mime']
+
+          related = []
+          publications = d['publication']
+          publications.each do |i|
+            o_related = {}
+            o_related['dc.title'] = i['title']
+            o_related['type'] = i['type']
+            pub = Puree::Publication.new base_url: @base_url,
+                                         username: @username,
+                                         password: @password,
+                                         basic_auth: @basic_auth
+            pub.find uuid: i['uuid']
+            doi = pub.doi
+            if doi
+              o_related['dc.identifier'] = doi
+            end
+            related << o_related
+          end
+          if !related.empty?
+            o['related'] = related
+          end
+
+          o
+      end
+
+    end
+
+  end
+
+end