preservation 0.1.0 → 0.2.0

data/lib/preservation/version.rb

@@ -1,5 +1,5 @@
 module Preservation
   # Semantic version number
   #
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/preservation.gemspec

@@ -8,9 +8,9 @@ Gem::Specification.new do |spec|
   spec.version       = Preservation::VERSION
   spec.authors       = ["Adrian Albin-Clark"]
   spec.email         = ["a.albin-clark@lancaster.ac.uk"]
-  spec.summary       = %q{Ingest management for Archivematica's Automation Tools.}
-  spec.description   = %q{Transfer preparation, reporting and disk space management for Archivematica's Automation Tools.}
-  spec.homepage      = "https://rubygems.org/gems/preservation"
+  spec.summary       = %q{Extraction and Transformation for Loading by Archivematica's Automation Tools.}
+  spec.description   = %q{Extraction and Transformation for Loading by Archivematica's Automation Tools. Includes transfer preparation, reporting and disk space management.}
+  spec.homepage      = "https://aalbinclark.gitbooks.io/preservation"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files -z`.split("\x0")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: preservation
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Adrian Albin-Clark
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-13 00:00:00.000000000 Z
+date: 2016-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: free_disk_space
@@ -52,8 +52,8 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
-description: Transfer preparation, reporting and disk space management for Archivematica's
-  Automation Tools.
+description: Extraction and Transformation for Loading by Archivematica's Automation
+  Tools. Includes transfer preparation, reporting and disk space management.
 email:
 - a.albin-clark@lancaster.ac.uk
 executables: []
@@ -68,14 +68,18 @@ files:
 - README.md
 - Rakefile
 - lib/preservation.rb
+- lib/preservation/builder.rb
 - lib/preservation/configuration.rb
+- lib/preservation/conversion.rb
 - lib/preservation/ingest.rb
-- lib/preservation/ingest_report.rb
-- lib/preservation/pure_ingest.rb
-- lib/preservation/string_util.rb
+- lib/preservation/report/database.rb
+- lib/preservation/report/transfer.rb
+- lib/preservation/storage.rb
+- lib/preservation/temporal.rb
+- lib/preservation/transfer/pure.rb
 - lib/preservation/version.rb
 - preservation.gemspec
-homepage: https://rubygems.org/gems/preservation
+homepage: https://aalbinclark.gitbooks.io/preservation
 licenses:
 - MIT
 metadata: {}
@@ -98,6 +102,6 @@ rubyforge_project:
 rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
-summary: Ingest management for Archivematica's Automation Tools.
+summary: Extraction and Transformation for Loading by Archivematica's Automation Tools.
 test_files: []
 has_rdoc: 

data/lib/preservation/ingest_report.rb

@@ -1,172 +0,0 @@
-module Preservation
-
-  # Ingest reporting
-  #
-  class IngestReport
-
-    def initialize
-      create_db_connection
-    end
-
-    # Transfers based on presence (or not) of a particular status
-    #
-    # @param status_to_find [String]
-    # @param status_presence [Boolean]
-    def transfer_status(status_to_find: nil, status_presence: true)
-      if status_presence === true
-        status_presence = '='
-      else
-        status_presence = '<>'
-      end
-
-      query = "SELECT id, uuid, hex(path) as hex_path, unit_type, status, microservice, current FROM unit WHERE status #{status_presence} ?"
-
-      # Archivematica stores path as BLOB, so need to convert path to Hex, to search for it
-      # and use hex function in DB query
-      records = []
-      @db.results_as_hash = true
-      @db.execute( query, [ status_to_find ] ) do |row|
-        id = row['id']
-        uuid = row['uuid']
-        bin_path = StringUtil.hex_to_bin row['hex_path']
-        unit_type = row['unit_type']
-        status = row['status']
-        microservice = row['microservice']
-        current = row['current']
-        o = {}
-        o['path'] = bin_path if !bin_path.empty?
-        o['unit_type'] = unit_type if !unit_type.empty?
-        o['status'] = status if !status.empty?
-        o['microservice'] = microservice if !microservice.empty?
-        o['current'] = current if current
-        o['id'] = id if id
-        o['uuid'] = uuid if !uuid.empty?
-
-        records << o
-      end
-
-      records
-    end
-
-    # Current transfer
-    #
-    # @return [Hash]
-    def transfer_current
-      query = "SELECT id, uuid, hex(path) as hex_path, unit_type, status, microservice, current FROM unit WHERE current = 1"
-
-      # Archivematica stores path as BLOB, so need to convert path to Hex, to search for it
-      # and use hex function in DB query
-      o = {}
-      @db.results_as_hash = true
-      @db.execute( query ) do |row|
-        id = row['id']
-        uuid = row['uuid']
-        bin_path = hex_to_bin row['hex_path']
-        unit_type = row['unit_type']
-        status = row['status']
-        microservice = row['microservice']
-        current = row['current']
-        o['path'] = bin_path if !bin_path.empty?
-        o['unit_type'] = unit_type if !unit_type.empty?
-        o['status'] = status if !status.empty?
-        o['microservice'] = microservice if !microservice.empty?
-        o['current'] = current if current
-        o['id'] = id if id
-        o['uuid'] = uuid if !uuid.empty?
-      end
-      o
-    end
-
-    # Count of complete transfers
-    #
-    # @return [Integer]
-    def transfer_complete_count
-      query = 'SELECT count(*) FROM unit WHERE status = ?'
-
-      status_to_find = 'COMPLETE'
-      @db.results_as_hash = true
-      @db.get_first_value( query, [status_to_find] )
-    end
-
-    # Compilation of statistics and data, with focus on exceptions
-    #
-    # @return [Hash]
-    def transfer_exception
-      incomplete = transfer_status(status_to_find: 'COMPLETE', status_presence: false)
-      failed = transfer_status(status_to_find: 'FAILED', status_presence: true)
-      current = transfer_current
-      complete_count = transfer_complete_count
-      report = {}
-      report['current'] = current if !current.empty?
-      report['failed'] = {}
-      report['failed']['count'] = failed.count
-      report['failed']['data'] = failed if !failed.empty?
-      report['incomplete'] = {}
-      report['incomplete']['count'] = incomplete.count
-      report['incomplete']['data'] = incomplete if !incomplete.empty?
-      report['complete'] = {}
-      report['complete']['count'] = complete_count if complete_count
-      report
-    end
-
-    # Is it in database?
-    # @param path_to_find [String] directory name within ingest path
-    # @return [Boolean]
-    def in_db?(path_to_find)
-      in_db = false
-
-      # Get path out of DB as a hex string
-      query = 'SELECT hex(path) FROM unit'
-
-      # Archivematica stores path as BLOB, so need to convert path to Hex, to search for it
-      # and use hex function in DB query
-      @db.execute( query ) do |row|
-        bin_path = StringUtil.hex_to_bin row[0]
-        if bin_path === path_to_find
-          in_db = true
-        end
-      end
-
-      in_db
-    end
-
-    # Has preservation been done?
-    # @param path_to_find [String] directory name within ingest path
-    # @return [Boolean]
-    def preserved?(path_to_find)
-      preserved = false
-
-      # 'ingest' value in unit_type and 'COMPLETE' value in status DB fields
-      # indicates completed
-      unit_type_to_find = 'ingest'
-      status_to_find = 'COMPLETE'
-
-      # Get path out of DB as a hex string for completed ingests
-      query = 'SELECT hex(path) FROM unit WHERE unit_type = ? AND status = ?'
-
-      # Archivematica stores path as BLOB, so need to convert path to Hex, to search for it
-      # and use hex function in DB query
-      @db.execute( query, [ unit_type_to_find, status_to_find ] ) do |row|
-        bin_path = StringUtil.hex_to_bin row[0]
-        if bin_path === path_to_find
-          preserved = true
-        end
-      end
-
-      preserved
-    end
-
-
-    private
-
-    def create_db_connection
-      if Preservation.db_path.nil?
-        puts 'Missing db_path'
-        exit
-      end
-      @db = SQLite3::Database.new Preservation.db_path
-    end
-
-  end
-
-end

data/lib/preservation/pure_ingest.rb

@@ -1,188 +0,0 @@
-module Preservation
-
-  # Ingest for Pure
-  #
-  class PureIngest < Ingest
-
-    def initialize
-      super
-    end
-
-    # For each uuid, if necessary, fetch the metadata,
-    # prepare a directory in the ingest path and populate it with the files and
-    # JSON description file.
-    #
-    # @param uuids [Array<String>] uuids to preserve
-    # @param dir_name_scheme [Symbol] method to make directory name
-    # @param delay [Integer] days to wait (after modification date) before preserving
-    def prepare_dataset(uuids: [],
-                        dir_name_scheme: :uuid,
-                        delay: 0)
-      dir_base_path = Preservation.ingest_path
-
-      uuids.each do |uuid|
-        dataset = Puree::Dataset.new
-        dataset.find uuid: uuid
-        d = dataset.metadata
-        if d.empty?
-          @logger.info 'No metadata for ' + uuid
-          next
-        end
-        # configurable to become more human-readable
-        dir_name = build_directory_name(d, dir_name_scheme)
-
-        # continue only if dir_name is not empty (e.g. because there was no DOI)
-        # continue only if there is no DB entry
-        # continue only if the dataset has a DOI
-        # continue only if there are files for this resource
-        # continue only if it is time to preserve
-        if !dir_name.nil? &&
-           !dir_name.empty? &&
-           !@report.in_db?(dir_name) &&
-           !d['doi'].empty? &&
-           !d['file'].empty? &&
-           time_to_preserve?(d['modified'], delay)
-
-          dir_file_path = dir_base_path + '/' + dir_name
-          dir_metadata_path = dir_file_path + '/metadata/'
-          metadata_filename = dir_metadata_path + 'metadata.json'
-
-          # calculate total size of data files
-          download_storage_required = 0
-          d['file'].each { |i| download_storage_required += i['size'].to_i }
-
-          # do we have enough space in filesystem to fetch data files?
-          if enough_storage_for_download? download_storage_required
-            # @logger.info 'Sufficient disk space for ' + dir_file_path
-          else
-            @logger.error 'Insufficient disk space to store files fetched from Pure. Skipping ' + dir_file_path
-            next
-          end
-
-          # has metadata file been created? if so, files and metadata are in place
-          # continue only if files not present in ingest location
-          if !File.size? metadata_filename
-
-            @logger.info 'Preparing ' + dir_name + ', Pure UUID ' + d['uuid']
-
-            data = []
-            d['file'].each do |f|
-              o = package_dataset_metadata d, f
-              data << o
-              wget_str = build_wget Puree.username,
-                                    Puree.password,
-                                    f['url']
-
-              Dir.mkdir(dir_file_path) if !Dir.exists?(dir_file_path)
-
-              # fetch the file
-              Dir.chdir(dir_file_path) do
-                # puts 'Changing dir to ' + Dir.pwd
-                # puts 'Size of ' + f['name'] + ' is ' + File.size(f['name']).to_s
-                if File.size?(f['name'])
-                  # puts 'Should be deleting ' + f['name']
-                  File.delete(f['name'])
-                end
-                # puts f['name'] + ' missing or empty'
-                # puts wget_str
-                `#{wget_str}`
-              end
-            end
-
-            Dir.mkdir(dir_metadata_path) if !Dir.exists?(dir_metadata_path)
-
-            pretty = JSON.pretty_generate( data, :indent => '  ')
-            # puts pretty
-            File.write(metadata_filename,pretty)
-            @logger.info 'Created ' + metadata_filename
-          end
-        else
-          @logger.info 'Skipping ' + dir_name + ', Pure UUID ' + d['uuid']
-        end
-      end
-    end
-
-    private
-
-    def package_dataset_metadata(d, f)
-        o = {}
-        o['filename'] = 'objects/' + f['name']
-        o['dc.title'] = d['title']
-        if !d['description'].empty?
-          o['dc.description'] = d['description']
-        end
-        o['dcterms.created'] = d['created']
-        if !d['available']['year'].empty?
-          o['dcterms.available'] = Puree::Date.iso(d['available'])
-        end
-        o['dc.publisher'] = d['publisher']
-        if !d['doi'].empty?
-          o['dc.identifier'] = d['doi']
-        end
-        if !d['spatial'].empty?
-          o['dcterms.spatial'] = d['spatial']
-        end
-        if !d['temporal']['start']['year'].empty?
-          temporal_range = ''
-          temporal_range << Puree::Date.iso(d['temporal']['start'])
-          if !d['temporal']['end']['year'].empty?
-            temporal_range << '/'
-            temporal_range << Puree::Date.iso(d['temporal']['end'])
-          end
-          o['dcterms.temporal'] = temporal_range
-        end
-        creators = []
-        contributors = []
-        person_types = %w(internal external other)
-        person_types.each do |person_type|
-          d['person'][person_type].each do |i|
-            if i['role'] == 'Creator'
-              creator = i['name']['last'] + ', ' + i['name']['first']
-              creators << creator
-            end
-            if i['role'] == 'Contributor'
-              contributor = i['name']['last'] + ', ' + i['name']['first']
-              contributors << contributor
-            end
-          end
-        end
-        o['dc.creator'] = creators
-        if !contributors.empty?
-          o['dc.contributor'] = contributors
-        end
-        keywords = []
-        d['keyword'].each { |i|
-          keywords << i
-        }
-        if !keywords.empty?
-          o['dc.subject'] = keywords
-        end
-        if !f['license']['name'].empty?
-          o['dcterms.license'] = f['license']['name']
-        end
-        # o['dc.format'] = f['mime']
-
-        related = []
-        publications = d['publication']
-        publications.each do |i|
-          o_related = {}
-          o_related['dc.title'] = i['title']
-          o_related['type'] = i['type']
-          pub = Puree::Publication.new
-          pub.find uuid: i['uuid']
-          doi = pub.doi
-          if doi
-            o_related['dc.identifier'] = doi
-          end
-          related << o_related
-        end
-        if !related.empty?
-          o['related'] = related
-        end
-
-        o
-    end
-
-  end
-
-end