pushmi_pullyu 0.2.7 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 257884fa2eaea476a4765f03063c07b19927cd61
-  data.tar.gz: 858b6ae6809fca4f0b291e9a7e868686de8d5b36
+SHA256:
+  metadata.gz: 3c97d2575b16539392c9b268d62bf037ae289fc4989dfa976ca13796f01dec01
+  data.tar.gz: 2b277188e5873035ebc1c9a2bebdcf7e4837016091d2bc7b4236be90a5f74cef
 SHA512:
-  metadata.gz: 17108f728721e24db410f4957fa10401252e458dc98663774a928655cb6ad1e71b10ecc6dcfef62484cbc164ea32e6317310e6775920fa278f8dbf5e8609f1a0
-  data.tar.gz: 1601222c2a1769f1a5174afea65e01952fe3cea9ad1f162e102a16e2586c5e77801c8b6e742b8f3106efd3735db7179162916810092f1255d7169e6f4efe2091
+  metadata.gz: 36131cfed011b68a8f006d2c25968d25717919d4eb9b8016d58e65887fbe933ea61243c649bfc66b385353c527ab81ec0c78b14d226d45379f51bfd237ef5eaa
+  data.tar.gz: b39f6b537f3e4617efb7c778908788b038bdd7aea9f2198c34ab5276ec5ca33ddd3a90694ee548868a3b7739aecfdb50d08448477bde8c3a6beee8092d9fcf47

data/.rubocop.yml

@@ -59,7 +59,7 @@ Style/ClassAndModuleChildren:
 Style/Documentation:
   Enabled: false
 
-Style/FileName:
+Naming/FileName:
   Exclude:
     - Dangerfile
     - Rakefile

data/README.md

@@ -17,11 +17,11 @@ Its primary job is to manage the flow of content from Fedora into Swift for pres
 
 ## Workflow
 
-1.  Any save (create or update) on a GenericFile in ERA will trigger an after save callback that will push the GenericFile unique identifier (NOID) into a Queue.
-2. The queue (Redis) is setup to be a unique set (which only allows one GenericFile NOID to be included in the queue at a single time), and ordered by priority from First In, First out (FIFO).
+1.  Any save (create or update) on a Item/Thesis in ERA/Jupiter will trigger an after save callback that will push the item's unique identifier (UUID or NOID) into a Queue.
+2. The queue (Redis) is setup to be a unique set (which only allows one item's UUID to be included in the queue at a single time), and ordered by priority from First In, First out (FIFO).
 3. PushmiPullyu will then monitor the queue. After a certain wait period has passed since an element has been on the queue, PushmiPullyu will then retrieve the elements off the queue and begin to process the preservation event.
-4. All the GenericFile information and data required for preservation are retrieved from Fedora and Solr using multiple REST calls.
-5. An Archival Information Package (AIP) is created from the GenericFile's information. It is then bagged and tarred.
+4. All the GenericFile information and data required for preservation are retrieved from Fedora using multiple REST calls. A database connection to the user database fetches (via ActiveRecord )owner emails and modifies the fetched documents, where applicable.
+5. An Archival Information Package (AIP) is created from the item's information. It is then bagged and tarred.
 6. The AIP tar is then uploaded to Swift via a REST call.
 7. On a successful Swift upload, a entry is added for this preservation event to the preservation event logs.
 

data/examples/pushmi_pullyu.yml

@@ -8,6 +8,7 @@
 # PushmiPullyu will run this file through ERB when reading it so you can
 # even put in dynamic logic, like consuming ENV Variables.
 
+aip_version: 'lightaip-2.0'
 debug: false
 logdir: log
 monitor: false
@@ -20,15 +21,18 @@ minimum_age: 0
 redis:
   url: redis://localhost:6379
 
-solr:
-  url: http://localhost:8983/solr/development
-
 fedora:
-  url: http://localhost:8983/fedora/rest
+  url: http://localhost:8080/fcrepo/rest
   user: fedoraAdmin
   password: fedoraAdmin
   base_path: /dev
 
+database:
+  encoding: utf8
+  url: postgresql://jupiter:mysecretpassword@127.0.0.1
+  database: jupiter_development
+  pool: 5
+
 #parameters project_name and project_domain_name are required only for keystone v3 authentication
 swift:
   tenant: tester

data/lib/pushmi_pullyu/aip/creator.rb

@@ -22,11 +22,15 @@ class PushmiPullyu::AIP::Creator
   private
 
   def bag_aip
-    bag = BagIt::Bag.new(@aip_directory)
+    bag = BagIt::Bag.new(@aip_directory, bag_metadata)
     bag.manifest!
     raise BagInvalid unless bag.valid?
   end
 
+  def bag_metadata
+    { 'AIP-Version' => PushmiPullyu.options[:aip_version] }
+  end
+
   def tar_bag
     # We want to change the directory to the work directory path so we get the tar file to be exactly
     # the contents of the noid directory and not the entire work directory structure. For example the noid.tar

data/lib/pushmi_pullyu/aip/downloader.rb

@@ -7,7 +7,18 @@ require 'rdf/n3'
 # related to an object
 class PushmiPullyu::AIP::Downloader
 
+  PREDICATE_URIS = {
+    filename: 'http://purl.org/dc/terms/title',
+    member_files: 'http://pcdm.org/models#hasFile',
+    member_file_sets: 'http://pcdm.org/models#hasMember',
+    original_file: 'http://pcdm.org/use#OriginalFile',
+    type: 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'
+  }.freeze
+
+  class NoFileSets < StandardError; end
+  class NoMemberFiles < StandardError; end
   class NoContentFilename < StandardError; end
+  class NoOriginalFile < StandardError; end
 
   def initialize(noid, aip_directory)
     @noid = noid
@@ -19,21 +30,45 @@ class PushmiPullyu::AIP::Downloader
 
     PushmiPullyu.logger.info("#{@noid}: Retreiving data from Fedora ...")
 
-    [:main_object, :fixity, :content_datastream_metadata, :versions, :thumbnail,
-     :characterization, :fedora3foxml, :fedora3foxml_metadata].each do |item|
-      path_spec = aip_paths[item]
-      download_and_log(path_spec, PushmiPullyu::AIP::FedoraFetcher.new(@noid))
-    end
+    # Main object metadata
+    object_downloader = PushmiPullyu::AIP::FedoraFetcher.new(@noid)
+    download_and_log(object_aip_paths[:main_object], object_downloader)
 
-    # Need content filename from metadata
-    path_spec = OpenStruct.new(
-      remote: '/content',
-      local: content_filename, # lookup filename derived from metadata
-      optional: false
-    )
-    download_and_log(path_spec, PushmiPullyu::AIP::FedoraFetcher.new(@noid))
+    # Construct the file ordering file
+    list_source_uri = object_downloader.object_url + object_aip_paths.list_source.remote
+    create_and_log_file_order_list(list_source_uri)
+
+    member_file_set_uuids.each do |file_set_uuid|
+      make_file_set_directories(file_set_uuid)
+
+      # FileSet metadata
+      file_set_downloader = PushmiPullyu::AIP::FedoraFetcher.new(file_set_uuid)
+      path_spec = file_set_aip_paths(file_set_uuid)[:main_object]
+      download_and_log(path_spec, file_set_downloader)
+
+      # Find the original file by looping through the files in the file_set
+      original_file_remote_base = nil
+      member_files(file_set_uuid).each do |file_path|
+        path_spec = OpenStruct.new(
+          remote: "/files/#{file_path}/fcr:metadata",
+          # Note: local file gets clobbered on each download until it finds the right one
+          local: "#{file_set_dirs(file_set_uuid).metadata}/original_file_metadata.n3",
+          optional: true
+        )
+        download_and_log(path_spec, file_set_downloader)
+        if original_file?(path_spec.local)
+          original_file_remote_base = "/files/#{file_path}"
+          break
+        end
+      end
 
-    download_permissions
+      raise NoOriginalFile unless original_file_remote_base.present?
+
+      [:content, :fixity].each do |item|
+        path_spec = file_aip_paths(file_set_uuid, original_file_remote_base)[item]
+        download_and_log(path_spec, file_set_downloader)
+      end
+    end
   end
 
   private
@@ -43,35 +78,24 @@ class PushmiPullyu::AIP::Downloader
 
     log_fetching(fedora_fetcher.object_url(path_spec.remote), output_file)
 
-    is_rdf = (output_file !~ /\.n3$/)
+    is_rdf = (output_file =~ /\.n3$/)
+    should_add_user_email = path_spec.to_h.fetch(:should_add_user_email, false)
 
     is_success = fedora_fetcher.download_object(output_file,
                                                 url_extra: path_spec.remote,
                                                 optional: path_spec.optional,
-                                                is_rdf: is_rdf)
+                                                is_rdf: is_rdf,
+                                                should_add_user_email: should_add_user_email)
     log_saved(is_success, output_file)
   end
 
-  def download_permissions
-    PushmiPullyu.logger.info("#{@noid}: looking up permissions from Solr ...")
-    results = PushmiPullyu::AIP::SolrFetcher.new(@noid).fetch_permission_object_ids
-    if results.empty?
-      PushmiPullyu.logger.info("#{@noid}: permissions not found")
-    else
-      results.each do |permission_id|
-        PushmiPullyu.logger.info("#{@noid}: permission object #{permission_id} found")
-        download_permission(permission_id)
-      end
-    end
-  end
-
-  def download_permission(permission_id)
-    path_spec = OpenStruct.new(
-      remote: nil,
-      local: "#{aip_dirs.metadata}/permission_#{permission_id}.n3",
-      optional: false
-    )
-    download_and_log(path_spec, PushmiPullyu::AIP::FedoraFetcher.new(permission_id))
+  def create_and_log_file_order_list(url)
+    output_file = object_aip_paths.file_ordering.local
+    PushmiPullyu::Logging.log_aip_activity(@aip_directory,
+                                           "#{@noid}: #{output_file} -- creating from #{url} ...")
+    PushmiPullyu::AIP::FileListCreator.new(url, output_file, member_file_set_uuids).run
+    PushmiPullyu::Logging.log_aip_activity(@aip_directory,
+                                           "#{@noid}: #{output_file} -- created")
   end
 
   ### Logging
@@ -92,8 +116,19 @@ class PushmiPullyu::AIP::Downloader
     @aip_dirs ||= OpenStruct.new(
       objects: "#{@aip_directory}/data/objects",
       metadata: "#{@aip_directory}/data/objects/metadata",
+      files: "#{@aip_directory}/data/objects/files",
+      files_metadata: "#{@aip_directory}/data/objects/metadata/files_metadata",
       logs: "#{@aip_directory}/data/logs",
-      thumbnails: "#{@aip_directory}/data/thumbnails"
+      file_logs: "#{@aip_directory}/data/logs/files_logs"
+    )
+  end
+
+  def file_set_dirs(file_set_uuid)
+    @file_set_dirs ||= {}
+    @file_set_dirs[file_set_uuid] ||= OpenStruct.new(
+      metadata: "#{aip_dirs.files_metadata}/#{file_set_uuid}",
+      files: "#{aip_dirs.files}/#{file_set_uuid}",
+      logs: "#{aip_dirs.file_logs}/#{file_set_uuid}"
     )
   end
 
@@ -106,6 +141,14 @@ class PushmiPullyu::AIP::Downloader
     PushmiPullyu.logger.debug("#{@noid}: Creating directories done")
   end
 
+  def make_file_set_directories(file_set_uuid)
+    PushmiPullyu.logger.debug("#{@noid}: Creating file set #{file_set_uuid} directories ...")
+    file_set_dirs(file_set_uuid).to_h.each_value do |path|
+      FileUtils.mkdir_p(path)
+    end
+    PushmiPullyu.logger.debug("#{@noid}: Creating file set #{file_set_uuid} directories done")
+  end
+
   def clean_directories
     return unless File.exist?(@aip_directory)
     PushmiPullyu.logger.debug("#{@noid}: Nuking directories ...")
@@ -114,64 +157,105 @@ class PushmiPullyu::AIP::Downloader
 
   ### Files
 
-  def aip_paths
-    @aip_paths ||= OpenStruct.new(
+  def object_aip_paths
+    @object_aip_paths ||= OpenStruct.new(
       main_object: OpenStruct.new(
         remote: nil, # Base path
         local: "#{aip_dirs.metadata}/object_metadata.n3",
+        should_add_user_email: true,
         optional: false
       ),
-      fixity: OpenStruct.new(
-        remote: '/content/fcr:fixity',
-        local: "#{aip_dirs.logs}/content_fixity_report.n3",
-        optional: false
+      list_source: OpenStruct.new(
+        # This is downloaded, but not saved
+        remote: '/list_source'
       ),
-      content_datastream_metadata: OpenStruct.new(
-        remote: '/content/fcr:metadata',
-        local: "#{aip_dirs.metadata}/content_fcr_metadata.n3",
-        optional: false
-      ),
-      versions: OpenStruct.new(
-        remote: '/content/fcr:versions',
-        local: "#{aip_dirs.metadata}/content_versions.n3",
+      # This is constructed, not downloaded
+      file_ordering: OpenStruct.new(
+        local: "#{aip_dirs.files_metadata}/file_order.xml"
+      )
+    ).freeze
+  end
+
+  def file_set_aip_paths(file_set_uuid)
+    @file_set_aip_paths ||= {}
+    @file_set_aip_paths[file_set_uuid] ||= OpenStruct.new(
+      main_object: OpenStruct.new(
+        remote: nil, # Base file_set path
+        local: "#{file_set_dirs(file_set_uuid).metadata}/file_set_metadata.n3",
+        should_add_user_email: true,
         optional: false
-      ),
+      )
+    ).freeze
+  end
 
-      # Optional downloads
-      thumbnail: OpenStruct.new(
-        remote: '/thumbnail',
-        local: "#{aip_dirs.thumbnails}/thumbnail",
-        optional: true
-      ),
-      characterization: OpenStruct.new(
-        remote: '/characterization',
-        local: "#{aip_dirs.logs}/content_characterization.n3",
-        optional: true
-      ),
-      fedora3foxml: OpenStruct.new(
-        remote: '/fedora3foxml',
-        local: "#{aip_dirs.metadata}/fedora3foxml.xml",
-        optional: true
+  def file_aip_paths(file_set_uuid, original_file_remote_base)
+    @file_aip_paths ||= {}
+    @file_aip_paths[file_set_uuid] ||= OpenStruct.new(
+      content: OpenStruct.new(
+        remote: original_file_remote_base,
+        local: file_set_filename(file_set_uuid),
+        optional: false
       ),
-      fedora3foxml_metadata: OpenStruct.new(
-        remote: '/fedora3foxml/fcr:metadata',
-        local: "#{aip_dirs.metadata}/fedora3foxml.n3",
-        optional: true
+      fixity: OpenStruct.new(
+        remote: "#{original_file_remote_base}/fcr:fixity",
+        local: "#{file_set_dirs(file_set_uuid)[:logs]}/content_fixity_report.n3",
+        optional: false
       )
     ).freeze
   end
 
-  # Extract filename from main object metadata
-  def content_filename
-    filename_predicate = RDF::URI('info:fedora/fedora-system:def/model#downloadFilename')
+  def member_file_set_uuids
+    @member_file_set_uuids ||= []
+    return @member_file_set_uuids unless @member_file_set_uuids.empty?
+
+    member_file_set_predicate = RDF::URI(PREDICATE_URIS[:member_file_sets])
+
+    graph = RDF::Graph.load(object_aip_paths.main_object.local)
+
+    graph.query(predicate: member_file_set_predicate) do |results|
+      # Get uuid from end of fedora path
+      @member_file_set_uuids << results.object.to_s.split('/').last
+    end
+    return @member_file_set_uuids unless @member_file_set_uuids.empty?
+
+    raise NoFileSets
+  end
+
+  def file_set_filename(file_set_uuid)
+    filename_predicate = RDF::URI(PREDICATE_URIS[:filename])
 
-    graph = RDF::Graph.load(aip_paths.main_object.local)
+    graph = RDF::Graph.load(file_set_aip_paths(file_set_uuid).main_object.local)
 
     graph.query(predicate: filename_predicate) do |results|
-      return "#{aip_dirs.objects}/#{results.object}"
+      return "#{file_set_dirs(file_set_uuid).files}/#{results.object}"
     end
 
     raise NoContentFilename
   end
 
+  def member_files(file_set_uuid)
+    member_file_predicate = RDF::URI(PREDICATE_URIS[:member_files])
+
+    graph = RDF::Graph.load(file_set_aip_paths(file_set_uuid).main_object.local)
+
+    member_files = []
+    graph.query(predicate: member_file_predicate) do |results|
+      # Get uuid from end of fedora path
+      member_files << results.object.to_s.split('/').last
+    end
+    return member_files if member_files.present?
+
+    raise NoMemberFiles
+  end
+
+  def original_file?(metadata_filename)
+    type_predicate = RDF::URI(PREDICATE_URIS[:type])
+    original_file_uri = RDF::URI(PREDICATE_URIS[:original_file])
+    graph = RDF::Graph.load(metadata_filename)
+    graph.query(predicate: type_predicate) do |results|
+      return true if results.object == original_file_uri
+    end
+    false
+  end
+
 end

data/lib/pushmi_pullyu/aip/fedora_fetcher.rb

@@ -19,7 +19,8 @@ class PushmiPullyu::AIP::FedoraFetcher
   # Return true on success, raise an error otherwise
   # (or use 'optional' to return false on 404)
   def download_object(download_path, url_extra: nil,
-                      optional: false, is_rdf: false)
+                      optional: false, is_rdf: false,
+                      should_add_user_email: false)
 
     uri = URI(object_url(url_extra))
 
@@ -34,8 +35,13 @@ class PushmiPullyu::AIP::FedoraFetcher
     end
 
     if response.is_a?(Net::HTTPSuccess)
+      body = if should_add_user_email
+               PushmiPullyu::AIP::OwnerEmailEditor.new(response.body).run
+             else
+               response.body
+             end
       file = File.open(download_path, 'wb')
-      file.write(response.body)
+      file.write(body)
       file.close
       return true
     elsif response.is_a?(Net::HTTPNotFound)

data/lib/pushmi_pullyu/aip/file_list_creator.rb

@@ -0,0 +1,115 @@
+require 'rdf'
+require 'rdf/n3'
+
+class PushmiPullyu::AIP::FileListCreator
+
+  IANA = 'http://www.iana.org/assignments/relation/'.freeze
+  PREDICATES = {
+    proxy_for: RDF::URI('http://www.openarchives.org/ore/terms/proxyFor'),
+    first: RDF::URI(IANA + 'first'),
+    last: RDF::URI(IANA + 'last'),
+    prev: RDF::URI(IANA + 'prev'),
+    next: RDF::URI(IANA + 'next'),
+    has_part: RDF::URI('http://purl.org/dc/terms/hasPart')
+  }.freeze
+
+  class NoProxyURIFound < StandardError; end
+  class NoFirstProxyFound < StandardError; end
+  class FirstProxyHasPrev < StandardError; end
+  class ListSourceFileSetMismatch < StandardError; end
+
+  def initialize(list_source_uri, output_xml_file, file_set_uuids)
+    @uri = RDF::URI(list_source_uri)
+    @output_file = output_xml_file
+
+    # These are the known fileset uuids, used for validation
+    @file_set_uuids = file_set_uuids
+  end
+
+  def run
+    extract_list_source_uuids
+    raise ListSourceFileSetMismatch, @uri.to_s if @list_source_uuids.sort != @file_set_uuids.sort
+
+    write_output_file
+  end
+
+  def extract_list_source_uuids
+    # Note: raises IOError if can't find
+    #       raises RDF::ReaderError if can't parse
+    @graph = RDF::Graph.load(@uri, validate: true)
+
+    @list_source_uuids = []
+
+    # Fetch first FileSet in list source
+    this_proxy = find_first_proxy
+
+    while @list_source_uuids.count <= num_proxies
+      @list_source_uuids << uuid_from_proxy(this_proxy)
+      next_proxy = find_next_proxy(this_proxy)
+
+      break if next_proxy.nil?
+
+      raise NextPreviousProxyMismatch if this_proxy != find_prev_proxy(next_proxy)
+      this_proxy = next_proxy
+    end
+
+    raise ProxyCountIncorrect if @list_source_uuids.count != num_proxies
+    raise LastProxyFailsValidation if this_proxy != find_last_proxy
+  end
+
+  def num_proxies
+    @num_proxies ||= @graph.query(subject: @uri, predicate: PREDICATES[:has_part]).count
+  end
+
+  def uuid_from_proxy(proxy_uri)
+    @graph.query(subject: proxy_uri, predicate: PREDICATES[:proxy_for]) do |statement|
+      return statement.object.to_s.split('/').last
+    end
+    raise NoProxyURIFound, proxy_uri.to_s
+  end
+
+  def find_first_proxy
+    @graph.query(subject: @uri, predicate: PREDICATES[:first]) do |statement|
+      first_uri = statement.object
+      # Validate that the first proxy doesn't have a previous one
+      raise FirstProxyHasPrev, @uri.to_s if find_prev_proxy(first_uri)
+      return first_uri
+    end
+    raise NoFirstProxyFound, @uri.to_s
+  end
+
+  def find_last_proxy
+    @graph.query(subject: @uri, predicate: PREDICATES[:last]) do |statement|
+      last_uri = statement.object
+      # Validate that the last proxy doesn't have a next one
+      raise LastProxyHasNext, @uri.to_s if find_next_proxy(last_uri)
+      return last_uri
+    end
+    raise LastProxyFound, @uri.to_s
+  end
+
+  def find_next_proxy(proxy_uri)
+    @graph.query(subject: proxy_uri, predicate: PREDICATES[:next]) do |statement|
+      return statement.object
+    end
+    nil
+  end
+
+  def find_prev_proxy(proxy_uri)
+    @graph.query(subject: proxy_uri, predicate: PREDICATES[:prev]) do |statement|
+      return statement.object
+    end
+    nil
+  end
+
+  def write_output_file
+    File.open(@output_file, 'w') do |file|
+      file.write("<file_order>\n")
+      @list_source_uuids.each do |uuid|
+        file.write("  <uuid>#{uuid}</uuid>\n")
+      end
+      file.write("</file_order>\n")
+    end
+  end
+
+end

data/lib/pushmi_pullyu/aip/owner_email_editor.rb

@@ -0,0 +1,62 @@
+require 'net/http'
+
+class PushmiPullyu::AIP::OwnerEmailEditor
+
+  OWNER_PREDICATE = RDF::URI('http://purl.org/ontology/bibo/owner').freeze
+
+  class NoOwnerPredicate < StandardError; end
+
+  def initialize(rdf_string)
+    @document = rdf_string
+  end
+
+  def run
+    ensure_database_connection
+
+    is_modified = false
+    prefixes = nil
+    # Read once to load prefixes (the @things at the top of an n3 file)
+    RDF::N3::Reader.new(input = @document) do |reader|
+      reader.each_statement { |_statement| }
+      prefixes = reader.prefixes
+    end
+    new_body = RDF::N3::Writer.buffer(prefixes: prefixes) do |writer|
+      RDF::N3::Reader.new(input = @document) do |reader|
+        reader.each_statement do |statement|
+          if statement.predicate == OWNER_PREDICATE
+            user = PushmiPullyu::AIP::User.find(statement.object.to_i)
+            writer << [statement.subject, statement.predicate, user.email]
+            is_modified = true
+          else
+            writer << statement
+          end
+        end
+      end
+    end
+    return new_body if is_modified
+    raise NoOwnerPredicate
+  end
+
+  private
+
+  def ensure_database_connection
+    return if ActiveRecord::Base.connected?
+    ActiveRecord::Base.establish_connection(database_configuration)
+  end
+
+  def database_configuration
+    # Config either from URL, or with more granular options (the later taking precedence)
+    config = {}
+    uri = URI.parse(PushmiPullyu.options[:database][:url])
+    config[:adapter] = PushmiPullyu.options[:database][:adaptor] || uri.scheme
+    config[:host] = PushmiPullyu.options[:database][:host] || uri.host
+    config[:database] = PushmiPullyu.options[:database][:database] || uri.path.split('/')[1].to_s
+    config[:username] = PushmiPullyu.options[:database][:username] || uri.user
+    config[:password] = PushmiPullyu.options[:database][:password] || uri.password
+    params = CGI.parse(uri.query || '')
+    config[:encoding] = PushmiPullyu.options[:database][:encoding] || params['encoding'].to_a.first
+    config[:pool] = PushmiPullyu.options[:database][:pool] || params['pool'].to_a.first
+    config
+  end
+
+end

data/lib/pushmi_pullyu/aip/user.rb

@@ -0,0 +1,2 @@
+class PushmiPullyu::AIP::User < ActiveRecord::Base
+end

data/lib/pushmi_pullyu/aip.rb

@@ -2,6 +2,7 @@ require 'fileutils'
 
 module PushmiPullyu::AIP
   class NoidInvalid < StandardError; end
+  module_function
 
   def create(noid)
     raise NoidInvalid if noid.blank? || noid.include?('/')
@@ -12,11 +13,9 @@ module PushmiPullyu::AIP
     PushmiPullyu::AIP::Downloader.new(noid, aip_directory).run
     PushmiPullyu::AIP::Creator.new(noid, aip_directory, aip_filename).run
 
-    yield aip_filename
+    yield aip_filename, aip_directory
 
     FileUtils.rm_rf(aip_filename) if File.exist?(aip_filename)
     FileUtils.rm_rf(aip_directory) if File.exist?(aip_directory)
   end
-
-  module_function :create
 end

data/lib/pushmi_pullyu/cli.rb

@@ -41,6 +41,7 @@ class PushmiPullyu::CLI
       Rollbar.error(e)
       raise e
     end
+    # rubocop:enable Lint/RescueException
   end
 
   def start_server
@@ -73,12 +74,11 @@ class PushmiPullyu::CLI
   end
 
   def parse_config(config_file)
-    opts = {}
     if File.exist?(config_file)
-      opts = YAML.safe_load(ERB.new(IO.read(config_file)).result).deep_symbolize_keys || opts
+      YAML.safe_load(ERB.new(IO.read(config_file)).result).deep_symbolize_keys || {}
+    else
+      {}
     end
-
-    opts
   end
 
   # Parse the options.
@@ -174,20 +174,20 @@ class PushmiPullyu::CLI
     Rollbar.scoped(noid: item) do
       begin
         # Download AIP from Fedora, bag and tar AIP directory and cleanup after block code
-        PushmiPullyu::AIP.create(item) do |aip_filename|
+        PushmiPullyu::AIP.create(item) do |aip_filename, aip_directory|
           # Push tarred AIP to swift API
           deposited_file = swift.deposit_file(aip_filename, options[:swift][:container])
           # Log successful preservation event to the log files
-          PushmiPullyu::Logging.log_preservation_event(deposited_file)
+          PushmiPullyu::Logging.log_preservation_event(deposited_file, aip_directory)
         end
-      # rubocop:disable RescueWithoutErrorClass
-      rescue => e
+      # rubocop:disable Lint/RescueException
+      rescue Exception => e
         Rollbar.error(e)
         logger.error(e)
         # TODO: we could re-raise here and let the daemon die on any preservation error, or just log the issue and
         # move on to the next item.
       end
-      # rubocop:enaable RescueWithoutErrorClass
+      # rubocop:enable Lint/RescueException
     end
   end
 

data/lib/pushmi_pullyu/logging.rb

@@ -13,58 +13,96 @@ module PushmiPullyu::Logging
 
   end
 
-  def self.initialize_logger(log_target = STDOUT)
-    @logger = Logger.new(log_target)
-    @logger.level = Logger::INFO
-    @logger
+  def logger
+    PushmiPullyu::Logging.logger
   end
 
-  def self.logger
-    @logger ||= initialize_logger
-  end
+  class << self
 
-  def self.log_aip_activity(aip_directory, message)
-    log_file = "#{aip_directory}/data/logs/aipcreation.log"
-    aip_logger = Logger.new(log_file)
-    aip_logger.level = logger.level
+    attr_writer :logger
 
-    # Log to both the application log, and the log file that gets archived in the AIP
-    logger.info(message)
-    aip_logger.info(message)
+    def initialize_logger(log_target = STDOUT)
+      @logger = Logger.new(log_target)
+      @logger.level = Logger::INFO
+      @logger
+    end
 
-    aip_logger.close
-  end
+    def logger
+      @logger ||= initialize_logger
+    end
 
-  def self.log_preservation_event(deposited_file)
-    preservation_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.log")
+    def log_aip_activity(aip_directory, message)
+      log_file = "#{aip_directory}/data/logs/aipcreation.log"
+      aip_logger = Logger.new(log_file)
+      aip_logger.level = logger.level
 
-    message = "#{deposited_file.name} was successfully deposited into Swift Storage! \n"\
-    "Here are the details of this preservation event: \n"\
-    "\t NOID: '#{deposited_file.name}' \n"\
-    "\t Timestamp of Completion: '#{deposited_file.last_modified}' \n"\
-    "\t AIP Checksum: '#{deposited_file.etag}' \n"\
-    "\t Metadata: #{deposited_file.metadata} \n"
+      # Log to both the application log, and the log file that gets archived in the AIP
+      logger.info(message)
+      aip_logger.info(message)
 
-    # Log to both the application log, and the preservation log file
-    logger.info(message)
-    preservation_logger.info(message)
+      aip_logger.close
+    end
 
-    preservation_logger.close
-  end
+    def log_preservation_event(deposited_file, aip_directory)
+      preservation_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.log")
 
-  def self.logger=(log)
-    @logger = log
-  end
+      message = "#{deposited_file.name} was successfully deposited into Swift Storage!\n"\
+      "Here are the details of this preservation event:\n"\
+      "\tNOID: '#{deposited_file.name}'\n"\
+      "\tTimestamp of Completion: '#{deposited_file.last_modified}'\n"\
+      "\tAIP Checksum: '#{deposited_file.etag}'\n"\
+      "\tMetadata: #{deposited_file.metadata}\n"\
+
+      file_details = file_log_details(aip_directory)
+
+      if file_details.present?
+        message << "\tFile Details:\n"
+        file_details.each do |file_detail|
+          message << %(\t\t{"fileset_uuid": "#{file_detail[:fileset_name]}",
+\t\t"details": {
+\t\t\t"file_name": "#{file_detail[:file_name]}",
+\t\t\t"file_type": "#{file_detail[:file_extension]}",
+\t\t\t"file_size": #{file_detail[:file_size]}
+\t\t}}\n)
+        end
+      end
 
-  def self.reopen
-    if @logger
-      @logger.reopen
-    else
-      @logger = initialize_logger
+      # Log to both the application log, and the preservation log file
+      logger.info(message)
+      preservation_logger.info(message)
+
+      preservation_logger.close
+    end
+
+    def reopen
+      if @logger
+        @logger.reopen
+      else
+        @logger = initialize_logger
+      end
+    end
+
+    private
+
+    def file_log_details(aip_directory)
+      file_details = []
+      data_files_location = "#{aip_directory}/data/objects/files"
+
+      if Dir.exist?(data_files_location)
+        Dir.glob("#{data_files_location}/*") do |folder|
+          Dir.glob("#{folder}/*") do |file|
+            file_details << {
+              fileset_name: File.dirname(file).split('/')[-1],
+              file_name: File.basename(file),
+              file_size: File.size(file),
+              file_extension: File.extname(file).strip.downcase[1..-1]
+            }
+          end
+        end
+      end
+
+      file_details
     end
-  end
 
-  def logger
-    PushmiPullyu::Logging.logger
   end
 end

data/lib/pushmi_pullyu/version.rb

@@ -1,3 +1,3 @@
 module PushmiPullyu
-  VERSION = '0.2.7'.freeze
+  VERSION = '1.0.1'.freeze
 end

data/lib/pushmi_pullyu.rb

@@ -8,8 +8,11 @@ require 'pushmi_pullyu/logging'
 require 'pushmi_pullyu/aip'
 require 'pushmi_pullyu/aip/creator'
 require 'pushmi_pullyu/aip/downloader'
-require 'pushmi_pullyu/aip/solr_fetcher'
 require 'pushmi_pullyu/aip/fedora_fetcher'
+require 'pushmi_pullyu/aip/file_list_creator'
+require 'pushmi_pullyu/aip/owner_email_editor'
+require 'active_record'
+require 'pushmi_pullyu/aip/user'
 require 'pushmi_pullyu/cli'
 require 'pushmi_pullyu/preservation_queue'
 require 'pushmi_pullyu/swift_depositer'
@@ -20,6 +23,7 @@ require 'active_support/core_ext'
 # PushmiPullyu main module
 module PushmiPullyu
   DEFAULTS = {
+    aip_version: 'lightaip-2.0',
     daemonize: false,
     debug: false,
     logdir: 'log',
@@ -32,12 +36,8 @@ module PushmiPullyu
     redis: {
       url: 'redis://localhost:6379'
     },
-    # TODO: rest of these are examples for solr/fedora/swift... feel free to fill them in correctly
-    solr: {
-      url: 'http://localhost:8983/solr/development'
-    },
     fedora: {
-      url: 'http://localhost:8983/fedora/rest',
+      url: 'http://localhost:8080/fcrepo/rest',
       user: 'fedoraAdmin',
       password: 'fedoraAdmin',
       base_path: '/dev'
@@ -52,6 +52,12 @@ module PushmiPullyu
       container: 'ERA'
     },
     rollbar: {
+    },
+    database: {
+      encoding: 'utf8',
+      pool: ENV['RAILS_MAX_THREADS'] || 5,
+      url: ENV['DATABASE_URL'] || ENV['JUPITER_DATABASE_URL'] || 'postgresql://jupiter:mysecretpassword@127.0.0.1',
+      database: 'jupiter_development'
     }
   }.freeze
 

data/pushmi_pullyu.gemspec

@@ -1,4 +1,4 @@
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'pushmi_pullyu/version'
 
@@ -21,15 +21,17 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '>= 2.3.1'
 
+  spec.add_runtime_dependency 'activerecord', '>= 5.1.5', '< 5.3.0'
   spec.add_runtime_dependency 'activesupport', '~> 5.0'
   spec.add_runtime_dependency 'bagit', '~> 0.4'
   spec.add_runtime_dependency 'connection_pool', '~> 2.2'
   spec.add_runtime_dependency 'daemons', '~> 1.2', '>= 1.2.4'
   spec.add_runtime_dependency 'minitar', '~> 0.6'
   spec.add_runtime_dependency 'openstack', '~> 3.3', '>= 3.3.10'
-  spec.add_runtime_dependency 'rdf', '~> 1.99'
-  spec.add_runtime_dependency 'rdf-n3', '~> 1.99'
-  spec.add_runtime_dependency 'redis', '~> 3.3'
+  spec.add_runtime_dependency 'pg', '~> 1.0.0'
+  spec.add_runtime_dependency 'rdf', '>= 1.99', '< 4.0'
+  spec.add_runtime_dependency 'rdf-n3', '>= 1.99', '< 4.0'
+  spec.add_runtime_dependency 'redis', '>= 3.3', '< 5.0'
   spec.add_runtime_dependency 'rollbar', '~> 2.14'
 
   spec.add_development_dependency 'bundler', '~> 1.14'
@@ -38,9 +40,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'pry', '~> 0.10', '>= 0.10.4'
   spec.add_development_dependency 'rake', '~> 12.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'rubocop', '~> 0.45'
+  spec.add_development_dependency 'rubocop', '~> 0.51'
   spec.add_development_dependency 'rubocop-rspec', '~> 1.10'
   spec.add_development_dependency 'timecop', '~> 0.8'
-  spec.add_development_dependency 'vcr', '~> 3.0'
-  spec.add_development_dependency 'webmock', '~> 2.1'
+  spec.add_development_dependency 'vcr', '~> 4.0'
+  spec.add_development_dependency 'webmock', '~> 3.3'
 end

metadata

@@ -1,15 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: pushmi_pullyu
 version: !ruby/object:Gem::Version
-  version: 0.2.7
+  version: 1.0.1
 platform: ruby
 authors:
 - Shane Murnaghan
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-11-07 00:00:00.000000000 Z
+date: 2018-07-26 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.1.5
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 5.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.1.5
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 5.3.0
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -107,47 +127,79 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 3.3.10
 - !ruby/object:Gem::Dependency
-  name: rdf
+  name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.99'
+        version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: rdf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.99'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.99'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: rdf-n3
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.99'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.99'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '3.3'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '3.3'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: rollbar
   requirement: !ruby/object:Gem::Requirement
@@ -258,14 +310,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.45'
+        version: '0.51'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.45'
+        version: '0.51'
 - !ruby/object:Gem::Dependency
   name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
@@ -300,28 +352,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '4.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '3.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '3.3'
 description: 
 email:
 - murnagha@ualberta.ca
@@ -354,7 +406,9 @@ files:
 - lib/pushmi_pullyu/aip/creator.rb
 - lib/pushmi_pullyu/aip/downloader.rb
 - lib/pushmi_pullyu/aip/fedora_fetcher.rb
-- lib/pushmi_pullyu/aip/solr_fetcher.rb
+- lib/pushmi_pullyu/aip/file_list_creator.rb
+- lib/pushmi_pullyu/aip/owner_email_editor.rb
+- lib/pushmi_pullyu/aip/user.rb
 - lib/pushmi_pullyu/cli.rb
 - lib/pushmi_pullyu/logging.rb
 - lib/pushmi_pullyu/preservation_queue.rb
@@ -384,7 +438,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Ruby application to manage flow of content from Fedora into Swift for preservation

data/lib/pushmi_pullyu/aip/solr_fetcher.rb

@@ -1,33 +0,0 @@
-require 'json'
-require 'net/http'
-
-class PushmiPullyu::AIP::SolrFetcher
-
-  class SolrFetchError < StandardError; end
-
-  def initialize(noid)
-    @noid = noid
-  end
-
-  def fetch_permission_object_ids
-    hash = JSON.parse(run_query_json)
-
-    return [] if hash['response']['docs'].empty?
-
-    hash['response']['docs'].map { |hit| hit['id'] }
-  end
-
-  private
-
-  # Return fetched results, else raise an error
-  def run_query_json
-    response = Net::HTTP.get_response(
-      URI("#{PushmiPullyu.options[:solr][:url]}/select?q=accessTo_ssim:#{@noid}&fl=id&wt=json")
-    )
-
-    return response.body if response.is_a?(Net::HTTPSuccess)
-
-    raise SolrFetchError
-  end
-
-end