pushmi_pullyu 1.0.3 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: aa937d14979ab2d2816e710c8ab07953161538d0
-  data.tar.gz: 4b941305b25f708ed1cec36d96e0573e73e83d64
+SHA256:
+  metadata.gz: f61b0350971eeec34f815847cae81887b53a67785c137a94199b38ced2ee6c85
+  data.tar.gz: 368ab2f54ddbdf7be31880cfc2ae91b6446245a161c5bf80e232311161768c19
 SHA512:
-  metadata.gz: ba32265c07780fb2bc9e894505cdc44c893c1c61eed805a409263f0f0208410f6313c09624b5bc2afec132b5d4c2203f0e4039ea8cd41f7744d4e2d6ddffa757
-  data.tar.gz: 9f5ba89beb76ce3102a2ebf232021d6f6d401553e46c310e79c0f8956f02855a79392b7b1831fba8b0b246bce2f9eaf463659244032ab09741ea5b7f6423c61c
+  metadata.gz: 88b77196bd1a8a9d4e7bc98857d1896541d8e26743df9edcf24ad69f458e290f50c3ff6ef89ae667c17a56b1bbf479b9b4ca04c5e7e942b7cecd2cd5485c737a
+  data.tar.gz: eef26b3d1c4bbebe0053a4097d5bafb2fa47eb275ce1bcd8f64179d6d1a8f2e785da89837d7a47197504d57ab274ac1147dde5077770be47499b2b179a306fc1

data/.github/CODEOWNERS

@@ -0,0 +1,2 @@
+# https://help.github.com/articles/about-codeowners/
+* @ConnorSheremeta @lagoan @mbarnett @murny @pgwillia @piyapongch

data/.github/PULL_REQUEST_TEMPLATE

@@ -0,0 +1,11 @@
+## Context
+
+Please describe any relevant motivation and context for these changes.
+
+Related to # (issue)
+
+## What's New
+
+Please summarize what's new or changed in this PR.
+
+Remember to request a reviewer when you submit!

data/.github/workflows/ruby.yml

@@ -0,0 +1,31 @@
+name: CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis:alpine
+        ports: ["6379:6379"]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.6.6
+    - name: Install dependencies
+      run: bundle install
+    - name: Lint with RuboCop
+      run: bundle exec rubocop --parallel
+    - name: Run Danger
+      env:
+        DANGER_GITHUB_API_TOKEN: ${{ secrets.DANGER_GITHUB_API_TOKEN }}
+      run: bundle exec danger
+    - name: Run tests
+      run: bundle exec rake spec

data/.rubocop.yml

@@ -10,8 +10,9 @@ AllCops:
   Exclude:
     - 'tmp/**/*'
     - 'vendor/**/*'
+    - 'Dangerfile'
   ExtraDetails: true
-  TargetRubyVersion: 2.3
+  TargetRubyVersion: 2.5
 
 # readability is Actually Good
 Layout/EmptyLinesAroundClassBody:
@@ -59,6 +60,15 @@ Style/ClassAndModuleChildren:
 Style/Documentation:
   Enabled: false
 
+Style/HashEachMethods:
+  Enabled: true
+
+Style/HashTransformKeys:
+  Enabled: true
+
+Style/HashTransformValues:
+  Enabled: true
+
 Naming/FileName:
   Exclude:
     - Dangerfile
@@ -88,3 +98,12 @@ RSpec/MultipleExpectations:
 
 RSpec/DescribedClass:
   EnforcedStyle: explicit
+
+Lint/RaiseException:
+  Enabled: true
+
+Lint/StructNewOverride:
+  Enabled: true
+
+RSpec/MultipleMemoizedHelpers:
+  Enabled: false

data/.travis.yml

@@ -9,8 +9,8 @@ before_install:
   - gem update bundler
 
 rvm:
-  - 2.3.3
-  - 2.4.0
+  - 2.5.0
+  - 2.6.0
 
 before_script:
   - bundle exec danger

data/CHANGELOG.md

@@ -0,0 +1,18 @@
+# Changelog
+All notable changes to PushmiPullyu project will be documented in this file. 
+
+PushmiPullyu is a Ruby application, whose primary job is to manage the flow of content from [Jupiter](https://github.com/ualbertalib/jupiter/) into Swift for preservation.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and releases in PushmiPullyu adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+- Fix dependency declaration for UUID gem
+
+## [2.0.0] - 2020-12-14
+
+### Removed
+- Data output for original_file information
+
+## [1.0.6] - 2018-11-29

data/Dangerfile

@@ -13,9 +13,13 @@ end
 # just leaving a title
 warn('Please add a detailed summary in the description.') if github.pr_body.length < 5
 
-# Ensure a clean commits history
-if git.commits.any? { |c| c.message =~ /^Merge branch '#{github.branch_for_base}'/ }
-  warn 'Please rebase to get rid of the merge commits in this PR'
+# Let people say that this isn't worth a CHANGELOG entry in the PR if they choose
+declared_trivial = (github.pr_title + github.pr_body).include?('#trivial') || !has_app_changes
+
+if !git.modified_files.include?('CHANGELOG.md') && !declared_trivial
+  error_message = "Please include a CHANGELOG entry. \nYou can find it at " \
+                  '[CHANGELOG.md](https://github.com/ualbertalib/pushmi_pullyu/blob/master/CHANGELOG.md).'
+  fail(error_message, sticky: false)
 end
 
 # Warn when there is a big PR

data/README.md

@@ -5,22 +5,21 @@
 </p>
 
 [![Gem Version](https://badge.fury.io/rb/pushmi_pullyu.svg)](https://rubygems.org/gems/pushmi_pullyu)
-[![Dependency Status](https://gemnasium.com/badges/github.com/ualbertalib/pushmi_pullyu.svg)](https://gemnasium.com/github.com/ualbertalib/pushmi_pullyu)
-[![Build Status](https://travis-ci.org/ualbertalib/pushmi_pullyu.svg?branch=master)](https://travis-ci.org/ualbertalib/pushmi_pullyu)
+[![Github Build Status](https://github.com/ualbertalib/pushmi_pullyu/workflows/CI/badge.svg)](https://github.com/ualbertalib/pushmi_pullyu/actions)
 [![Coverage Status](https://coveralls.io/repos/github/ualbertalib/pushmi_pullyu/badge.svg?branch=master)](https://coveralls.io/github/ualbertalib/pushmi_pullyu?branch=master)
 
 PushmiPullyu is a Ruby application, running behind the firewall that protects our Swift environment.
 
-Its primary job is to manage the flow of content from Fedora into Swift for preservation.
+Its primary job is to manage the flow of content from Jupiter into Swift for preservation.
 
 ![System Infrastructure Diagram](docs/images/system-infrastructure-diagram.png)
 
 ## Workflow
 
-1.  Any save (create or update) on a Item/Thesis in ERA/Jupiter will trigger an after save callback that will push the item's unique identifier (UUID or NOID) into a Queue.
+1.  Any save (create or update) on a Item/Thesis in ERA/Jupiter will trigger an after save callback that will push the item's unique identifier (UUID) into a Queue.
 2. The queue (Redis) is setup to be a unique set (which only allows one item's UUID to be included in the queue at a single time), and ordered by priority from First In, First out (FIFO).
 3. PushmiPullyu will then monitor the queue. After a certain wait period has passed since an element has been on the queue, PushmiPullyu will then retrieve the elements off the queue and begin to process the preservation event.
-4. All the GenericFile information and data required for preservation are retrieved from Fedora using multiple REST calls. A database connection to the user database fetches (via ActiveRecord )owner emails and modifies the fetched documents, where applicable.
+4. All the Item/Thesis information and data required for preservation are retrieved from Jupiter using multiple REST calls to Jupiter's AIP API.
 5. An Archival Information Package (AIP) is created from the item's information. It is then bagged and tarred.
 6. The AIP tar is then uploaded to Swift via a REST call.
 7. On a successful Swift upload, a entry is added for this preservation event to the preservation event logs.

data/examples/pushmi_pullyu.yml

@@ -21,12 +21,6 @@ minimum_age: 0
 redis:
   url: redis://localhost:6379
 
-fedora:
-  url: http://localhost:8080/fcrepo/rest
-  user: fedoraAdmin
-  password: fedoraAdmin
-  base_path: /dev
-
 database:
   url: postgresql://jupiter:mysecretpassword@127.0.0.1/jupiter_development
 
@@ -44,3 +38,9 @@ rollbar:
   token: 'abc123xyz'
   proxy_host: 'your_proxy_host_url'
   proxy_port: '80'
+
+jupiter:
+  user: jupiter@ualberta.ca
+  api_key: 5042c4ad-6d22-486d-bc63-2b9e5b9a630a
+  jupiter_url: http://localhost:3000/
+  aip_api_path: aip/v1

data/lib/pushmi_pullyu.rb

@@ -1,22 +1,15 @@
-# require 'pushmi_pullyu/version' must be first as it declares the PushmiPullyu module.
-# (This fixes a weird NameError bug when using the nested compact syntax for
-# defining modules/classes like `module PushmiPullyu::Logging`)
-require 'pushmi_pullyu/version'
+# require 'pushmi_pullyu/version' must be first as it declares the PushmiPullyu
+# module. (This fixes a weird NameError bug when using the nested compact syntax
+# for defining modules/classes like `module PushmiPullyu::Logging`)
 
+require 'pushmi_pullyu/version'
 require 'pushmi_pullyu/logging'
-
 require 'pushmi_pullyu/aip'
 require 'pushmi_pullyu/aip/creator'
 require 'pushmi_pullyu/aip/downloader'
-require 'pushmi_pullyu/aip/fedora_fetcher'
-require 'pushmi_pullyu/aip/file_list_creator'
-require 'pushmi_pullyu/aip/owner_email_editor'
-require 'active_record'
-require 'pushmi_pullyu/aip/user'
 require 'pushmi_pullyu/cli'
 require 'pushmi_pullyu/preservation_queue'
 require 'pushmi_pullyu/swift_depositer'
-
 require 'active_support'
 require 'active_support/core_ext'
 
@@ -36,12 +29,6 @@ module PushmiPullyu
     redis: {
       url: 'redis://localhost:6379'
     },
-    fedora: {
-      url: 'http://localhost:8080/fcrepo/rest',
-      user: 'fedoraAdmin',
-      password: 'fedoraAdmin',
-      base_path: '/dev'
-    },
     swift: {
       tenant: 'tester',
       username: 'test:tester',
@@ -58,6 +45,12 @@ module PushmiPullyu
       pool: ENV['RAILS_MAX_THREADS'] || 5,
       url: ENV['DATABASE_URL'] || ENV['JUPITER_DATABASE_URL'] || 'postgresql://jupiter:mysecretpassword@127.0.0.1',
       database: 'jupiter_development'
+    },
+    jupiter: {
+      user: ENV['JUPITER_USER'],
+      api_key: ENV['JUPITER_API_KEY'],
+      jupiter_url: ENV['JUPITER_URL'] || 'http://localhost:3000/',
+      aip_api_path: ENV['JUPITER_AIP_API_PATH'] || 'aip/v1'
     }
   }.freeze
 

data/lib/pushmi_pullyu/aip.rb

@@ -1,17 +1,20 @@
 require 'fileutils'
+require 'uuid'
 
 module PushmiPullyu::AIP
-  class NoidInvalid < StandardError; end
+  class EntityInvalid < StandardError; end
   module_function
 
-  def create(noid)
-    raise NoidInvalid if noid.blank? || noid.include?('/')
+  def create(entity)
+    raise EntityInvalid if entity.nil? ||
+                           UUID.validate(entity[:uuid]) != true ||
+                           entity[:type].blank?
 
-    aip_directory = "#{PushmiPullyu.options[:workdir]}/#{noid}"
+    aip_directory = "#{PushmiPullyu.options[:workdir]}/#{entity[:uuid]}"
     aip_filename = "#{aip_directory}.tar"
 
-    PushmiPullyu::AIP::Downloader.new(noid, aip_directory).run
-    PushmiPullyu::AIP::Creator.new(noid, aip_directory, aip_filename).run
+    PushmiPullyu::AIP::Downloader.new(entity, aip_directory).run
+    PushmiPullyu::AIP::Creator.new(entity[:uuid], aip_directory, aip_filename).run
 
     yield aip_filename, aip_directory
 

data/lib/pushmi_pullyu/aip/downloader.rb

@@ -2,260 +2,213 @@ require 'fileutils'
 require 'ostruct'
 require 'rdf'
 require 'rdf/n3'
+require 'net/http'
+require 'uri'
+require 'digest'
 
-# Download all of the metadata/datastreams and associated data
-# related to an object
+# Download all of the metadata/datastreams and associated data related to an object
 class PushmiPullyu::AIP::Downloader
 
   PREDICATE_URIS = {
     filename: 'http://purl.org/dc/terms/title',
     member_files: 'http://pcdm.org/models#hasFile',
     member_file_sets: 'http://pcdm.org/models#hasMember',
-    original_file: 'http://pcdm.org/use#OriginalFile',
     type: 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'
   }.freeze
 
-  class NoFileSets < StandardError; end
-  class NoMemberFiles < StandardError; end
-  class NoContentFilename < StandardError; end
-  class NoOriginalFile < StandardError; end
+  class JupiterDownloadError < StandardError; end
+  class JupiterCopyError < StandardError; end
+  class JupiterAuthenticationError < StandardError; end
 
-  def initialize(noid, aip_directory)
-    @noid = noid
+  def initialize(entity, aip_directory)
+    @entity = entity
+    @entity_identifier = "[#{entity[:type]} - #{entity[:uuid]}]".freeze
     @aip_directory = aip_directory
   end
 
   def run
-    make_directories
+    PushmiPullyu.logger.info("#{@entity_identifier}: Retreiving data from Jupiter ...")
 
-    PushmiPullyu.logger.info("#{@noid}: Retreiving data from Fedora ...")
+    authenticate_http_calls
+    make_directories
 
     # Main object metadata
-    object_downloader = PushmiPullyu::AIP::FedoraFetcher.new(@noid)
-    download_and_log(object_aip_paths[:main_object], object_downloader)
-
-    # Construct the file ordering file
-    list_source_uri = object_downloader.object_url + object_aip_paths.list_source.remote
-    create_and_log_file_order_list(list_source_uri)
-
-    member_file_set_uuids.each do |file_set_uuid|
-      make_file_set_directories(file_set_uuid)
-
-      # FileSet metadata
-      file_set_downloader = PushmiPullyu::AIP::FedoraFetcher.new(file_set_uuid)
-      path_spec = file_set_aip_paths(file_set_uuid)[:main_object]
-      download_and_log(path_spec, file_set_downloader)
-
-      # Find the original file by looping through the files in the file_set
-      original_file_remote_base = nil
-      member_files(file_set_uuid).each do |file_path|
-        path_spec = OpenStruct.new(
-          remote: "/files/#{file_path}/fcr:metadata",
-          # Note: local file gets clobbered on each download until it finds the right one
-          local: "#{file_set_dirs(file_set_uuid).metadata}/original_file_metadata.n3",
-          optional: true
-        )
-        download_and_log(path_spec, file_set_downloader)
-        if original_file?(path_spec.local)
-          original_file_remote_base = "/files/#{file_path}"
-          break
-        end
-      end
-
-      raise NoOriginalFile unless original_file_remote_base.present?
-
-      [:content, :fixity].each do |item|
-        path_spec = file_aip_paths(file_set_uuid, original_file_remote_base)[item]
-        download_and_log(path_spec, file_set_downloader)
-      end
+    download_and_log(object_aip_paths[:main_object_remote],
+                     object_aip_paths[:main_object_local])
+    download_and_log(object_aip_paths[:file_sets_remote],
+                     object_aip_paths[:file_sets_local])
+
+    # Get file paths for processing
+    file_paths = get_file_paths(object_aip_paths[:file_paths_remote])
+
+    file_paths[:files].each do |file_path|
+      file_uuid = file_path[:file_uuid]
+      make_file_set_directories(file_uuid)
+      copy_and_log(file_uuid, file_path)
+      file_aip_path = file_aip_paths(file_uuid)
+      download_and_log(file_aip_path[:fixity_remote],
+                       file_aip_path[:fixity_local])
+      download_and_log(file_aip_path[:file_set_remote],
+                       file_aip_path[:file_set_local])
     end
   end
 
   private
 
-  def download_and_log(path_spec, fedora_fetcher)
-    output_file = path_spec.local
-
-    log_fetching(fedora_fetcher.object_url(path_spec.remote), output_file)
+  def copy_and_log(file_uuid, file_path)
+    remote = file_path[:file_path]
+    remote_checksum = file_path[:file_checksum]
+    files_path = file_set_dirs(file_uuid)[:files]
+    output_file = "#{files_path}/#{file_path[:file_name]}"
+    log_downloading(remote, output_file)
+    FileUtils.copy_file(remote, output_file)
 
-    is_rdf = (output_file =~ /\.n3$/)
-    should_add_user_email = path_spec.to_h.fetch(:should_add_user_email, false)
+    is_success = File.exist?(output_file) &&
+                 File.size(remote) == File.size(output_file) &&
+                 compare_md5(output_file, remote_checksum)
 
-    is_success = fedora_fetcher.download_object(output_file,
-                                                url_extra: path_spec.remote,
-                                                optional: path_spec.optional,
-                                                is_rdf: is_rdf,
-                                                should_add_user_email: should_add_user_email)
     log_saved(is_success, output_file)
+
+    raise JupiterCopyError unless is_success
+  end
+
+  def compare_md5(local, remote_checksum)
+    local_md5 = Digest::MD5.file local
+    local_md5.base64digest == remote_checksum
+  end
+
+  def authenticate_http_calls
+    @uri = URI.parse(PushmiPullyu.options[:jupiter][:jupiter_url])
+    @http = Net::HTTP.new(@uri.host, @uri.port)
+    request = Net::HTTP::Post.new(@uri.request_uri + 'auth/system')
+    request.set_form_data(
+      email: PushmiPullyu.options[:jupiter][:user],
+      api_key: PushmiPullyu.options[:jupiter][:api_key]
+    )
+    response = @http.request(request)
+    # If we cannot find the set-cookie header then the session was not set
+    raise JupiterAuthenticationError if response.response['set-cookie'].nil?
+
+    @cookies = response.response['set-cookie']
+  end
+
+  def download_and_log(remote, local)
+    log_downloading(remote, local)
+
+    @uri = URI.parse(PushmiPullyu.options[:jupiter][:jupiter_url])
+    request = Net::HTTP::Get.new(@uri.request_uri + remote)
+    # add previously stored cookies
+    request['Cookie'] = @cookies
+
+    response = @http.request(request)
+    is_success = if response.is_a?(Net::HTTPSuccess)
+                   File.open(local, 'wb') do |file|
+                     file.write(response.body)
+                   end
+                   # Response was a success and the file was saved to local
+                   File.exist? local
+                 end
+
+    log_saved(is_success, local)
+    raise JupiterDownloadError unless is_success
+  end
+
+  def get_file_paths(url)
+    request = Net::HTTP::Get.new(@uri.request_uri + url)
+    # add previously stored cookies
+    request['Cookie'] = @cookies
+
+    response = @http.request(request)
+
+    JSON.parse(response.body, symbolize_names: true)
   end
 
-  def create_and_log_file_order_list(url)
-    output_file = object_aip_paths.file_ordering.local
-    PushmiPullyu::Logging.log_aip_activity(@aip_directory,
-                                           "#{@noid}: #{output_file} -- creating from #{url} ...")
-    PushmiPullyu::AIP::FileListCreator.new(url, output_file, member_file_set_uuids).run
-    PushmiPullyu::Logging.log_aip_activity(@aip_directory,
-                                           "#{@noid}: #{output_file} -- created")
+  def object_uri
+    aip_api_url = PushmiPullyu.options[:jupiter][:aip_api_path]
+    @object_uri ||= "#{aip_api_url}/#{@entity[:type]}/#{@entity[:uuid]}"
   end
 
   ### Logging
 
-  def log_fetching(url, output_file)
-    message = "#{@noid}: #{output_file} -- fetching from #{url} ..."
+  def log_downloading(url, output_file)
+    message = "#{@entity_identifier}: #{output_file} -- Downloading from #{url} ..."
     PushmiPullyu::Logging.log_aip_activity(@aip_directory, message)
   end
 
   def log_saved(is_success, output_file)
-    message = "#{@noid}: #{output_file} -- #{is_success ? 'saved' : 'not_found'}"
+    message = "#{@entity_identifier}: #{output_file} -- #{is_success ? 'Saved' : 'Failed'}"
     PushmiPullyu::Logging.log_aip_activity(@aip_directory, message)
   end
 
   ### Directories
 
   def aip_dirs
-    @aip_dirs ||= OpenStruct.new(
+    @aip_dirs ||= {
       objects: "#{@aip_directory}/data/objects",
       metadata: "#{@aip_directory}/data/objects/metadata",
       files: "#{@aip_directory}/data/objects/files",
       files_metadata: "#{@aip_directory}/data/objects/metadata/files_metadata",
       logs: "#{@aip_directory}/data/logs",
       file_logs: "#{@aip_directory}/data/logs/files_logs"
-    )
+    }
   end
 
   def file_set_dirs(file_set_uuid)
     @file_set_dirs ||= {}
-    @file_set_dirs[file_set_uuid] ||= OpenStruct.new(
-      metadata: "#{aip_dirs.files_metadata}/#{file_set_uuid}",
-      files: "#{aip_dirs.files}/#{file_set_uuid}",
-      logs: "#{aip_dirs.file_logs}/#{file_set_uuid}"
-    )
+    @file_set_dirs[file_set_uuid] ||= {
+      metadata: "#{aip_dirs[:files_metadata]}/#{file_set_uuid}",
+      files: "#{aip_dirs[:files]}/#{file_set_uuid}",
+      logs: "#{aip_dirs[:file_logs]}/#{file_set_uuid}"
+    }
   end
 
   def make_directories
+    PushmiPullyu.logger.debug("#{@entity_identifier}: Creating directories ...")
     clean_directories
-    PushmiPullyu.logger.debug("#{@noid}: Creating directories ...")
-    aip_dirs.to_h.each_value do |path|
+    aip_dirs.each_value do |path|
       FileUtils.mkdir_p(path)
     end
-    PushmiPullyu.logger.debug("#{@noid}: Creating directories done")
+    PushmiPullyu.logger.debug("#{@entity_identifier}: Creating directories done")
   end
 
   def make_file_set_directories(file_set_uuid)
-    PushmiPullyu.logger.debug("#{@noid}: Creating file set #{file_set_uuid} directories ...")
-    file_set_dirs(file_set_uuid).to_h.each_value do |path|
+    PushmiPullyu.logger.debug("#{@entity_identifier}: Creating file set #{file_set_uuid} directories ...")
+    file_set_dirs(file_set_uuid).each_value do |path|
       FileUtils.mkdir_p(path)
     end
-    PushmiPullyu.logger.debug("#{@noid}: Creating file set #{file_set_uuid} directories done")
+    PushmiPullyu.logger.debug("#{@entity_identifier}: Creating file set #{file_set_uuid} directories done")
   end
 
   def clean_directories
     return unless File.exist?(@aip_directory)
-    PushmiPullyu.logger.debug("#{@noid}: Nuking directories ...")
+
+    PushmiPullyu.logger.debug("#{@entity_identifier}: Nuking directories ...")
     FileUtils.rm_rf(@aip_directory)
   end
 
   ### Files
 
   def object_aip_paths
-    @object_aip_paths ||= OpenStruct.new(
-      main_object: OpenStruct.new(
-        remote: nil, # Base path
-        local: "#{aip_dirs.metadata}/object_metadata.n3",
-        should_add_user_email: true,
-        optional: false
-      ),
-      list_source: OpenStruct.new(
-        # This is downloaded, but not saved
-        remote: '/list_source'
-      ),
-      # This is constructed, not downloaded
-      file_ordering: OpenStruct.new(
-        local: "#{aip_dirs.files_metadata}/file_order.xml"
-      )
-    ).freeze
-  end
-
-  def file_set_aip_paths(file_set_uuid)
-    @file_set_aip_paths ||= {}
-    @file_set_aip_paths[file_set_uuid] ||= OpenStruct.new(
-      main_object: OpenStruct.new(
-        remote: nil, # Base file_set path
-        local: "#{file_set_dirs(file_set_uuid).metadata}/file_set_metadata.n3",
-        should_add_user_email: true,
-        optional: false
-      )
-    ).freeze
-  end
-
-  def file_aip_paths(file_set_uuid, original_file_remote_base)
+    @object_aip_paths ||= {
+      # Base path
+      main_object_remote: object_uri,
+      main_object_local: "#{aip_dirs[:metadata]}/object_metadata.n3",
+      file_sets_remote: "#{object_uri}/filesets",
+      file_sets_local: "#{aip_dirs[:files_metadata]}/file_order.xml",
+      # This is downloaded for processing but not saved
+      file_paths_remote: "#{object_uri}/file_paths"
+    }.freeze
+  end
+
+  def file_aip_paths(file_set_uuid)
+    file_set_paths = file_set_dirs(file_set_uuid)
     @file_aip_paths ||= {}
-    @file_aip_paths[file_set_uuid] ||= OpenStruct.new(
-      content: OpenStruct.new(
-        remote: original_file_remote_base,
-        local: file_set_filename(file_set_uuid),
-        optional: false
-      ),
-      fixity: OpenStruct.new(
-        remote: "#{original_file_remote_base}/fcr:fixity",
-        local: "#{file_set_dirs(file_set_uuid)[:logs]}/content_fixity_report.n3",
-        optional: false
-      )
-    ).freeze
-  end
-
-  def member_file_set_uuids
-    @member_file_set_uuids ||= []
-    return @member_file_set_uuids unless @member_file_set_uuids.empty?
-
-    member_file_set_predicate = RDF::URI(PREDICATE_URIS[:member_file_sets])
-
-    graph = RDF::Graph.load(object_aip_paths.main_object.local)
-
-    graph.query(predicate: member_file_set_predicate) do |results|
-      # Get uuid from end of fedora path
-      @member_file_set_uuids << results.object.to_s.split('/').last
-    end
-    return @member_file_set_uuids unless @member_file_set_uuids.empty?
-
-    raise NoFileSets
-  end
-
-  def file_set_filename(file_set_uuid)
-    filename_predicate = RDF::URI(PREDICATE_URIS[:filename])
-
-    graph = RDF::Graph.load(file_set_aip_paths(file_set_uuid).main_object.local)
-
-    graph.query(predicate: filename_predicate) do |results|
-      return "#{file_set_dirs(file_set_uuid).files}/#{results.object}"
-    end
-
-    raise NoContentFilename
-  end
-
-  def member_files(file_set_uuid)
-    member_file_predicate = RDF::URI(PREDICATE_URIS[:member_files])
-
-    graph = RDF::Graph.load(file_set_aip_paths(file_set_uuid).main_object.local)
-
-    member_files = []
-    graph.query(predicate: member_file_predicate) do |results|
-      # Get uuid from end of fedora path
-      member_files << results.object.to_s.split('/').last
-    end
-    return member_files if member_files.present?
-
-    raise NoMemberFiles
-  end
-
-  def original_file?(metadata_filename)
-    type_predicate = RDF::URI(PREDICATE_URIS[:type])
-    original_file_uri = RDF::URI(PREDICATE_URIS[:original_file])
-    graph = RDF::Graph.load(metadata_filename)
-    graph.query(predicate: type_predicate) do |results|
-      return true if results.object == original_file_uri
-    end
-    false
+    @file_aip_paths[file_set_uuid] ||= {
+      fixity_remote: "#{object_uri}/filesets/#{file_set_uuid}/fixity",
+      fixity_local: "#{file_set_paths[:logs]}/content_fixity_report.n3",
+      file_set_remote: "#{object_uri}/filesets/#{file_set_uuid}",
+      file_set_local: "#{file_set_paths[:metadata]}/file_set_metadata.n3"
+    }.freeze
   end
 
 end