newspaper_works 0.1.0

data/lib/newspaper_works/data/work_derivatives.rb

@@ -0,0 +1,314 @@
+require 'hyrax'
+
+module NewspaperWorks
+  module Data
+    #   TODO: consider compositional refactoring (not mixins), but this
+    #         may make readability/comprehendability higher, and yield
+    #         higher applied/practical complexity.
+    class WorkDerivatives
+      include NewspaperWorks::Data::FilesetHelper
+      include NewspaperWorks::Data::PathHelper
+
+      # Work is primary adapted context
+      # @return [ActiveFedora::Base] Hyrax work-type object
+      attr_accessor :work
+
+      # FileSet is secondary adapted context
+      # @return [FileSet] fileset for work, with regard to these derivatives
+      attr_accessor :fileset
+
+      # Parent pointer to WorkFile object representing fileset
+      # @return [NewspaperWorks::Data::WorkFile] WorkFile for fileset, work pair
+      attr_accessor :parent
+
+      # Assigned attachment queue (of paths)
+      # @return [Array<String>] list of paths queued for attachment
+      attr_accessor :assigned
+
+      # Assigned deletion queue (of destination names)
+      # @return [Array<String>] list of destination names queued for deletion
+      attr_accessor :unassigned
+
+      # mapping of special names Hyrax uses for derivatives, not extension:
+      @remap_names = {
+        'jpeg' => 'thumbnail'
+      }
+      class << self
+        attr_accessor :remap_names
+      end
+
+      # alternate constructor spelling:
+      def self.of(work, fileset = nil, parent = nil)
+        new(work, fileset, parent)
+      end
+
+      # Adapt work and either specific or first fileset
+      def initialize(work, fileset = nil, parent = nil)
+        # adapted context usually work, may be string id of FileSet
+        @work = work
+        @fileset = fileset.nil? ? first_fileset : fileset
+        # computed name-to-path mapping, initially nil as sentinel for JIT load
+        @paths = nil
+        # assignments for attachment
+        @assigned = []
+        # un-assignments for deletion
+        @unassigned = []
+        # parent is NewspaperWorks::Data::WorkFile object for derivatives
+        @parent = parent
+      end
+
+      # Assignment state
+      # @return [String] A label describing the state of assignment queues
+      def state
+        load_paths
+        return 'dirty' unless @unassigned.empty? && @assigned.empty?
+        return 'empty' if @paths.keys.empty?
+        'saved'
+      end
+
+      # Assign a path to assigned queue for attachment
+      # @param path [String] Path to source file
+      def assign(path)
+        path = normalize_path(path)
+        validate_path(path)
+        @assigned.push(path)
+        # We are keeping assignment both in ephemeral, transient @assigned
+        #   and mirroring to db to share context with other components:
+        log_assignment(path, path_destination_name(path))
+      end
+
+      # Assign a destination name to unassigned queue for deletion -- OR --
+      #   remove a path from queue of assigned items
+      # @param name [String] Destination name (file extension), or source path
+      def unassign(name)
+        # if name is queued path, remove from @assigned queue:
+        if @assigned.include?(name)
+          @assigned.delete(name)
+          unlog_assignment(name, path_destination_name(name))
+        end
+        # if name is known destination name, remove
+        @unassigned.push(name) if exist?(name)
+      end
+
+      # commit pending changes to work files
+      #   beginning with removals, then with new assignments
+      def commit!
+        @unassigned.each { |name| delete(name) }
+        @assigned.each do |path|
+          attach(path, path_destination_name(path))
+        end
+        # reset queues after work is complete
+        @assigned = []
+        @unassigned = []
+      end
+
+      # Given a fileset meeting both of the following conditions:
+      #   1. a non-nil import_url value;
+      #   2. is attached to a work (persisted in Fedora, if not yet in Solr)...
+      # ...this method gets associated derivative paths queued and attach all.
+      # @param file_set [FileSet] saved file set, attached to work,
+      #   with identifier, and a non-nil import_url
+      def commit_queued!(file_set)
+        raise ArgumentError, 'No FileSet import_url' if file_set.import_url.nil?
+        import_path = file_url_to_path(file_set.import_url)
+        work = file_set.member_of.select(&:work?)[0]
+        raise ArgumentError, 'Work not found for fileset' if work.nil?
+        derivatives = WorkDerivatives.of(work, file_set)
+        IngestFileRelation.derivatives_for_file(import_path).each do |path|
+          next unless File.exist?(path)
+          attachment_record = DerivativeAttachment.where(path: path).first
+          derivatives.attach(path, attachment_record.destination_name)
+          # update previously nil fileset id
+          attachment_record.fileset_id = file_set.id
+          attachment_record.save!
+        end
+        @fileset ||= file_set
+        load_paths
+      end
+
+      # attach a single derivative file to work
+      # @param file [String, IO] path to file or IO object
+      # @param name [String] destination name, usually file extension
+      def attach(file, name)
+        raise 'Cannot save for nil fileset' if fileset.nil?
+        mkdir_pairtree
+        path = path_factory.derivative_path_for_reference(fileset, name)
+        # if file argument is path, copy file
+        if file.class == String
+          FileUtils.copy(file, path)
+        else
+          # otherwise, presume file is an IO, read, write it
+          #   note: does not close input file/IO, presume that is caller's
+          #   responsibility.
+          orig_pos = file.tell
+          file.seek(0)
+          File.open(path, 'w') { |dstfile| dstfile.write(file.read) }
+          file.seek(orig_pos)
+        end
+        # finally, reload @paths after mutation
+        load_paths
+      end
+
+      # Delete a derivative file from work, by destination name
+      # @param name [String] destination name, usually file extension
+      def delete(name, force: nil)
+        raise 'Cannot save for nil fileset' if fileset.nil?
+        path = path_factory.derivative_path_for_reference(fileset, name)
+        # will remove file, if it exists; won't remove pairtree, even
+        #   if it becomes empty, as that is excess scope.
+        FileUtils.rm(path, force: force) if File.exist?(path)
+        # finally, reload @paths after mutation
+        load_paths
+      end
+
+      # Load all paths/names to @paths once, upon first access
+      def load_paths
+        fsid = fileset_id
+        if fsid.nil?
+          @paths = {}
+          return
+        end
+        # list of paths
+        paths = path_factory.derivatives_for_reference(fsid)
+        # names from paths
+        @paths = paths.map { |e| [path_destination_name(e), e] }.to_h
+      end
+
+      # path to existing derivative file for destination name
+      # @param name [String] destination name, usually file extension
+      # @return [String, NilClass] path (or nil)
+      def path(name)
+        load_paths if @paths.nil?
+        result = @paths[name]
+        return if result.nil?
+        File.exist?(result) ? result : nil
+      end
+
+      # Run a block in context of the opened derivative file for reading
+      # @param name [String] destination name, usually file extension
+      # @param block [Proc] block/proc to run in context of file IO
+      def with_io(name, &block)
+        mode = ['xml', 'txt', 'html'].include?(name) ? 'rb:UTF-8' : 'rb'
+        filepath = path(name)
+        return if filepath.nil?
+        File.open(filepath, mode, &block)
+      end
+
+      # Get number of derivatives or, if a destination name argument
+      #   is provided, the size of derivative file
+      # @param name [String] optional destination name, usually file extension
+      # @return [Integer] size in bytes
+      def size(name = nil)
+        load_paths if @paths.nil?
+        return @paths.size if name.nil?
+        File.size(@paths[name])
+      end
+
+      # Check if derivative file exists for destination name
+      # @param name [String] optional destination name, usually file extension
+      # @return [TrueClass, FalseClass] boolean
+      def exist?(name)
+        keys.include?(name) && File.exist?(self[name])
+      end
+
+      # Get raw binary or encoded text data of file as a String
+      # @param name [String] destination name, usually file extension
+      # @return [String] Raw bytes, or if text file, a UTF-8 encoded String
+      def data(name)
+        result = ''
+        with_io(name) do |io|
+          result += io.read
+        end
+        result
+      end
+
+      private
+
+        def primary_file_path
+          if fileset.nil?
+            # if there is a nil fileset, we look for *intent* in the form
+            #   of the first assigned file path for single-file work.
+            work_file = parent
+            return if work_file.nil?
+            work_files = work_file.parent
+            return if work_files.nil?
+            work_files.assigned[0]
+          else
+            file_url_to_path(fileset.import_url) unless fileset.import_url.nil?
+          end
+        end
+
+        def file_url_to_path(url)
+          url.gsub('file://', '')
+        end
+
+        def log_primary_file_relation(path)
+          file_path = primary_file_path
+          return if file_path.nil?
+          NewspaperWorks::IngestFileRelation.create!(
+            file_path: file_path,
+            derivative_path: path
+          )
+        end
+
+        def log_assignment(path, name)
+          NewspaperWorks::DerivativeAttachment.create!(
+            fileset_id: fileset_id,
+            path: path,
+            destination_name: name
+          )
+          log_primary_file_relation(path)
+        end
+
+        def unlog_assignment(path, name)
+          if fileset_id.nil?
+            NewspaperWorks::DerivativeAttachment.where(
+              path: path,
+              destination_name: name
+            ).destroy_all
+          else
+            NewspaperWorks::DerivativeAttachment.where(
+              fileset_id: fileset_id,
+              path: path,
+              destination_name: name
+            ).destroy_all
+          end
+          # note: there is deliberately no attempt to "unlog" primary
+          #   file relation, as leaving it should have no side-effect.
+        end
+
+        def path_destination_name(path)
+          ext = path.split('.')[-1]
+          self.class.remap_names[ext] || ext
+        end
+
+        def respond_to_missing?(symbol, include_priv = false)
+          {}.respond_to?(symbol, include_priv)
+        end
+
+        def method_missing(method, *args, &block)
+          # if we proxy mapping/hash enumertion methods,
+          #   make sure @paths loaded, then proxy to it.
+          if respond_to_missing?(method)
+            load_paths if @paths.nil?
+            return @paths.send(method, *args, &block)
+          end
+          super
+        end
+
+        def path_factory
+          Hyrax::DerivativePath
+        end
+
+        # make shared path for derivatives to live, given
+        def mkdir_pairtree
+          # Hyrax::DerivativePath has no public method to directly get the
+          #   bare pairtree path for derivatives for a fileset, but we
+          #   can infer it...
+          path = path_factory.derivative_path_for_reference(fileset, '')
+          dir = File.join(path.split('/')[0..-2])
+          FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+        end
+    end
+  end
+end

data/lib/newspaper_works/data/work_file.rb

@@ -0,0 +1,92 @@
+# encoding=utf-8
+
+require 'hyrax'
+
+module NewspaperWorks
+  module Data
+    # WorkFile is a read-only convenience wrapper for just-in-time
+    #   file operations, and is the type of values returned by
+    #   NewspaperWorks::Data::WorkFiles (container) adapter.
+    class WorkFile
+      # accessors for adaptation relationships:
+      attr_accessor :work, :parent, :fileset
+      # delegate these metadata properties to @fileset.original_file:
+      delegate :size, :date_created, :date_modified, :mime_type, to: :unwrapped
+
+      # alternate constructor spelling:
+      def self.of(work, fileset = nil, parent = nil)
+        new(work, fileset, parent)
+      end
+
+      def initialize(work, fileset = nil, parent = nil)
+        @work = work
+        # If fileset is nil, presume *first* fileset of work, as in
+        #   the single-file-per-work use-case:
+        @fileset = fileset
+        # Parent is WorkFiles (container) object, if applciable:
+        @parent = parent
+      end
+
+      # Get original repository object representing file (not fileset).
+      # @return [ActiveFedora::File] repository file persistence object
+      def unwrapped
+        return nil if @fileset.nil?
+        @fileset.original_file
+      end
+
+      def ==(other)
+        return false if @fileset.nil?
+        unwrapped.id == other.unwrapped.id
+      end
+
+      # Get path to working copy of file on local filesystem;
+      #   checkout file from repository/source as needed.
+      # @return [String] path to working copy of binary
+      def path
+        return nil if @fileset.nil?
+        checkout
+      end
+
+      # Read data from working copy of file on local filesystem;
+      #   checkout file from repository/source as needed.
+      # @return [String] byte data of binary/file payload
+      def data
+        return '' if @fileset.nil?
+        File.read(path, mode: 'rb')
+      end
+
+      # Run block/proc upon data of file;
+      #   checkout file from repository/source as needed.
+      # @yield [io] read-only IO or File object to block/proc.
+      def with_io(&block)
+        filepath = path
+        return if filepath.nil?
+        File.open(filepath, 'rb', &block)
+      end
+
+      # Get filename from stored metadata
+      # @return [String] file name stored in repository metadata for file
+      def name
+        return nil if @fileset.nil?
+        unwrapped.original_name
+      end
+
+      # Derivatives for fileset associated with this primary file object
+      # @return [NewspaperWorks::Data::WorkDerviatives] derivatives adapter
+      def derivatives
+        NewspaperWorks::Data::WorkDerivatives.of(work, fileset, self)
+      end
+
+      private
+
+        def checkout
+          file = @fileset.original_file
+          # find_or_retrieve returns path to working copy, but only
+          #   fetches from Fedora if no working copy exists on filesystem.
+          # NOTE: there may be some benefit to memoizing to avoid
+          #   call and File.exist? IO operation, but YAGNI for now.
+          Hyrax::WorkingDirectory.find_or_retrieve(file.id, @fileset.id)
+        end
+    end
+  end
+end

data/lib/newspaper_works/data/work_files.rb

@@ -0,0 +1,181 @@
+module NewspaperWorks
+  module Data
+    class WorkFiles
+      include NewspaperWorks::Data::PathHelper
+
+      attr_accessor :work, :assigned, :unassigned
+      delegate :include?, to: :keys
+
+      # alternate constructor spelling:
+      def self.of(work)
+        new(work)
+      end
+
+      def initialize(work)
+        @work = work
+        @assigned = []
+        @unassigned = []
+        @derivatives = nil
+      end
+
+      # Derivatives for specified fileset or first fileset found.
+      #   The `WorkDerivatives` adapter as assign/commmit! semantics just
+      #   like `WorkFiles`, and also acts like a hash/mapping of
+      #   destination names (usually file extension) to path of saved
+      #   derviative.  Always returns same instance (memoized after first
+      #   use) of `WorkDerivatives`.
+      # @return [NewspaperWorks::Data::WorkDerviatives] derivatives adapter
+      def derivatives(fileset: nil)
+        fileset ||= @fileset
+        return @derivatives unless @derivatives.nil?
+        if fileset.nil?
+          # for the deferred assignement case, we have no fileset yet...
+          work_file = NewspaperWorks::Data::WorkFile.of(work, nil, self)
+          return work_file.derivatives
+        end
+        # Otherwise, delegate actual construction to WorkFile.derivatives:
+        @derivatives = values[0].derivatives
+      end
+
+      # Assignment state
+      # @return [String] A label describing the state of assignment queues
+      def state
+        return 'dirty' unless @assigned.empty? && @unassigned.empty?
+        return 'empty' if keys.empty?
+        # TODO: implement 'pending' as intermediate state between 'dirty'
+        #   and saved, where we look for saved state that matches what was
+        #   previously assigned in THIS instance.  We can only know that
+        #   changes initiated by this instance in this thread are pending
+        #   because there's no global storage for the assignment queue.
+        'saved'
+      end
+
+      # List of fileset (not file) id keys, presumes system like Hyrax
+      #   is only keeping a 1:1 between fileset and contained PCDM file,
+      #   because derivatives are not stored in the FileSet.
+      # @return [String] fileset ids
+      def keys
+        filesets.map(&:id)
+      end
+
+      # List of WorkFile for each primary file
+      # @return [Array<NewspaperWorks::Data::WorkFile>] adapter for persisted
+      #   primary file
+      def values
+        keys.map(&method(:get))
+      end
+
+      # Array of [id, WorkFile] for each primary file
+      # @return [Array<Array>] key/value pairs for primary files of work
+      def entries
+        filesets.map { |fs| [fs.id, self[fs.id]] }
+      end
+
+      # List of local file names for attachments, based on original ingested
+      #   or uploaded file name.
+      # @return [Array<String>]
+      def names
+        filesets.map(&method(:original_name))
+      end
+
+      # Get a WorkFile adapter representing primary file, either by name or id
+      # @param name_or_id [String] Fileset id or work-local file name
+      # @return [NewspaperWorks::Data::WorkFile] adapter for persisted
+      #   primary file
+      def get(name_or_id)
+        return get_by_fileset_id(name_or_id) if keys.include?(name_or_id)
+        get_by_filename(name_or_id)
+      end
+
+      # Assign a path to assigned queue for attachment
+      # @param path [String] Path to source file
+      def assign(path)
+        path = normalize_path(path)
+        validate_path(path)
+        @assigned.push(path)
+      end
+
+      # Assign a name or id to unassigned queue for deletion -- OR -- remove a
+      #   path from queue of assigned items
+      # @param name_or_id [String] Fileset id, local file name, or source path
+      def unassign(name_or_id)
+        # if name_or_id is queued path, remove from @assigned queue:
+        @assigned.delete(name_or_id) if @assigned.include?(name_or_id)
+        # if name_or_id is known id or name, remove
+        @unassigned.push(name_or_id) if include?(name_or_id)
+      end
+
+      # commit pending changes to work files
+      #   beginning with removals, then with new assignments
+      def commit!
+        commit_unassigned
+        commit_assigned
+      end
+
+      alias [] :get
+
+      private
+
+        def get_by_fileset_id(id)
+          nil unless keys.include?(id)
+          fileset = FileSet.find(id)
+          NewspaperWorks::Data::WorkFile.of(work, fileset, self)
+        end
+
+        # Get one WorkFile object based on filename in metadata
+        def get_by_filename(name)
+          r = filesets.select { |fs| original_name(fs) == name }
+          # checkout first match
+          r.empty? ? nil : NewspaperWorks::Data::WorkFile.of(work, r[0], self)
+        end
+
+        def original_name(fileset)
+          fileset.original_file.original_name
+        end
+
+        def filesets
+          # file sets with non-nil original file contained:
+          work.members.select { |m| m.class == FileSet && m.original_file }
+        end
+
+        def user
+          return User.find_by(email: work.depositor) unless work.depositor.nil?
+          defined?(current_user) ? current_user : User.batch_user
+        end
+
+        def ensure_depositor
+          return unless @work.depositor.nil?
+          @work.depositor = user.user_key
+        end
+
+        def commit_unassigned
+          # for each (name or) id to be removed from work, use actor to destroy
+          @unassigned.each do |id|
+            # "actor" here is simply a multi-adapter of Fileset, User
+            # Calling destroy will:
+            #   1. unlink fileset from work, and save work
+            #   2. Destroy fileset:
+            #     - :before_destroy callback will delegate derivative cleanup
+            #       to derivatives service component(s).
+            #     - Remove fileset from storage/persistence layers
+            #     - Invoke (logging or other) :after_destroy callback
+            Hyrax::Actors::FileSetActor.new(get(id).fileset, user).destroy
+            work.reload
+          end
+        end
+
+        def commit_assigned
+          return if @assigned.nil? || @assigned.empty?
+          ensure_depositor
+          remote_files = @assigned.map do |path|
+            { url: path_to_uri(path), file_name: File.basename(path) }
+          end
+          attrs = { remote_files: remote_files }
+          # Create an environment for actor stack:
+          env = Hyrax::Actors::Environment.new(@work, Ability.new(user), attrs)
+          # Invoke default Hyrax actor stack middleware:
+          Hyrax::CurationConcern.actor.create(env)
+        end
+    end
+  end
+end

data/lib/newspaper_works/data.rb

@@ -0,0 +1,35 @@
+require 'newspaper_works/data/fileset_helper'
+require 'newspaper_works/data/path_helper'
+require 'newspaper_works/data/work_derivatives'
+require 'newspaper_works/data/work_files'
+require 'newspaper_works/data/work_file'
+
+module NewspaperWorks
+  # Module for data access helper / adapter classes supporting, enhancing
+  #   NewspaperWorks work models
+  module Data
+    # Handler for after_create_fileset, to be called by block subscribing to
+    #   and overriding default Hyrax `:after_create_fileset` handler, via
+    #   app integrating newspaper_works.
+    def self.handle_after_create_fileset(file_set, user)
+      handle_queued_derivative_attachments(file_set)
+      # Hyrax queues this job by default, and since newspaper_works
+      #   overrides the single subscriber Hyrax uses to do so, we
+      #   must call this here:
+      FileSetAttachedEventJob.perform_later(file_set, user)
+      work = file_set.member_of[0]
+      # Hyrax CreateWithRemoteFilesActor has glaring omission re: this job,
+      #   so we call it here, once we have a fileset to copy permissions to.
+      InheritPermissionsJob.perform_later(work) unless work.nil?
+    end
+
+    def self.handle_queued_derivative_attachments(file_set)
+      return if file_set.import_url.nil?
+      work = file_set.member_of.select(&:work?)[0]
+      derivatives = NewspaperWorks::Data::WorkDerivatives.of(work)
+      # For now, becuase this is IO-bound operation, it makes sense to have
+      #   this not be a job, but run inline:
+      derivatives.commit_queued!(file_set)
+    end
+  end
+end

data/lib/newspaper_works/engine.rb

@@ -0,0 +1,42 @@
+require 'active_fedora'
+require 'hyrax'
+require 'blacklight_iiif_search'
+require 'blacklight_advanced_search'
+
+module NewspaperWorks
+  # module constants:
+  GEM_PATH = Gem::Specification.find_by_name("newspaper_works").gem_dir
+
+  # Engine Class
+  class Engine < ::Rails::Engine
+    isolate_namespace NewspaperWorks
+
+    config.to_prepare do
+      # Inject PluggableDerivativeService ahead of Hyrax default.
+      #   This wraps Hyrax default, but allows multiple valid services
+      #   to be configured, instead of just the _first_ valid service.
+      #
+      #   To configure specific services, inject each service, in desired order
+      #   to NewspaperWorks::PluggableDerivativeService.plugins array.
+
+      Hyrax::DerivativeService.services.unshift(
+        NewspaperWorks::PluggableDerivativeService
+      )
+
+      # Register specific derivative services to be considered by
+      #   PluggableDerivativeService:
+      [
+        NewspaperWorks::JP2DerivativeService,
+        NewspaperWorks::PDFDerivativeService,
+        NewspaperWorks::TextExtractionDerivativeService,
+        NewspaperWorks::TIFFDerivativeService
+      ].each do |plugin|
+        NewspaperWorks::PluggableDerivativeService.plugins.push plugin
+      end
+
+      # Register actor to handle any NewspaperWorks upload behaviors before
+      #   CreateWithFilesActor gets to them:
+      Hyrax::CurationConcern.actor_factory.insert_before Hyrax::Actors::CreateWithFilesActor, NewspaperWorks::Actors::NewspaperWorksUploadActor
+    end
+  end
+end

data/lib/newspaper_works/errors.rb

@@ -0,0 +1,14 @@
+module NewspaperWorks
+  # generic/base NewspaperWorks-specific exception:
+  class NewspaperWorksError < StandardError
+  end
+
+  # Data transformation or read-error:
+  class DataError < NewspaperWorksError
+  end
+
+  # Specific exception for temporary state where one or more PDF page source
+  #   files are not ready, for which a retry at a later time is warranted.
+  class PagesNotReady < DataError
+  end
+end