newspaper_works 0.1.0

data/lib/newspaper_works/ingest/ndnp/issue_ingester.rb

@@ -0,0 +1,101 @@
+module NewspaperWorks
+  module Ingest
+    module NDNP
+      class IssueIngester
+        include NewspaperWorks::Logging
+        include NewspaperWorks::Ingest::NDNP::NDNPAssetHelper
+        include NewspaperWorks::Ingest::PubFinder
+
+        attr_accessor :issue, :target, :opts
+
+        delegate :path, to: :issue
+
+        COPY_FIELDS = [
+          :lccn,
+          :edition_number,
+          :edition_name,
+          :volume,
+          :publication_date,
+          :held_by,
+          :issue_number
+        ].freeze
+
+        # @param issue [NewspaperWorks::Ingest::NDNP::IssueIngest]
+        #   source issue data
+        # @param opts [Hash]
+        #   ingest options, e.g. administrative metadata
+        def initialize(issue, opts = {})
+          @issue = issue
+          @opts = opts
+          @target = nil
+          configure_logger('ingest')
+        end
+
+        def ingest
+          construct_issue
+          ingest_pages
+          NewspaperWorks::ComposeIssuePDFJob.perform_later(@target)
+        end
+
+        def construct_issue
+          create_issue
+          find_or_create_linked_publication
+        end
+
+        def ingest_pages
+          issue.each do |page|
+            page_ingester(page).ingest
+          end
+        end
+
+        private
+
+          def page_ingester(page_data)
+            NewspaperWorks::Ingest::NDNP::PageIngester.new(
+              page_data,
+              @target,
+              @opts
+            )
+          end
+
+          def publication_date
+            parsed = DateTime.iso8601(issue.metadata.publication_date)
+            parsed.strftime('%B %-d, %Y')
+          end
+
+          def publication_title(issue)
+            issue.metadata.publication_title.strip.split(/ \(/)[0]
+          end
+
+          def issue_title
+            "#{publication_title(issue)}: #{publication_date}"
+          end
+
+          def copy_issue_metadata
+            metadata = issue.metadata
+            # set (required, plural) title from single value obtained from reel:
+            @target.title = [issue_title]
+            # copy all fields with singular (non-repeatable) values on both
+            #   target NewspaperIssue object, and metadata source:
+            COPY_FIELDS.each do |fieldname|
+              @target.send("#{fieldname}=", metadata.send(fieldname.to_s))
+            end
+          end
+
+          def create_issue
+            @target = NewspaperIssue.create
+            copy_issue_metadata
+            assign_administrative_metadata
+            @target.save!
+            write_log("Saved metadata to new NewspaperIssue #{@target.id}")
+          end
+
+          def find_or_create_linked_publication
+            title = publication_title(issue)
+            lccn = issue.metadata.lccn
+            find_or_create_publication_for_issue(@target, lccn, title, @opts)
+          end
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/ndnp/issue_metadata.rb

@@ -0,0 +1,96 @@
+module NewspaperWorks
+  module Ingest
+    module NDNP
+      class IssueMetadata
+        include NewspaperWorks::Ingest::NDNP::NDNPMetsHelper
+
+        attr_accessor :path, :doc, :parent
+
+        def initialize(path, parent = nil)
+          @path = path
+          @parent = parent
+          @doc = nil
+          load_doc
+        end
+
+        def inspect
+          format(
+            "<#{self.class}:0x000000000%<oid>x\n" \
+              "\tpath: '#{path}',\n",
+            oid: object_id << 1
+          )
+        end
+
+        # LCCN (mandatory)
+        # @return [String]
+        def lccn
+          xpath("//mods:identifier[@type='lccn']").text
+        end
+
+        # Volume number (optional)
+        # @return [String,NilClass]
+        def volume
+          result = xpath("//mods:detail[@type='volume']/mods:number")
+          return if result.size.zero?
+          result.text
+        end
+
+        # Issue number (optional)
+        # @return [String,NilClass]
+        def issue_number
+          result = xpath("//mods:detail[@type='issue']/mods:number")
+          return if result.size.zero?
+          result.text
+        end
+
+        # Edition name
+        #   Edition name is optional ("caption" / "label") is optional
+        #     in NDNP, but as it may be used as a label for readability.
+        # @return [String,NilClass]
+        def edition_name
+          ed_name = xpath("//mods:detail[@type='edition']/mods:caption")
+          return ed_name.text unless ed_name.size.zero?
+        end
+
+        # Edition name, with fallback to edition number (mandatory)
+        # @return [String]
+        def edition_number
+          xpath("//mods:detail[@type='edition']/mods:number").text
+        end
+
+        # Issue date (mandatory field) as ISO 8601 datestamp string
+        # @return [String] (ISO-8601 date) publication date
+        def publication_date
+          xpath("//mods:originInfo/mods:dateIssued").text
+        end
+
+        def publication_title
+          # try from reel first
+          reel = parent.nil? ? nil : parent.container
+          return reel.metadata.title unless reel.nil?
+          # fallback to parsing //mets/@LABEL
+          label = xpath('//mets:mets/@LABEL').first
+          v = label.nil? ? '' : label.value.split(/[,] [0-9]/)[0]
+          # based on label convention:
+          #   "ACME Times (Springfield, UT), 1911-01-25, First Edition"
+          #   Returns the name and (*for now TBD*) place of publication
+          #   as a string in parentheses.
+          v.split(/, [0-9]/)[0]
+        end
+
+        # Original Source Repository (NDNP-mandatory)
+        # @return [String]
+        def held_by
+          xpath("//mods:physicalLocation").first['displayLabel']
+        end
+
+        private
+
+          def load_doc
+            @doc = @parent.doc unless @parent.nil?
+            @doc = Nokogiri::XML(File.open(path)) if @doc.nil?
+          end
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/ndnp/ndnp_asset_helper.rb

@@ -0,0 +1,20 @@
+module NewspaperWorks
+  module Ingest
+    module NDNP
+      # Mixin for mets-specific XPath and traversal of issue/page data
+      module NDNPAssetHelper
+        # Set administrative metadata for asset, based on options saved
+        #   on ingester state.
+        # Pre-conditions for use:
+        #   consuming class implements @target pointing to work asset
+        #   consuming class implements @opts pointing to Hash
+        def assign_administrative_metadata(work = nil)
+          NewspaperWorks::Ingest.assign_administrative_metadata(
+            work || @target,
+            @opts
+          )
+        end
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/ndnp/ndnp_mets_helper.rb

@@ -0,0 +1,70 @@
+require 'nokogiri'
+
+module NewspaperWorks
+  module Ingest
+    module NDNP
+      # Mixin for mets-specific XPath and traversal of issue/page data
+      module NDNPMetsHelper
+        XML_NS = {
+          mets: 'http://www.loc.gov/METS/',
+          METS: 'http://www.loc.gov/METS/',
+          mods: 'http://www.loc.gov/mods/v3',
+          MODS: 'http://www.loc.gov/mods/v3',
+          ndnp: 'http://www.loc.gov/ndnp',
+          NDNP: 'http://www.loc.gov/ndnp'
+        }.freeze
+
+        # DRY XPath without repeatedly specifying default namespace urlmap
+        def xpath(expr, context = nil)
+          context ||= doc
+          context.xpath(
+            expr,
+            **XML_NS
+          )
+        end
+
+        def dmd_node
+          xpath("//mets:dmdSec[@ID='#{dmdid}']")
+        end
+
+        def normalize_path(specified_path)
+          return specified_path if specified_path.start_with?('/')
+          basename = File.dirname(path)
+          File.join(basename, specified_path)
+        end
+
+        # returns hash of "use" key string to path value
+        def page_files
+          # get pointers from structmap:
+          file_group = xpath("//mets:structMap//mets:div[@DMDID='#{dmdid}']")
+          result = xpath('mets:fptr', file_group).map do |fptr|
+            file_id = fptr['FILEID']
+            file_node = xpath(
+              "//mets:fileSec//mets:fileGrp//mets:file[@ID='#{file_id}']"
+            ).first
+            [
+              file_node['USE'],
+              xpath('mets:FLocat', file_node).first.attribute_with_ns(
+                'href',
+                'http://www.w3.org/1999/xlink'
+              ).to_s
+            ]
+          end
+          result.to_h
+        end
+
+        def container_path
+          reel_dir = File.expand_path('..', File.dirname(path))
+          reel_base = File.basename(reel_dir)
+          File.join(reel_dir, "#{reel_base}_1.xml")
+        end
+
+        def container
+          reel_path = container_path
+          return unless File.exist?(reel_path)
+          NewspaperWorks::Ingest::NDNP::ContainerIngest.new(reel_path)
+        end
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/ndnp/page_ingest.rb

@@ -0,0 +1,47 @@
+module NewspaperWorks
+  module Ingest
+    module NDNP
+      class PageIngest
+        include NewspaperWorks::Ingest::NDNP::NDNPMetsHelper
+
+        attr_accessor :path, :dmdid, :doc, :files
+
+        def initialize(path = nil, dmdid = nil, parent = nil)
+          raise ArgumentError, 'No path provided' if path.nil?
+          @path = path
+          @dmdid = dmdid
+          @doc = nil
+          @parent = parent
+          @metadata = nil
+          load_doc
+          @files = page_files.values.map(&method(:normalize_path))
+        end
+
+        def inspect
+          format(
+            "<#{self.class}:0x000000000%<oid>x\n" \
+              "\tpath: '#{path}',\n" \
+              "\tdmdid: '#{dmdid}' ...>",
+            oid: object_id << 1
+          )
+        end
+
+        def metadata
+          return @metadata unless @metadata.nil?
+          @metadata = NewspaperWorks::Ingest::NDNP::PageMetadata.new(
+            path,
+            self,
+            dmdid
+          )
+        end
+
+        private
+
+          def load_doc
+            @doc = @parent.doc unless @parent.nil?
+            @doc = Nokogiri::XML(File.open(path)) if @doc.nil?
+          end
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/ndnp/page_ingester.rb

@@ -0,0 +1,157 @@
+require 'newspaper_works/logging'
+
+module NewspaperWorks
+  module Ingest
+    module NDNP
+      class PageIngester
+        include NewspaperWorks::Logging
+        include NewspaperWorks::Ingest::NDNP::NDNPAssetHelper
+
+        attr_accessor :page, :issue, :target, :opts
+
+        delegate :path, :dmdid, to: :page
+
+        COPY_FIELDS = [
+          :width,
+          :height,
+          :page_number,
+          :identifier
+        ].freeze
+
+        COPY_FIELDS_PLURALIZE = [
+          :identifier
+        ].freeze
+
+        # @param page [NewspaperWorks::Ingest::NDNP::PageIngest]
+        #   source page data
+        # @param issue [NewspaperIssue]
+        #   source issue data
+        # @param opts [Hash]
+        #   ingest options, e.g. administrative metadata
+        def initialize(page, issue, opts = {})
+          @page = page
+          @issue = issue
+          @opts = opts
+          # target is to-be-created NewspaperPage:
+          @target = nil
+          @work_files = nil
+          configure_logger('ingest')
+        end
+
+        def ingest
+          construct_page
+          ingest_page_files
+          link_reel
+        end
+
+        def construct_page
+          @target = NewspaperPage.create!(title: page_title)
+          write_log(
+            "Created NewspaperPage work #{@target.id} "\
+            "with title '#{@target.title[0]}'"
+          )
+          copy_page_metadata
+          assign_administrative_metadata
+          link_issue
+          @target.save!
+          write_log("Saved metadata to NewspaperPage work #{@target.id}")
+        end
+
+        # Ingest primary, derivative files; other derivatives including
+        #   thumbnail, plain-text, json will be made by NewspaperWorks
+        #   derivative service components as a consequence of commiting
+        #   files assigned (via actor stack, via WorkFiles).
+        def ingest_page_files
+          @work_files = NewspaperWorks::Data::WorkFiles.new(@target)
+          page.files.each do |path|
+            ext = path.downcase.split('.')[-1]
+            if ['tif', 'tiff'].include?(ext)
+              ingest_primary_file(path)
+            else
+              ingest_derivative_file(path)
+            end
+          end
+          write_log("Beginning file attachment process (WorkFiles.commit!) "\
+            "for work #{@target.id}")
+          @work_files.commit!
+        end
+
+        def link_reel
+          reel_data = @page.container
+          return if reel_data.nil?
+          ingester = NewspaperWorks::Ingest::NDNP::ContainerIngester.new(
+            reel_data,
+            issue.publication,
+            @opts
+          )
+          # find-or-create container, linked to publication:
+          ingester.ingest
+          # link target page to container asset for reel:
+          ingester.link(@target)
+        end
+
+        private
+
+          def ingest_primary_file(path)
+            unless File.exist?(path)
+              pdf_path = page.files.select { |p| p.end_with?('pdf') }[0]
+              # make and get TIFF path (to generated tmp file):
+              path = make_tiff(pdf_path)
+            end
+            write_log("Assigned primary file to work #{@target.id}, #{path}")
+            @work_files.assign(path)
+          end
+
+          def ingest_derivative_file(path)
+            write_log("Assigned derivative file to work #{@target.id}, #{path}")
+            @work_files.derivatives.assign(path)
+          end
+
+          def link_issue
+            issue.ordered_members << @target # page
+            issue.save!
+            write_log(
+              "Linked NewspaperIssue work #{issue.id} "\
+              "to NewspaperPage work #{@target.id}"
+            )
+          end
+
+          # dir whitelist
+          def whitelist
+            Hyrax.config.whitelisted_ingest_dirs
+          end
+
+          # Generate TIFF in temporary file, return its path, given path to PDF
+          # @param pdf_path [String] path to single-page PDF
+          # @return [String] path to generated TIFF
+          def make_tiff(pdf_path)
+            write_log(
+              "Creating TIFF from PDF in lieu of missing for work "\
+              " (#{@target.id})",
+              Logger::WARN
+            )
+            whitelist.push(Dir.tmpdir) unless whitelist.include?(Dir.tmpdir)
+            NewspaperWorks::Ingest::PdfPages.new(pdf_path).to_a[0]
+          end
+
+          # Page title as issue title plus page title
+          #   e.g. "ACME Tribune (1910-01-02): Page 2"
+          # @return [String] composed page title
+          def page_title
+            ["#{issue.title.first}: Page #{@page.metadata.page_number}"]
+          end
+
+          def copy_page_metadata
+            metadata = page.metadata
+            # copy all fields with singular (non-repeatable) values on both
+            #   target NewspaperIssue object, and metadata source:
+            COPY_FIELDS.each do |fieldname|
+              value = metadata.send(fieldname.to_s)
+              pluralize = COPY_FIELDS_PLURALIZE.include?(fieldname)
+              @target.send("#{fieldname}=", pluralize ? [value] : value)
+            end
+          end
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/ndnp/page_metadata.rb

@@ -0,0 +1,112 @@
+require 'nokogiri'
+
+module NewspaperWorks
+  module Ingest
+    module NDNP
+      class PageMetadata
+        # mixin convenience methods for NDNP XML, plus XML_NS hash
+        include NewspaperWorks::Ingest::NDNP::NDNPMetsHelper
+
+        attr_accessor :path, :dmdid, :doc
+
+        def initialize(path = nil, parent = nil, dmdid = nil)
+          raise ArgumentError, 'No context provided' if path.nil? && parent.nil?
+          @path = path
+          @parent = parent
+          @dmdid = dmdid
+          @doc = nil
+          load_doc
+        end
+
+        def inspect
+          format(
+            "<#{self.class}:0x000000000%<oid>x\n" \
+              "\tpath: '#{path}',\n" \
+              "\tdmdid: '#{dmdid}' ...>",
+            oid: object_id << 1
+          )
+        end
+
+        # Printed page number, if printed; optional field in NDNP spec.
+        #   "Number" is used liberally, and may contain both alpha
+        #   and numeric characters.  As such, return value is String.
+        #
+        #   If NDNP issue data fails to provide an explicitly
+        #   human-readable page number, fallback to sequence
+        #   number, in String form.
+        #
+        # @return [String, NilClass] Page "number" string
+        def page_number
+          detail = dmd_node.xpath(
+            ".//mods:mods//mods:detail[@type='page number']",
+            **XML_NS
+          )
+          if detail.size.zero?
+            fallback = page_sequence_number
+            return fallback.nil? ? nil : fallback.to_s
+          end
+          detail.xpath("mods:number", **XML_NS).first.text
+        end
+
+        # Page sequence number, indexical to order in issue.
+        #   "Number" here is one-indexed positive integer, position in
+        #   issue.  Mandatory for page of issue, nil for page of reel.
+        # @return [Integer,NilClass] Page sequence number, positive integer
+        def page_sequence_number
+          detail = dmd_node.xpath(
+            ".//mods:mods//mods:extent[@unit='pages']",
+            **XML_NS
+          )
+          node = detail.xpath("mods:start", **XML_NS).first
+          node.text.to_i unless node.nil?
+        end
+
+        # Extract identifier from page ALTO, based on file name.
+        #   XML parsing of big documents are expensive, so use regex to
+        #   scan for fileName element, and return its value.
+        # @return [String,NilClass] file name or path, or nil.
+        def identifier
+          matches = page_alto.scan(/<fileName>([^<]*)<\/fileName>/).first
+          matches.size.zero? ? nil : stripped_filename(matches[0])
+        end
+
+        def height
+          alto_page_meta('HEIGHT').to_i
+        end
+
+        def width
+          alto_page_meta('WIDTH').to_i
+        end
+
+        private
+
+          # filename stripped of base path and file extension
+          def stripped_filename(path)
+            File.basename(path).split('.')[0]
+          end
+
+          def load_doc
+            @doc = @parent.doc unless @parent.nil?
+            @doc = Nokogiri::XML(File.open(path)) if @doc.nil?
+          end
+
+          def alto_path
+            specified_path = page_files['ocr']
+            normalize_path(specified_path)
+          end
+
+          def page_alto
+            File.read(alto_path)
+          end
+
+          def alto_page_meta(key)
+            matches = page_alto.scan(/(<Page [^>]*>)/).first
+            return if matches.size.zero?
+            # parse xml <Page> start tag fragment, get attributes:
+            page_tag = Nokogiri::XML(matches[0]).root
+            page_tag[key]
+          end
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/ndnp.rb

@@ -0,0 +1,21 @@
+require 'newspaper_works/ingest/ndnp/ndnp_mets_helper'
+require 'newspaper_works/ingest/ndnp/ndnp_asset_helper'
+require 'newspaper_works/ingest/ndnp/page_ingest'
+require 'newspaper_works/ingest/ndnp/page_ingester'
+require 'newspaper_works/ingest/ndnp/page_metadata'
+require 'newspaper_works/ingest/ndnp/issue_ingest'
+require 'newspaper_works/ingest/ndnp/issue_ingester'
+require 'newspaper_works/ingest/ndnp/issue_metadata'
+require 'newspaper_works/ingest/ndnp/container_ingest'
+require 'newspaper_works/ingest/ndnp/container_ingester'
+require 'newspaper_works/ingest/ndnp/container_metadata'
+require 'newspaper_works/ingest/ndnp/batch_xml_ingest'
+require 'newspaper_works/ingest/ndnp/batch_ingester'
+
+module NewspaperWorks
+  module Ingest
+    # Module for NDNP-specific ingest components
+    module NDNP
+    end
+  end
+end

data/lib/newspaper_works/ingest/newspaper_issue_ingest.rb

@@ -0,0 +1,56 @@
+module NewspaperWorks
+  module Ingest
+    class NewspaperIssueIngest < BaseIngest
+      @configured = false
+
+      class << self
+        def configure
+          return if @configured == true
+          # PDF ingest may save page images to /tmp (via Dir.tmpdir), which
+          # needs whitelisting for use by NewspaperWorks::Data::WorkFiles.commit!
+          # via Hyrax CreateWithRemoteFilesActor:
+          whitelist = Hyrax.config.whitelisted_ingest_dirs
+          whitelist.push(Dir.tmpdir) unless whitelist.include?(Dir.tmpdir)
+          @configured = true
+        end
+      end
+
+      def import
+        # first, handle the PDF itself on the issue...
+        super
+        # ...then create child works from split pages
+        create_child_pages
+      end
+
+      # Creates child pages with attached TIFF masters, can be called by
+      #   `import`, or independently if `load` is called first.  The
+      #   latter is appropriate if framework is already handling the
+      #   NewspaperIssue file attachment (e.g. Hyrax upload via browser).
+      def create_child_pages
+        self.class.configure
+        pages = NewspaperWorks::Ingest::PdfPages.new(path).to_a
+        pages.each_with_index do |tiffpath, idx|
+          page = new_child_page_with_file(tiffpath, idx)
+          @work.ordered_members << page
+        end
+        @work.save!(validate: false) unless pages.empty?
+      end
+
+      def new_child_page_with_file(tiffpath, idx)
+        page_number = idx + 1
+        page = NewspaperPage.new
+        page.title = ["#{@work.title.first}: Page #{page_number}"]
+        # technically, a sequence number distinct from displayed page number
+        page.page_number = page_number.to_s
+        # Set depositor and admin-set id:
+        page.depositor = @work.depositor
+        page.admin_set_id = @work.admin_set_id
+        # copying permissions also by effect copies visibility:
+        page.permissions_attributes = @work.permissions.map(&:to_hash)
+        NewspaperPageIngest.new(page).ingest(tiffpath)
+        page.save!
+        page
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/newspaper_page_ingest.rb

@@ -0,0 +1,6 @@
+module NewspaperWorks
+  module Ingest
+    class NewspaperPageIngest < BaseIngest
+    end
+  end
+end