newspaper_works 0.1.0

data/lib/newspaper_works/ingest/page_image.rb

@@ -0,0 +1,52 @@
+module NewspaperWorks
+  module Ingest
+    # Represents TIFF/JP2 page, access to file, page-numbering metadata
+    class PageImage
+      attr_accessor :path, :issue, :sequence
+
+      delegate :lccn, to: :issue
+
+      def initialize(path, issue, sequence)
+        # path to image:
+        @path = path
+        validate_path
+        # Issue is NewspaperWorks::Ingest::IssueImages object
+        @issue = issue
+        # sequence is page sequence number (Integer)
+        @sequence = sequence.to_i
+      end
+
+      # Page number inferred from image filename, or nil, presuming that:
+      #   - The page number follows the actual word "page" (case-insenstive)
+      #     in filename, possibly separated by a dash or underscore.
+      #   - The page number is terminated by the period-plus-file-extension.
+      #   - Both of the above can be determined by regular expression match.
+      #   - Extraneous leading information in filename (e.g. datestamp) will
+      #     be ignored.
+      #   - Examples:
+      #     - 'Page1.tiff'
+      #     - '2019091801-page_1.jp2'
+      #     - 'page_C2.tiff'
+      # @return [String, NilClass] page number string, or nil if indecipherable
+      def named_page_number
+        pattern = /(page)([_-]?)([^.]+)([.])/i
+        match = pattern.match(path)
+        match.nil? ? nil : match[3]
+      end
+
+      def page_number
+        named_page_number || @sequence.to_s
+      end
+
+      def title
+        ["#{@issue.title.first}: Page #{page_number}"]
+      end
+
+      def validate_path
+        # expect path to be regular file, that exists:
+        raise ArgumentError unless File.exist?(path)
+        raise ArgumentError unless File.ftype(path) == 'file'
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/path_enumeration.rb

@@ -0,0 +1,52 @@
+module NewspaperWorks
+  module Ingest
+    # Provides enumeration of path keys to object values, where:
+    #   - Consuming class:
+    #     - Defines a `paths` method returning array of paths.
+    #     - Defines an `info` method that returns an object for a path.
+    #     - Also mixes in Enumerable
+    module PathEnumeration
+      delegate :size, :include?, to: :_paths
+
+      def _paths
+        paths
+      end
+
+      def _info(path)
+        info(path)
+      end
+
+      def each
+        return enum_for(:each) unless block_given?
+        paths.each do |path|
+          yield [path, info(path)]
+        end
+      end
+
+      def each_key
+        enum_for(:each_key) unless block_given?
+        paths.each { |path| yield path }
+      end
+
+      def each_value
+        return enum_for(:each_value) unless block_given?
+        paths.each do |path|
+          yield info(path)
+        end
+      end
+
+      def values
+        each_value.to_a
+      end
+
+      def entries
+        each.to_a
+      end
+
+      alias each_pair each
+      alias keys _paths
+      alias has_key? include?
+      alias [] _info
+    end
+  end
+end

data/lib/newspaper_works/ingest/pdf_images.rb

@@ -0,0 +1,85 @@
+require 'open3'
+require 'mini_magick'
+
+module NewspaperWorks
+  module Ingest
+    # PdfImages uses poppler 0.19+ pdfimages command to extract image
+    #   listing metadata from PDF files.
+    #   For dpi extraction, falls back to calculating using MiniMagick,
+    #   if neccessary.
+    class PdfImages
+      # class constant column numbers
+      COL_WIDTH = 3
+      COL_HEIGHT = 4
+      COL_COLOR = 5
+      COL_CHANNELS = 6
+      COL_BITS = 7
+      # only poppler 0.25+ has this column in output:
+      COL_XPPI = 12
+
+      def initialize(path)
+        @path = path
+        @cmd = format('pdfimages -list %<path>s', path: path)
+        @output = nil
+        @entries = nil
+      end
+
+      def process
+        # call just once
+        if @output.nil?
+          Open3.popen3(@cmd) do |_stdin, stdout, _stderr, _wait_thr|
+            @output = stdout.read.split("\n")
+          end
+        end
+        @output.slice(2, @output.size - 1)
+      end
+
+      def entries
+        if @entries.nil?
+          @entries = []
+          output = process
+          (0..output.size - 1).each do |i|
+            @entries.push(output[i].gsub(/\s+/m, ' ').strip.split(" "))
+          end
+        end
+        @entries
+      end
+
+      def selectcolumn(i, &block)
+        result = entries.map { |e| e[i] }
+        return result.map!(&block) if block_given?
+        result
+      end
+
+      def width
+        selectcolumn(COL_WIDTH, &:to_i).max
+      end
+
+      def height
+        selectcolumn(COL_HEIGHT, &:to_i).max
+      end
+
+      def color
+        # desc is either 'gray', 'cmyk', 'rgb', but 1-bit gray is black/white
+        #   so caller may want all of this information, and in case of
+        #   mixed color spaces across images, this returns maximum
+        desc = entries.any? { |e| e[COL_COLOR] != 'gray' } ? 'rgb' : 'gray'
+        channels = entries.map { |e| e[COL_CHANNELS].to_i }.max
+        bits = entries.map { |e| e[COL_BITS].to_i }.max
+        [desc, channels, bits]
+      end
+
+      def ppi
+        if entries[0].size <= 12
+          # poppler < 0.25
+          pdf = MiniMagick::Image.open(@path)
+          width_points = pdf.width
+          width_px = width
+          return (72 * width_px / width_points).to_i
+        end
+        # with poppler 0.25+, pdfimages just gives us this:
+        selectcolumn(COL_XPPI, &:to_i).max
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/pdf_issue.rb

@@ -0,0 +1,20 @@
+require 'date'
+
+module NewspaperWorks
+  module Ingest
+    class PDFIssue
+      attr_accessor :path, :publication
+
+      # most acccessors for issue/edition metadata, publication metadata
+      #   provided by including this mixin:
+      include NewspaperWorks::Ingest::NamedIssueMetadata
+
+      def initialize(path, publication)
+        @path = path
+        validate_path
+        # as a NewspaperWorks::Ingest::PublicationInfo object:
+        @publication = publication
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/pdf_issues.rb

@@ -0,0 +1,39 @@
+require 'find'
+
+module NewspaperWorks
+  module Ingest
+    class PDFIssues
+      include Enumerable
+      include NewspaperWorks::Ingest::PathEnumeration
+
+      attr_accessor :path, :publication, :pdf_paths
+
+      alias paths pdf_paths
+
+      def initialize(path, publication)
+        @path = path
+        # as a NewspaperWorks::Ingest::PublicationInfo object:
+        @publication = publication
+        @pdf_paths = valid_pdfs(path)
+      end
+
+      def valid_pdfs(path)
+        target = []
+        Find.find(path) do |p|
+          next if File.directory?(p)
+          next unless p.end_with?('.pdf')
+          target.push(p)
+        end
+        target
+      end
+
+      def lccn
+        @publication.lccn
+      end
+
+      def info(path)
+        NewspaperWorks::Ingest::PDFIssue.new(path, @publication)
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/pdf_pages.rb

@@ -0,0 +1,114 @@
+require 'open3'
+require 'securerandom'
+require 'tmpdir'
+
+module NewspaperWorks
+  module Ingest
+    class PdfPages
+      include Enumerable
+
+      def initialize(path)
+        @baseid = SecureRandom.uuid
+        @pdfpath = path
+        @info = nil
+        @entries = nil
+        @tmpdir = nil
+        @size = nil
+        @pagecount = nil
+        @pdftext = nil
+      end
+
+      # return
+      def pdfinfo
+        @info = PdfImages.new(@pdfpath) if @info.nil?
+        @info
+      end
+
+      def tmpdir
+        @tmpdir = Dir.mktmpdir if @tmpdir.nil?
+        @tmpdir
+      end
+
+      def colordevice(channels, bpc)
+        bits = bpc * channels
+        # will be either 8bpc/16bpd color TIFF,
+        #   with any CMYK source transformed to 8bpc RBG
+        bits = 24 unless [24, 48].include? bits
+        "tiff#{bits}nc"
+      end
+
+      def gsdevice
+        color, channels, bpc = pdfinfo.color
+        device = nil
+        # CCITT Group 4 Black and White, if applicable:
+        device = 'tiffg4' if color == 'gray' && bpc == 1
+        # 8 Bit Grayscale, if applicable:
+        device = 'tiffgray' if color == 'gray' && bpc > 1
+        # otherwise color:
+        device = colordevice(channels, bpc) if device.nil?
+        device
+      end
+
+      def gstext
+        cmd = "gs -q -dNOPAUSE -dBATCH -sDEVICE=txtwrite " \
+              "-sOutputFile=- -f #{@pdfpath}"
+        Open3.popen3(cmd) do |_stdin, stdout, _stderr, _wait_thr|
+          @pdftext = stdout.read
+        end
+        @pdftext
+      end
+
+      def pagecount
+        cmd = "pdfinfo #{@pdfpath}"
+        Open3.popen3(cmd) do |_stdin, stdout, _stderr, _wait_thr|
+          output = stdout.read.split("\n")
+          pages_e = output.select { |e| e.start_with?('Pages:') }[0]
+          @pagecount = pages_e.split[-1].to_i
+        end
+        @pagecount
+      end
+
+      def looks_scanned
+        max_image_px = pdfinfo.width * pdfinfo.height
+        single_image_per_page = pdfinfo.entries.length == pagecount
+        # single 10mp+ image per page?
+        single_image_per_page && max_image_px > 1024 * 1024 * 10
+      end
+
+      def ppi
+        unless looks_scanned
+          # 400 dpi for something that does not look like scanned media:
+          return 400
+        end
+        # For scanned media, defer to detected image PPI:
+        pdfinfo.ppi
+      end
+
+      # ghostscript convert all pages to TIFF
+      def gsconvert
+        output_base = File.join(tmpdir, "#{@baseid}-page%d.tiff")
+        cmd = "gs -dNOPAUSE -dBATCH -sDEVICE=#{gsdevice} " \
+              "-dTextAlphaBits=4 " \
+              "-sOutputFile=#{output_base} -r#{ppi} -f #{@pdfpath}"
+        Open3.popen3(cmd) do |_stdin, stdout, _stderr, _wait_thr|
+          output = stdout.read.split("\n")
+          @size = output.select { |e| e.start_with?('Page ') }.length
+        end
+        # Return an array of expected filenames
+        (1..@size).map { |n| File.join(tmpdir, "#{@baseid}-page#{n}.tiff") }
+      end
+
+      # entries for each page
+      def entries
+        @entries = gsconvert if @entries.nil?
+        @entries
+      end
+
+      def each
+        entries.each do |e|
+          yield(e)
+        end
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/pub_finder.rb

@@ -0,0 +1,89 @@
+require 'newspaper_works/logging'
+require 'newspaper_works/ingest'
+
+module NewspaperWorks
+  module Ingest
+    # mixin for find-or-create of publication, for use by various ingests
+    module PubFinder
+      include NewspaperWorks::Logging
+
+      COPY_FIELDS = [
+        :title,
+        :lccn,
+        :oclcnum,
+        :issn,
+        :place_of_publication,
+        :language,
+        :preceded_by,
+        :succeeded_by
+      ].freeze
+
+      MULTI_VALUED = [
+        :title,
+        :language,
+        :preceded_by,
+        :succeeded_by,
+        :place_of_publication
+      ].freeze
+
+      WRAPPERS = {
+        place_of_publication: Hyrax::ControlledVocabularies::Location
+      }.freeze
+
+      # @param lccn [String] Library of Congress Control Number
+      #   of Publication
+      # @return [NewspaperTitle, NilClass] publication or nil if not found
+      def find_publication(lccn)
+        NewspaperTitle.where(lccn: lccn).first
+      end
+
+      # Copy publication metadata from authority lookup for LCCN
+      # @param publication [NewspaperTitle]
+      # @param metadata [NewspaperWorks::Ingest::PublicationInfo]
+      def copy_publication_metadata(publication, metadata, lccn, title = nil)
+        COPY_FIELDS.each do |name|
+          value = metadata.send(name)
+          next if value.nil?
+          # wrapped value, if applicable:
+          value = WRAPPERS[name].new(value) if WRAPPERS.include?(name)
+          # value in array, if applicable:
+          value = [value] if MULTI_VALUED.include?(name)
+          publication.send("#{name}=", value)
+        end
+        # prefer locally-specified title to looked-up title:
+        publication.title = [title] unless title.nil?
+        # final fallback, nothing specified, title mandatory: use LCCN
+        publication.title = [lccn] if publication.title.empty?
+      end
+
+      def create_publication(lccn, title = nil, opts = {})
+        publication = NewspaperTitle.create
+        info = NewspaperWorks::Ingest::PublicationInfo.new(lccn)
+        copy_publication_metadata(publication, info, lccn, title)
+        publication.lccn ||= lccn
+        NewspaperWorks::Ingest.assign_administrative_metadata(publication, opts)
+        publication.save!
+        write_log(
+          "Created NewspaperTitle work #{publication.id} for LCCN #{lccn}"
+        )
+        publication
+      end
+
+      def find_or_create_publication_for_issue(issue, lccn, title, opts)
+        publication = find_publication(lccn)
+        unless publication.nil?
+          write_log(
+            "Found existing NewspaperTitle #{publication.id}, LCCN #{lccn}"
+          )
+        end
+        publication = create_publication(lccn, title, opts) if publication.nil?
+        publication.members << issue
+        publication.save!
+        write_log(
+          "Linked NewspaperIssue #{issue.id} to "\
+          "NewspaperTitle work #{publication.id}"
+        )
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/publication_info.rb

@@ -0,0 +1,44 @@
+require 'faraday'
+require 'nokogiri'
+require 'uri'
+
+module NewspaperWorks
+  module Ingest
+    class PublicationInfo
+      attr_accessor :implementation, :lccn
+
+      def initialize(lccn)
+        @lccn = lccn
+        @implementation = nil
+        load
+      end
+
+      def load_chronam_fallback
+        @implementation = ChronAmPublicationInfo.new(@lccn)
+      end
+
+      def load
+        @implementation = LCPublicationInfo.new(@lccn)
+        @implementation.load
+        # Empty mods is equivalent to 404 for LCCN in LC Catalog:
+        load_chronam_fallback if @implementation.empty?
+      end
+
+      def respond_to_missing?(symbol, include_priv = false)
+        @implementation.respond_to?(symbol, include_priv)
+      end
+
+      def method_missing(method, *args, &block)
+        # proxy call to underlying implementation:
+        if respond_to_missing?(method)
+          return @implementation.send(
+            method,
+            *args,
+            &block
+          )
+        end
+        super
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest.rb

@@ -0,0 +1,90 @@
+require 'faraday'
+require 'nokogiri'
+require 'uri'
+require 'newspaper_works/ingest/from_command'
+require 'newspaper_works/ingest/base_publication_info'
+require 'newspaper_works/ingest/chronam_publication_info'
+require 'newspaper_works/ingest/lc_publication_info'
+require 'newspaper_works/ingest/publication_info'
+require 'newspaper_works/ingest/pub_finder'
+require 'newspaper_works/ingest/pdf_images'
+require 'newspaper_works/ingest/named_issue_metadata'
+require 'newspaper_works/ingest/path_enumeration'
+require 'newspaper_works/ingest/pdf_issue'
+require 'newspaper_works/ingest/pdf_issues'
+require 'newspaper_works/ingest/batch_ingest_helper'
+require 'newspaper_works/ingest/batch_issue_ingester'
+require 'newspaper_works/ingest/pdf_pages'
+require 'newspaper_works/ingest/issue_images'
+require 'newspaper_works/ingest/page_image'
+require 'newspaper_works/ingest/image_ingest_issues'
+require 'newspaper_works/ingest/base_ingest'
+require 'newspaper_works/ingest/ndnp'
+require 'newspaper_works/ingest/newspaper_page_ingest'
+require 'newspaper_works/ingest/newspaper_issue_ingest'
+
+module NewspaperWorks
+  # Module for Ingest adapters that import files into model objects
+  module Ingest
+    # Get Geonames URI for closest place match
+    #   Requires Qa::Authorities::Geonames.username is set, likely via
+    #   `Hyrax.config.geonames_username=` setter in
+    #   config/initializers/hyrax.rb of consuming app.
+    # @param place_name [String] Name of place as human-readable text
+    # @return [String, NilClass] URI to Geonames RDF or nil
+    def self.geonames_place_uri(place_name)
+      username = Qa::Authorities::Geonames.username
+      return if username.nil? || username.empty?
+      place_name = place_name.delete('.').split(/[\[\(]/)[0].strip
+      query = URI.encode(place_name)
+      geo_qs = "q=#{query}&username=#{username}"
+      url = "http://api.geonames.org/search?#{geo_qs}"
+      resp = NewspaperWorks::ResourceFetcher.get url
+      doc = Nokogiri.XML(resp['body'])
+      geonames_id = doc.xpath('//geonames/geoname[1]/geonameId').first
+      return if geonames_id.nil?
+      "http://sws.geonames.org/#{geonames_id.text}/"
+    end
+
+    # Normalize publication title from catalog data
+    #   Presently strips trailing period
+    # @param title [String]
+    # @return [String] normalized title
+    def self.normalize_title(title)
+      title.strip.sub(/[.]+$/, '')
+    end
+
+    # Get publication metadata from LC catalog MODS data, if available,
+    #   and from ChronAm, as a fallback.
+    # @param lccn [String] Library of Congress Control number for publication
+    # @return [NewspaperWorks::Ingest::PublicationInfo] proxy to metadata
+    #   source, an object for accessors for publication fields.
+    def self.publication_metadata(lccn)
+      PublicationInfo.new(lccn)
+    end
+
+    def self.find_admin_set(admin_set = nil)
+      return admin_set if admin_set.class == AdminSet
+      admin_set = AdminSet::DEFAULT_ID if admin_set.nil?
+      begin
+        AdminSet.find(admin_set)
+      rescue
+        # only create if default admin set
+        raise unless admin_set == AdminSet::DEFAULT_ID
+        AdminSet.find(AdminSet.find_or_create_default_admin_set_id)
+      end
+    end
+
+    def self.assign_administrative_metadata(work, opts = {})
+      work.depositor = opts.fetch(:email, User.batch_user.user_key)
+      work.admin_set = find_admin_set(opts.fetch(:admin_set, nil))
+      work.visibility = opts.fetch(:visibility, 'open')
+      work.resource_type = ['Newspapers']
+      work.date_modified ||= Hyrax::TimeService.time_in_utc
+      work.date_uploaded ||= work.date_modified
+      work.state = RDF::URI(
+        'http://fedora.info/definitions/1/0/access/ObjState#active'
+      )
+    end
+  end
+end

data/lib/newspaper_works/issue_pdf_composer.rb

@@ -0,0 +1,111 @@
+require 'open3'
+require 'tmpdir'
+
+module NewspaperWorks
+  # Adapter class composes a PDF derivative for issue, if it requires one.
+  class IssuePDFComposer
+    attr_accessor :issue, :page_pdfs
+
+    CMD_BASE = "gs -dBATCH -dNOPAUSE -q -sDEVICE=pdfwrite".freeze
+
+    # @param issue [NewspaperIssue] adapts issue work object
+    def initialize(issue)
+      @issue = issue
+      # paths to page PDFs
+      @page_pdfs = []
+    end
+
+    def compose
+      # we will not step on any existing PDF
+      return if issue_pdf_exists?
+      # we can not compose a multi-page issue PDF if constituent page PDFs
+      #   do not exist (yet == not ready, possibly waiting on an async job).
+      @page_pdfs = validated_page_pdfs
+      # Compose a Ghostscript command to merge all paths in @page_pdfs into
+      #   a single output document, execute:
+      compose_from_pages
+    end
+
+    def compose_from_pages
+      outfile = File.join(Dir.mktmpdir, output_filename)
+      sources = @page_pdfs.join(' ')
+      cmd = "#{CMD_BASE} -sOutputFile=#{outfile} #{sources}"
+      # rubocop:disable Lint/UnusedBlockArgument
+      Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thr|
+        unless wait_thr.value.success?
+          e = "Ghostscript Error: \n#{stderr.read}"
+          raise NewspaperWorks::DataError, e
+        end
+      end
+      # rubocop:enable Lint/UnusedBlockArgument
+      # at this point, something should exist and validate at path `outfile`:
+      raise NewspaperWorks::DataError, "Generated PDF invalid" unless validate_pdf(outfile)
+      # Assign for attachment to issue, commit:
+      attach_to_issue(outfile)
+    end
+
+    def output_filename
+      "#{@issue.id}_full-issue.pdf"
+    end
+
+    # Validate PDF with poppler `pdfinfo` command, which will detect
+    #   error conditions in cases like truncated PDF, and only in those
+    #   error conditions will write to stderr.
+    # @param path [String] path to PDF file
+    # @return [Boolean] true or false
+    def validate_pdf(path)
+      return false if path.nil? || !File.exist?(path)
+      return false if File.size(path).zero?
+      result = ''
+      cmd = "pdfinfo #{path}"
+      # rubocop:disable Lint/UnusedBlockArgument
+      Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thr|
+        result = stderr.read
+      end
+      # rubocop:enable Lint/UnusedBlockArgument
+      # only zero bytes stderr output from `pdfinfo` considered valid PDF:
+      result.size.zero?
+    end
+
+    private
+
+      # @return [Array] list of paths to page PDFs, in page order
+      # @raises [NewspaperWorks::PagesNotReady] if any page has invalid
+      #   or non-ready PDF source.
+      def validated_page_pdfs
+        result = []
+        # if any page PDF invalid, raise; otherwise append to result:
+        issue.pages.to_a.each_with_index do |page, idx|
+          e = "Page PDFs not ready for issue "\
+            "(Issue id: #{issue.id}, Page index: #{idx})"
+          path = derivatives_of(page).path('pdf')
+          raise NewspaperWorks::PagesNotReady, e unless validate_pdf(path)
+          result.push(path)
+        end
+        result
+      end
+
+      def issue_pdf_exists?
+        derivatives_of(@issue).exist?('pdf')
+      end
+
+      def derivatives_of(work)
+        NewspaperWorks::Data::WorkDerivatives.of(work)
+      end
+
+      def ensure_whitelist
+        whitelist = Hyrax.config.whitelisted_ingest_dirs
+        whitelist.push(Dir.tmpdir) unless whitelist.include?(Dir.tmpdir)
+      end
+
+      def attach_to_issue(path)
+        ensure_whitelist
+        # We rely upon WorkFiles to create fileset, and by consequence of
+        #   running primary file attachment through actor stack,
+        #   visibility of the FileSet is copied from the work:
+        attachment = NewspaperWorks::Data::WorkFiles.of(@issue)
+        attachment.assign(path)
+        attachment.commit!
+      end
+  end
+end