newspaper_works 0.1.0

data/lib/newspaper_works/ingest/base_ingest.rb

@@ -0,0 +1,69 @@
+require 'newspaper_works/data'
+
+module NewspaperWorks
+  module Ingest
+    # base class for ingesting works, implements, as-needed, temp files
+    class BaseIngest
+      include NewspaperWorks::Data::PathHelper
+
+      attr_accessor :work, :io, :path, :filename
+
+      def initialize(work)
+        # adapted context:
+        @work = work
+      end
+
+      def loadpath(source)
+        # quick check the file exists and is readable on filesystem:
+        raise ArgumentError, 'File not found or readable' unless
+          File.readable?(source)
+        # path may be relative to Dir.pwd, but no matter for our use
+        @path = source.to_s
+        @io = File.open(@path)
+        @filename ||= File.split(@path)[-1]
+      end
+
+      def loadio(source)
+        # either an IO with a path, or an IO with filename passed in
+        #   args; presume we need a filename to describe/identify.
+        raise ArgumentError, 'Explicit or inferred file name required' unless
+          source.respond_to?('path') || @filename
+        @io = source
+        @path = source.respond_to?('path') ? source.path : nil
+        @filename ||= File.split(@path)[-1]
+      end
+
+      def load(source, filename: nil)
+        # source is a string path, Pathname object, or quacks like an IO
+        unless source.class == String ||
+               source.class == Pathname ||
+               source.respond_to?('read')
+          raise ArgumentError, 'Source is neither path nor IO object'
+        end
+        # permit the possibility of a filename identifier metadata distinct
+        #   from the actual path on disk:
+        @filename = filename
+        ispath = source.class == String || source.class == Pathname
+        loader = ispath ? method(:loadpath) : method(:loadio)
+        loader.call(source)
+      end
+
+      # default handler attaches file to work's file set, subclasses
+      #   may overwride or wrap this.
+      def import
+        files = NewspaperWorks::Data::WorkFiles.new(work)
+        files.assign(path)
+        files.commit!
+      end
+
+      def user
+        defined?(current_user) ? current_user : User.batch_user
+      end
+
+      def ingest(source, filename: nil)
+        load(source, filename: filename)
+        import
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/base_publication_info.rb

@@ -0,0 +1,35 @@
+module NewspaperWorks
+  module Ingest
+    class BasePublicationInfo
+      attr_accessor :lccn, :issn
+
+      def initialize(lccn)
+        @lccn = lccn
+        load
+      end
+
+      def load
+        raise NotImplementedError, "abstract"
+      end
+
+      # Return normalized, prefixed OCLC number from numeric Integer or
+      #   String inputs; prefxes based on number of digits, leaves any
+      #   prefix in input unchanged.
+      # @param oclcnum [String, Integer] prefixed or unprefixed OCLC control #
+      # @return [String] normalized, prefixed OCLC number
+      def oclc_prefixed(oclcnum)
+        # unprefixed number, as string
+        digits = oclcnum.to_s.gsub(/[A-Za-z]/, '')
+        return "ocm#{digits}" if digits.size == 8
+        return "ocn#{digits}" if digits.size == 9
+        "on#{digits}"
+      end
+
+      def place_name_from_title(title)
+        parts = title.split(/ [\(]/)
+        return if parts.size < 2
+        parts[1].split(')')[0]
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/batch_ingest_helper.rb

@@ -0,0 +1,44 @@
+require 'find'
+
+module NewspaperWorks
+  module Ingest
+    # mixin module for common batch ingest steps
+    module BatchIngestHelper
+      def detect_media(path)
+        result = 'pdf' # default
+        Find.find(path) do |p|
+          result = 'image' if p.end_with?('jp2') || /TIF[F]?$/i.match(p)
+        end
+        result
+      end
+
+      def lccn_from_path(path)
+        File.basename(path)
+      end
+
+      def normalize_lccn(v)
+        p = /^[A-Za-z]{0,3}[0-9]{8}([0-9]{2})?$/
+        v = v.gsub(/\s+/, '').downcase.slice(0, 13)
+        raise ArgumentError, "LCCN appears invalid: #{v}" unless p.match(v)
+        v
+      end
+
+      def issue_title(issue_data)
+        issue_data.title
+      end
+
+      def copy_issue_metadata(source, target)
+        target.title = issue_title(source)
+        target.lccn = source.lccn
+        target.publication_date = source.publication_date
+        target.edition_number = source.edition_number
+      end
+
+      def attach_file(work, path)
+        attachment = NewspaperWorks::Data::WorkFiles.of(work)
+        attachment.assign(path)
+        attachment.commit!
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/batch_issue_ingester.rb

@@ -0,0 +1,129 @@
+require 'open3'
+require 'tmpdir'
+
+module NewspaperWorks
+  module Ingest
+    class BatchIssueIngester
+      # CLI constructor, related class methods:
+      extend NewspaperWorks::Ingest::FromCommand
+
+      include NewspaperWorks::Ingest::PubFinder
+      include NewspaperWorks::Ingest::BatchIngestHelper
+      include NewspaperWorks::Logging
+
+      attr_accessor :path, :lccn, :publication, :opts, :issues
+
+      def initialize(path, opts = {})
+        @path = path
+        lccn = opts[:lccn]
+        @lccn = normalize_lccn(lccn.nil? ? lccn_from_path(path) : lccn)
+        # get publication info for LCCN from authority web service:
+        @publication = NewspaperWorks::Ingest::PublicationInfo.new(@lccn)
+        # issues for publication, as enumerable of PDFIssue
+        @issues = issue_enumerator
+        @opts = opts
+        configure_logger('ingest')
+      end
+
+      def issue_enumerator
+        impl = NewspaperWorks::Ingest::PDFIssues
+        impl = NewspaperWorks::Ingest::ImageIngestIssues if detect_media(path) == 'image'
+        # issue enumerator depends on detected media:
+        impl.new(path, publication)
+      end
+
+      def link_publication(issue)
+        find_or_create_publication_for_issue(
+          issue,
+          @lccn,
+          @publication.title,
+          @opts
+        )
+      end
+
+      def create_issue(issue_data)
+        issue = NewspaperIssue.create
+        copy_issue_metadata(issue_data, issue)
+        NewspaperWorks::Ingest.assign_administrative_metadata(issue, @opts)
+        issue.save!
+        write_log(
+          "Created new NewspaperIssue work with date, lccn, edition metadata:"\
+          "\n"\
+          "\tLCCN: #{@lccn}\n"\
+          "\tPublication Date: #{issue_data.publication_date}\n"\
+          "\tEdition number: #{issue_data.edition_number}"
+        )
+        link_publication(issue)
+        issue
+      end
+
+      def ingest_pdf(issue, path)
+        # ingest primary PDF for issue:
+        attach_file(issue, path)
+        # queue page creation job:
+        CreateIssuePagesJob.perform_later(issue, [path], nil, nil)
+      end
+
+      def create_page(page_image, issue)
+        page = NewspaperPage.create
+        page.title = page_image.title
+        page.page_number = page_image.page_number
+        page.save!
+        # Link page as a child of issue, via ordered members:
+        issue.ordered_members << page
+        NewspaperWorks::Ingest.assign_administrative_metadata(page, @opts)
+        issue.save!
+        # Ensure we have a source TIFF file, attach to page:
+        path = page_image.path
+        path = page_image.path.end_with?('jp2') ? make_tiff(path) : path
+        attach_file(page, path)
+      end
+
+      def ingest_pages(issue, issue_data)
+        # Create pages in order they appear (lexical)
+        issue_data.each_value { |page_image| create_page(page_image, issue) }
+        # Make an issue PDF from constituent pages, via retryable async job,
+        #   which will not succeed until the PDF derivatives are created
+        #   for each page, but should eventually succeed on that condition:
+        NewspaperWorks::ComposeIssuePDFJob.perform_later(issue)
+      end
+
+      def make_tiff(path)
+        raise ArgumentError unless path.end_with?('jp2')
+        Hyrax.config.whitelisted_ingest_dirs |= [Dir.tmpdir]
+        name = File.basename(path).split('.')[0]
+        # OpenJPEG2000 has weird quirk, only likes 3-char file ext TIF:
+        tiff_path = File.join(Dir.mktmpdir, "#{name}.tif")
+        cmd = "opj_decompress -i #{path} -o #{tiff_path}"
+        Open3.popen3(cmd) do |_stdin, _stdout, stderr, _wait_thr|
+          unless stderr.read.strip.empty?
+            msg = "Error converting JP2 to TIFF: #{path}"
+            write_log(msg, Logger::ERROR)
+            raise msg
+          end
+        end
+        tiff_path
+      end
+
+      def ingest_type
+        return 'issue_pdf' if @issues.class == NewspaperWorks::Ingest::PDFIssues
+        'page_image'
+      end
+
+      def ingest
+        write_log("Beginning issue(s) batch ingest for #{@path}")
+        write_log("\tPublication: #{@publication.title} (LCCN: #{@lccn})")
+        @issues.each do |path, issue_data|
+          issue = create_issue(issue_data)
+          tactic = ingest_type
+          ingest_pdf(issue, path) if tactic == 'issue_pdf'
+          ingest_pages(issue, issue_data) if tactic == 'page_image'
+        end
+        write_log(
+          "Issue ingest completed for LCCN #{@lccn}. Asyncrhonous jobs "\
+          "may still be creating derivatives for issue, and child page works."
+        )
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/chronam_publication_info.rb

@@ -0,0 +1,133 @@
+require 'faraday'
+require 'nokogiri'
+
+module NewspaperWorks
+  module Ingest
+    # Publication info from ChronAm as remote authority for metadata
+    class ChronAmPublicationInfo < BasePublicationInfo
+      attr_accessor :issn, :title, :place_name, :place_of_publication, :language
+
+      XML_NS = {
+        dcterms: 'http://purl.org/dc/terms/',
+        frbr: 'http://purl.org/vocab/frbr/core#',
+        owl: 'http://www.w3.org/2002/07/owl#',
+        rda: 'http://rdvocab.info/elements/',
+        rdf: 'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
+        rdfs: 'http://www.w3.org/2000/01/rdf-schema#'
+      }.freeze
+
+      BASE_URL = 'https://chroniclingamerica.loc.gov/lccn'.freeze
+
+      def initialize(lccn)
+        # true until loaded
+        @empty = true
+        super(lccn)
+        @issn = nil # chronam doesn't have this
+      end
+
+      def empty?
+        @empty
+      end
+
+      def inspect
+        format(
+          "<#{self.class}:0x000000000%<oid>x " \
+            "\tlccn: '#{@lccn}'>",
+          oid: object_id << 1
+        )
+      end
+
+      def load_place
+        place_match = find('//rda:placeOfPublication')
+        return if place_match.nil?
+        @place_name = place_match.first.text
+        @place_of_publication = NewspaperWorks::Ingest.geonames_place_uri(
+          @place_name
+        )
+      end
+
+      def url
+        "#{BASE_URL}/#{@lccn}.rdf"
+      end
+
+      def load
+        resp = NewspaperWorks::ResourceFetcher.get url
+        return if resp['status'] == 404
+        @doc = Nokogiri.XML(resp['body'])
+        @title = normalize_title(find('//dcterms:title').first.text)
+        @language = iso_language_for(find('//dcterms:language').first.text)
+        @empty = false
+        load_place
+      end
+
+      def oclcnum
+        key = 'info:oclcnum'
+        selected = sameas_resources.select { |v| v.text.start_with?(key) }
+        return if selected.empty?
+        oclc_prefixed(selected[0].text.split('/')[1])
+      end
+
+      def preceded_by
+        return if empty?
+        found = find('//frbr:successorOf/@rdf:resource').first
+        return if found.nil?
+        normalize_related(found.text)
+      end
+
+      def succeeded_by
+        return if empty?
+        found = find('//frbr:successor/@rdf:resource').first
+        return if found.nil?
+        normalize_related(found.text)
+      end
+
+      private
+
+        def normalize_title(value)
+          NewspaperWorks::Ingest.normalize_title(value)
+        end
+
+        # Returns URL to LC catalog, provided such exists, on the basis of
+        #   non-empty MODS for given LCCN.  Otherwise returns nil.
+        def lc_catalog_url(lccn)
+          content_url = "https://lccn.loc.gov/#{lccn}"
+          url = "#{content_url}/mods"
+          resp = NewspaperWorks::ResourceFetcher.get url
+          doc = Nokogiri.XML(resp['body'])
+          return content_url unless doc.root.children.empty?
+        end
+
+        def normalize_related(value)
+          lccn = value.split('/')[-1].split('#')[0]
+          lc_url = lc_catalog_url(lccn)
+          # URL to lccn.loc.gov is preferred authority for publication URL
+          return lc_url unless lc_url.nil?
+          # URL to HTML representation of content on ChronAm is fallback
+          "#{BASE_URL}/#{lccn}"
+        end
+
+        def sameas_resources
+          find('//owl:sameAs/@rdf:resource') || []
+        end
+
+        def find(expr, context = nil)
+          context ||= @doc
+          return if context.nil?
+          context.xpath(expr, **XML_NS)
+        end
+
+        # ISO 639-2 three-character code from ISO 639-1 two-character code
+        #   or equivalent lingvoj resource URL used by ChronAm;
+        #   uses HTML language tables maintained by LOC.
+        def iso_language_for(code)
+          # handle case where source language code is lingvoj url:
+          code = code.split('/')[-1]
+          lookup_url = 'https://www.loc.gov/standards/iso639-2/php/langcodes_name.php'
+          lookup_url += "?iso_639_1=#{code}"
+          resp = NewspaperWorks::ResourceFetcher.get lookup_url
+          html = Nokogiri::HTML(resp['body'])
+          html.xpath('//table[1]/tr[2]/td[2]').first.text.strip
+        end
+    end
+  end
+end

data/lib/newspaper_works/ingest/from_command.rb

@@ -0,0 +1,52 @@
+module NewspaperWorks
+  module Ingest
+    # class-method mixin module for ingest command-line invocation
+    #   usage in classes: `extend NewspaperWorks::Ingest::FromCommand`
+    #   These are all expected to be class methods in various CLI ingests.
+    module FromCommand
+      # alternate constructor from ARGV
+      # @param options [Array<String>]
+      def from_command(options, cmd_name)
+        path, opts = batch_path(options, cmd_name)
+        missing_path(cmd_name) if path.nil?
+        path = normalize_path(path)
+        missing_path(cmd_name, "Not found: #{path}") unless File.exist?(path)
+        Hyrax.config.whitelisted_ingest_dirs.push(File.dirname(path))
+        new(path, opts)
+      end
+
+      def missing_path(cmd_name, msg = "Missing path argument")
+        STDERR.puts "Usage: #{cmd_name} -- --path=PATH"
+        STDERR.puts "#{msg}. Exiting."
+        # rubocop:disable Rails/Exit
+        exit(1) if cmd_name.start_with?('rake')
+        # rubocop:enable Rails/Exit
+      end
+
+      def batch_path(options, cmd_name)
+        path = nil
+        params = {}
+        parser = OptionParser.new
+        args = parser.order!(options) {}
+        parser.banner = "Usage: #{cmd_name} -- --path=PATH"
+        parser.on('-i PATH', '--path PATH') do |p|
+          path = p
+        end
+        parser.on('--admin_set=ADMIN_SET')
+        parser.on('--depositor=DEPOSITOR')
+        parser.on('--visibility=VISIBILITY')
+        # lccn used by PDF issue ingest, but not NDNP ingest:
+        parser.on('--lccn=LCCN')
+        parser.parse!(args, into: params)
+        [path, params]
+      end
+
+      # default normalization is no normalization of path
+      # @param path [String]
+      # @return [String]
+      def normalize_path(path)
+        path
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/image_ingest_issues.rb

@@ -0,0 +1,43 @@
+module NewspaperWorks
+  module Ingest
+    class ImageIngestIssues
+      include Enumerable
+      include NewspaperWorks::Ingest::PathEnumeration
+
+      attr_accessor :path, :publication
+
+      delegate :lccn, to: :publication
+
+      def initialize(path, publication)
+        # path is path to publication directory containing issues:
+        @path = path
+        # Publication info
+        @publication = publication
+        @issue_paths = nil
+      end
+
+      def paths
+        return @issue_paths unless @issue_paths.nil?
+        result = []
+        entries = Dir.entries(path).map { |n| File.join(path, n) }
+        entries.select { |p| !File.basename(p).start_with?('.') }.each do |p|
+          next unless File.directory?(p)
+          next unless path_validates?(p)
+          result.push(p)
+        end
+        @issue_paths = result
+      end
+
+      def info(path)
+        NewspaperWorks::Ingest::IssueImages.new(path, @publication)
+      end
+
+      private
+
+        def path_validates?(p)
+          ptn = /^([0-9]{4})(1[012]|[0][1-9])(3[01]|[12][0-9]|0[1-9])([0-9]{2})?/
+          ptn.match(File.basename(p)) ? true : false
+        end
+    end
+  end
+end

data/lib/newspaper_works/ingest/issue_images.rb

@@ -0,0 +1,51 @@
+require 'date'
+require 'find'
+
+module NewspaperWorks
+  module Ingest
+    # Represents TIFF/JP2 issue, provides metadata, enumerates PageImage objects
+    class IssueImages
+      # most acccessors for issue/edition metadata, publication metadata
+      #   provided by including this mixin:
+      include NewspaperWorks::Ingest::NamedIssueMetadata
+
+      # Path enumeration by mixing in Enumerable, PathEnumeration
+      include Enumerable
+      include NewspaperWorks::Ingest::PathEnumeration
+
+      attr_accessor :path, :publication
+
+      # things that look like images, by file extension:
+      IMAGE_EXT = ['tiff', 'tif', 'jp2', 'jpg', 'png'].freeze
+
+      def initialize(path, publication)
+        @path = path
+        raise ArgumentError, 'Path not directory' unless File.directory?(path)
+        validate_path
+        # as a NewspaperWorks::Ingest::PublicationInfo object:
+        @publication = publication
+        @pages = nil
+      end
+
+      def page_paths
+        return @pages unless @pages.nil?
+        @pages = []
+        entries = Dir.entries(path).map { |n| File.join(path, n) }
+        entries.sort.each do |p|
+          next unless File.ftype(p) == 'file'
+          ext = File.basename(p).downcase.split('.')[-1]
+          next unless IMAGE_EXT.include?(ext)
+          @pages.push(p)
+        end
+        @pages
+      end
+
+      def info(path)
+        page_seq_num = page_paths.index(path) + 1
+        NewspaperWorks::Ingest::PageImage.new(path, self, page_seq_num)
+      end
+
+      alias paths page_paths
+    end
+  end
+end