newspaper_works 0.1.0

data/lib/newspaper_works/logging.rb

@@ -0,0 +1,54 @@
+module NewspaperWorks
+  module Logging
+    class << self
+      attr_accessor :configured
+    end
+    self.configured = []
+
+    def logger
+      @logger = Rails.logger
+    end
+
+    # Log message, as in standard logger, but use message_format on message.
+    # @param severity [Integer] log level/severity, e.g. Logger::INFO == 2
+    # @param msg [String] Log message to be formatted by message_format
+    # @param progname [String] (optional)
+    def log(severity, msg, progname = nil, &block)
+      logger.add(severity, message_format(msg), progname, &block)
+    end
+
+    # Simpler alternative to .log, with default severity, message_format
+    #   wrapping.
+    # @param msg [String] Log message to be formatted by message_format
+    # @param severity [Integer] log level/severity, e.g. Logger::INFO == 2
+    # @param progname [String]
+    def write_log(msg, severity = Logger::INFO, progname = nil)
+      logger.add(severity, message_format(msg), progname)
+    end
+
+    # format message, distinct from per-output formatting, to be used in
+    #   all logging channels Rails.logger broadcasts to.  This wrapping
+    #   indicates in parenthetical prefix which class is acting to
+    #   produce message.
+    # @param msg [String]
+    def message_format(msg)
+      "(#{self.class}) #{msg}"
+    end
+
+    # Should be called by consuming class, prior to use of .logger method
+    #   has checks to prevent duplicate configuration if already configured.
+    def configure_logger(name)
+      @logger = Rails.logger
+      return if NewspaperWorks::Logging.configured.include?(name)
+      path = Rails.root.join("log/#{name}.log")
+      @named_log = ActiveSupport::Logger.new(path)
+      @named_log.formatter = proc do |_severity, datetime, _progname, msg|
+        "#{datetime}: #{msg}\n"
+      end
+      # rails will log to named_log in addition to any other configured
+      #   or default logging destinations:
+      @logger.extend(ActiveSupport::Logger.broadcast(@named_log))
+      NewspaperWorks::Logging.configured.push(name)
+    end
+  end
+end

data/lib/newspaper_works/page_finder.rb

@@ -0,0 +1,62 @@
+# useful methods for retrieving and ordering NewspaperPage objects
+module NewspaperWorks
+  module PageFinder
+    ##
+    # find all pages for an issue, return in order
+    # @param issue_id [String]
+    # @return [Array] ordered NewspaperPage SolrDocuments for an issue
+    def pages_for_issue(issue_id)
+      solr_params = ["has_model_ssim:\"NewspaperPage\""]
+      solr_params << "issue_id_ssi:\"#{issue_id}\""
+      solr_resp = Blacklight.default_index.search(fq: solr_params.join(' AND '))
+      all_pages = solr_resp.documents
+      return [] if all_pages.blank?
+      ordered_pages(all_pages)
+    end
+
+    ##
+    # return an ordered array of NewspaperPage documents
+    # @param documents [Array] NewspaperPage SolrDocuments for an issue
+    # @return [Array] ordered NewspaperPage SolrDocuments for an issue
+    def ordered_pages(documents)
+      return documents if documents.length <= 1
+      ordered_list = []
+      next_page_id, final_page_id = nil
+      documents.each do |doc|
+        if doc['is_following_page_of_ssi'].blank?
+          ordered_list.insert(0, doc)
+          next_page_id = doc['is_preceding_page_of_ssi']
+        elsif doc['is_preceding_page_of_ssi'].blank?
+          ordered_list.insert(-1, doc)
+          final_page_id = doc['id']
+        end
+      end
+      return documents if next_page_id.nil?
+      while next_page_id != final_page_id
+        next_page = documents.select { |doc| doc['id'] == next_page_id }.first
+        ordered_list.insert(-2, next_page)
+        next_page_id = next_page['is_preceding_page_of_ssi']
+      end
+      ordered_list
+    end
+
+    ##
+    # return the index of the current page
+    # @param page_id [String] id of the NewspaperPage
+    # @param issue_id [String] id of the parent NewspaperIssue
+    # @return [Integer] the page's index
+    def get_page_index(page_id, issue_id = nil)
+      default_index = 0
+      unless issue_id
+        page_doc = SolrDocument.find(page_id)
+        return default_index unless page_doc &&
+                                    page_doc['issue_id_ssi'] &&
+                                    page_doc['is_following_page_of_ssi']
+        issue_id = page_doc['issue_id_ssi']
+      end
+      all_pages = pages_for_issue(issue_id)
+      return default_index if all_pages.blank?
+      all_pages.index { |page| page['id'] == page_id }
+    end
+  end
+end

data/lib/newspaper_works/resource_fetcher.rb

@@ -0,0 +1,78 @@
+module NewspaperWorks
+  # in-memory caching fetcher for HTTP GET requests, wraps Faraday.get
+  class ResourceFetcher
+    # only cache following HTTP response codes, per Section 6.1, RFC 7231
+    CACHEABLE_STATUS = [
+      200, 203, 204, 206, 300, 301, 404, 405, 410, 414, 501
+    ].freeze
+
+    class << self
+      attr_accessor :cache
+    end
+
+    def self.get(url, stale_after = 3600)
+      new(stale_after).get(url)
+    end
+
+    def self.include?(url)
+      return false if cache.nil?
+      cache.keys.include?(url)
+    end
+
+    def initialize(stale_after = 3600)
+      @stale_after = stale_after # seconds
+      # initialize shared state only if missing:
+      self.class.cache = {} if self.class.cache.nil?
+    end
+
+    def get(url)
+      cache_get(url) || miss_get(url)
+    end
+
+    # @return [Hash] shared cache state
+    def cache
+      self.class.cache
+    end
+
+    # @return [NilClass, Hash] hash of status, response body — or nil if no HIT
+    def cache_get(url)
+      return unless cache.include?(url)
+      check_expiry(url)
+      # in case of expiration, cache will no longer include URL:
+      return unless cache.include?(url)
+      # return non-expired cache HIT:
+      cache[url]
+    end
+
+    # Get URL from original source, by URL; will cache any cachable response
+    #   in self.class.cache (shared state).
+    # @param url [String] URL to GET
+    # @raise [Faraday::ConnectionFailed] if DNS or TCP connection error.
+    # @return [Hash] hash containing status, response headers, response body
+    def miss_get(url)
+      resp = Faraday.get url
+      # create a new hash from headers
+      result = resp.headers.to_h
+      # add status and body to
+      result['status'] = resp.status
+      result['body'] = resp.body
+      # set (new or replaced previously) cached value for URL:
+      if CACHEABLE_STATUS.include?(resp.status)
+        result['cached_time'] = DateTime.now.to_time.to_i
+        cache[url] = result
+      end
+      result
+    end
+
+    def check_expiry(url)
+      return unless cache.include?(url)
+      cache.delete(url) if expired(cache[url])
+    end
+
+    def expired(record)
+      now = DateTime.now.to_time.to_i
+      # does elapsed seconds between store and now exceed threshold?
+      (now - record['cached_time']) > @stale_after
+    end
+  end
+end

data/lib/newspaper_works/text_extraction/alto_reader.rb

@@ -0,0 +1,122 @@
+require 'active_support/core_ext/module/delegation'
+require 'json'
+require 'nokogiri'
+
+module NewspaperWorks
+  # Module for text extraction
+  module TextExtraction
+    # Class to obtain plain text and JSON word-coordinates from ALTO source
+    class AltoReader
+      attr_accessor :source, :doc_stream
+      delegate :text, to: :doc_stream
+
+      # SAX Document Stream class to gather text and word tokens from ALTO
+      class AltoDocStream < Nokogiri::XML::SAX::Document
+        attr_accessor :text, :words
+
+        def initialize(image_width = nil)
+          super()
+          # scaling matters:
+          @image_width = image_width
+          @scaling = 1.0 # pt to px, if ALTO using points
+          # plain text buffer:
+          @text = ''
+          # list of word hash, containing word+coord:
+          @words = []
+        end
+
+        # Return coordinates from String element attribute hash
+        #
+        # @param attrs [Hash] hash containing ALTO `String` element attributes.
+        # @return [Array] Array of position x, y, width, height in px.
+        def s_coords(attrs)
+          height = scale_value((attrs['HEIGHT'] || 0).to_i)
+          width = scale_value((attrs['WIDTH'] || 0).to_i)
+          hpos = scale_value((attrs['HPOS'] || 0).to_i)
+          vpos = scale_value((attrs['VPOS'] || 0).to_i)
+          [hpos, vpos, width, height]
+        end
+
+        def compute_scaling(attrs)
+          return if @image_width.nil?
+          match = attrs.select { |e| e[0].casecmp?('WIDTH') }[0]
+          return if match.empty?
+          page_width = match[1].to_i
+          return if @image_width == page_width
+          @scaling = page_width / @image_width.to_f
+        end
+
+        def scale_value(v)
+          (v / @scaling).to_i
+        end
+
+        # Callback for element start, implementation of which ignores
+        #   non-String elements.
+        #
+        # @param name [String] element name.
+        # @param attrs [Array] Array of key, value pair Arrays.
+        def start_element(name, attrs = [])
+          values = attrs.to_h
+          compute_scaling(attrs) if name == 'Page'
+          return if name != 'String'
+          token = values['CONTENT']
+          @text << token
+          @words << {
+            word: token,
+            coordinates: s_coords(values)
+          }
+        end
+
+        # Callback for element end, used here to manage endings of lines and
+        #   blocks.
+        #
+        # @param name [String] element name.
+        def end_element(name)
+          @text << " " if name == 'String'
+          @text << "\n" if name == 'TextBlock'
+          @text << "\n" if name == 'TextLine'
+        end
+
+        # Callback for completion of parsing ALTO, used to normalize generated
+        #   text content (strip unneeded whitespace incidental to output).
+        def end_document
+          # postprocess @text to remove trailing spaces on lines
+          @text = @text.split("\n").map(&:strip).join("\n")
+          # remove trailing whitespace at end of buffer
+          @text.strip!
+        end
+      end
+
+      # Construct with either path
+      #
+      # @param xml [String], and process document
+      def initialize(xml, image_width = nil, image_height = nil)
+        @source = isxml?(xml) ? xml : File.read(xml)
+        @image_width = image_width
+        @image_height = image_height
+        @doc_stream = AltoDocStream.new(image_width)
+        parser = Nokogiri::XML::SAX::Parser.new(doc_stream)
+        parser.parse(@source)
+      end
+
+      # Determine if source parameter is path or xml
+      #
+      # @param xml [String] either path to xml file or xml source
+      # @return [true, false] true if string appears to be XML source, not path
+      def isxml?(xml)
+        xml.lstrip.start_with?('<')
+      end
+
+      # Output JSON flattened word coordinates
+      #
+      # @return [String] JSON serialization of flattened word coordinates
+      def json
+        words = @doc_stream.words
+        builder = NewspaperWorks::TextExtraction::WordCoordsBuilder.new(words,
+                                                                        @image_width,
+                                                                        @image_height)
+        builder.to_json
+      end
+    end
+  end
+end

data/lib/newspaper_works/text_extraction/page_ocr.rb

@@ -0,0 +1,100 @@
+require 'json'
+require 'open3'
+require 'rtesseract'
+
+# --
+module NewspaperWorks
+  # Module for text extraction (OCR or otherwise)
+  module TextExtraction
+    class PageOCR
+      def self.alto_from(path)
+        new(path).alto
+      end
+
+      def initialize(path)
+        @path = path
+        @words = nil
+        @processor = "mini_magick"
+        @source_meta = nil
+        @use_gm = extension.start_with?('jp2')
+        @box = nil
+        @plain = nil
+      end
+
+      def extension
+        @path.split('.')[-1].downcase
+      end
+
+      def load_box
+        if @box.nil?
+          if @use_gm
+            MiniMagick.with_cli(:graphicsmagick) do
+              @box = RTesseract::Box.new(@path, processor: @processor)
+              @plain = @box.to_s
+            end
+          else
+            @box = RTesseract::Box.new(@path, processor: @processor)
+            @plain = @box.to_s
+          end
+        end
+        @box
+      end
+
+      def words
+        @words = load_box.words if @words.nil?
+        @words
+      end
+
+      def normalized_coordinate(word)
+        {
+          word: word[:word],
+          coordinates: [
+            word[:x_start],
+            word[:y_start],
+            (word[:x_end] - word[:x_start]),
+            (word[:y_end] - word[:y_start])
+          ]
+        }
+      end
+
+      def word_json
+        save_words = words.map { |w| normalized_coordinate(w) }
+        builder = NewspaperWorks::TextExtraction::WordCoordsBuilder.new(save_words,
+                                                                        width,
+                                                                        height)
+        builder.to_json
+      end
+
+      def plain
+        load_box
+        @plain
+      end
+
+      def identify
+        if @source_geometry.nil?
+          path = @path
+          cmd = "identify -verbose #{path}"
+          cmd = 'gm ' + cmd if @use_gm
+          lines = `#{cmd}`.lines
+          geo = lines.select { |line| line.strip.start_with?('Geometry') }[0]
+          img_geo = geo.strip.split(':')[-1].strip.split('+')[0]
+          @source_geometry = img_geo.split('x').map(&:to_i)
+        end
+        @source_geometry
+      end
+
+      def width
+        identify[0]
+      end
+
+      def height
+        identify[1]
+      end
+
+      def alto
+        writer = NewspaperWorks::TextExtraction::RenderAlto.new(width, height)
+        writer.to_alto(words)
+      end
+    end
+  end
+end

data/lib/newspaper_works/text_extraction/render_alto.rb

@@ -0,0 +1,84 @@
+require 'nokogiri'
+
+module NewspaperWorks
+  # Module for text extraction (OCR or otherwise)
+  module TextExtraction
+    class RenderAlto
+      def initialize(width, height, scaling = 1.0)
+        @height = height
+        @width = width
+        @scaling = scaling
+      end
+
+      def to_alto(words)
+        page = alto_page(@width, @height) do |xml|
+          words.each do |word|
+            xml.String(
+              CONTENT: word[:word],
+              HEIGHT: scale_point(word[:y_end] - word[:y_start]).to_s,
+              WIDTH: scale_point(word[:x_end] - word[:x_start]).to_s,
+              HPOS: scale_point(word[:x_start]).to_s,
+              VPOS: scale_point(word[:y_start]).to_s
+            ) { xml.text '' }
+          end
+        end
+        page.to_xml
+      end
+
+      private
+
+        # given block to manage word generation, wrap with page/block/line
+        def alto_page(pxwidth, pxheight, &block)
+          builder = Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml|
+            xml.alto(xmlns: 'http://www.loc.gov/standards/alto/ns-v2#') do
+              xml.Description do
+                xml.MeasurementUnit 'pixel'
+              end
+              alto_layout(xml, pxwidth, pxheight, &block)
+            end
+          end
+          builder
+        end
+
+        def scale_point(value)
+          # note: presuming non-fractional, even though ALTO 2.1
+          #   specifies coordinates are xsd:float, not xsd:int,
+          #   simplify to integer value for output:
+          (value * @scaling).to_i
+        end
+
+        # return layout for page
+        def alto_layout(xml, pxwidth, pxheight, &block)
+          xml.Layout do
+            xml.Page(ID: 'ID1',
+                     PHYSICAL_IMG_NR: '1',
+                     HEIGHT: pxheight.to_i,
+                     WIDTH: pxwidth.to_i) do
+              xml.PrintSpace(HEIGHT: pxheight.to_i,
+                             WIDTH: pxwidth.to_i,
+                             HPOS: '0',
+                             VPOS: '0') do
+                alto_blockline(xml, pxwidth, pxheight, &block)
+              end
+            end
+          end
+        end
+
+        # make block line and call word-block
+        def alto_blockline(xml, pxwidth, pxheight)
+          xml.TextBlock(ID: 'ID1a',
+                        HEIGHT: pxheight.to_i,
+                        WIDTH: pxwidth.to_i,
+                        HPOS: '0',
+                        VPOS: '0') do
+            xml.TextLine(HEIGHT: pxheight.to_i,
+                         WIDTH: pxwidth.to_i,
+                         HPOS: '0',
+                         VPOS: '0') do
+              yield(xml)
+            end
+          end
+        end
+    end
+  end
+end

data/lib/newspaper_works/text_extraction/word_coords_builder.rb

@@ -0,0 +1,30 @@
+module NewspaperWorks
+  # Module for text extraction (OCR or otherwise)
+  module TextExtraction
+    class WordCoordsBuilder
+      def initialize(words, width = nil, height = nil)
+        @words = words
+        @width = width
+        @height = height
+      end
+
+      # Output JSON flattened word coordinates
+      #
+      # @return [String] JSON serialization of flattened word coordinates
+      def to_json
+        coordinates = {}
+        @words.each do |w|
+          word_chars = w[:word]
+          word_coords = w[:coordinates]
+          if coordinates[word_chars]
+            coordinates[word_chars] << word_coords
+          else
+            coordinates[word_chars] = [word_coords]
+          end
+        end
+        payload = { width: @width, height: @height, coords: coordinates }
+        JSON.generate(payload)
+      end
+    end
+  end
+end

data/lib/newspaper_works/text_extraction.rb

@@ -0,0 +1,10 @@
+require 'newspaper_works/text_extraction/alto_reader'
+require 'newspaper_works/text_extraction/page_ocr'
+require 'newspaper_works/text_extraction/render_alto'
+require 'newspaper_works/text_extraction/word_coords_builder'
+
+module NewspaperWorks
+  # Module for text extraction (OCR or otherwise)
+  module TextExtraction
+  end
+end

data/lib/newspaper_works/version.rb

@@ -0,0 +1,3 @@
+module NewspaperWorks
+  VERSION = '0.1.0'.freeze
+end

data/lib/newspaper_works.rb

@@ -0,0 +1,19 @@
+require "newspaper_works/engine"
+require "newspaper_works/errors"
+require "newspaper_works/ingest"
+require "newspaper_works/issue_pdf_composer"
+require "newspaper_works/text_extraction"
+require "newspaper_works/data"
+require "newspaper_works/configuration"
+require "newspaper_works/page_finder"
+require "newspaper_works/logging"
+require "newspaper_works/resource_fetcher"
+
+# Newspaper works modules
+module NewspaperWorks
+  def self.config(&block)
+    @config ||= NewspaperWorks::Configuration.new
+    yield @config if block
+    @config
+  end
+end

data/lib/tasks/newspaper_works_tasks.rake

@@ -0,0 +1,39 @@
+namespace :newspaper_works do
+  def use_application
+    ENV['RAILS_ENV'] = Rails.env if ENV['RAILS_ENV'].nil?
+    Rails.application.require_environment!
+  end
+
+  desc 'Ingest an NDNP batch: "rake newspaper_works:ingest_ndnp -- --path="'
+  task :ingest_ndnp do
+    use_application
+    ingester = NewspaperWorks::Ingest::NDNP::BatchIngester.from_command(
+      ARGV,
+      'rake newspaper_works:ingest_ndnp --'
+    )
+    puts "Beginning NDNP batch ingest..."
+    ingester.ingest
+    puts "NDNP batch ingest complete! See log/ingest.log for details."
+  end
+
+  desc 'Ingest a directory of PDF issues for a single publication: '\
+    '"rake newspaper_works:ingest_pdf_issues -- --path="'
+  task :ingest_issues do
+    use_application
+    ingester = NewspaperWorks::Ingest::BatchIssueIngester.from_command(
+      ARGV,
+      'rake newspaper_works:ingest_issues --'
+    )
+    puts "Beginning batch ingest of issues for single publication..."
+    ingester.ingest
+    puts "Ingest of issue(s) ingest complete, but may be pending background "\
+         "jobs. See log/ingest.log for details."
+  end
+
+  # Aliases to media-specific task ingest names
+  # rubocop:disable Style/HashSyntax
+  task :ingest_pdf_issues => :ingest_issues
+  task :ingest_tiff_issues => :ingest_issues
+  task :ingest_jp2_issues => :ingest_issues
+  # rubocop:enable Style/HashSyntax
+end

data/newspaper_works.gemspec

@@ -0,0 +1,49 @@
+$LOAD_PATH.push File.expand_path('../lib', __FILE__)
+
+# version updated in one place:
+require 'newspaper_works/version'
+
+# Gem description:
+Gem::Specification.new do |spec|
+  spec.name        = 'newspaper_works'
+  spec.version     = NewspaperWorks::VERSION
+  spec.authors     = ['Sean Upton', 'Jacob Reed', 'Brian McBride',
+                      'Eben English']
+  spec.email       = ['sean.upton@utah.edu', 'jacob.reed@utah.edu',
+                      'brian.mcbride@utah.edu', 'eenglish@bpl.org']
+  spec.homepage    = 'https://github.com/marriott-library/newspaper_works'
+  spec.description = 'Gem/Engine for Newspaper Works in Hyrax-based Samvera
+                      Application.'
+  spec.summary     = <<-SUMMARY
+  newspaper_works is a Rails Engine gem providing model and administrative
+  functions to Hyrax-based Samvera applications, for management of
+  (primarily scanned) archival newspaper content.
+SUMMARY
+  spec.license = 'Apache-2.0'
+  spec.files = `git ls-files`.split($OUTPUT_RECORD_SEPARATOR)
+  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
+  spec.add_dependency 'blacklight_iiif_search'
+  spec.add_dependency 'blacklight_advanced_search', '6.4.1'
+  spec.add_dependency 'hyrax', '2.5.1'
+  spec.add_dependency 'nokogiri'
+  spec.add_dependency 'rails', '~> 5.1'
+  spec.add_dependency 'rtesseract', '~> 2.2.0'
+  spec.add_dependency 'sass-rails', '~> 5.0'
+
+  spec.add_development_dependency 'bixby'
+  spec.add_development_dependency 'capybara', '~> 2.4', '< 2.18.0'
+  spec.add_development_dependency 'chromedriver-helper', '~> 2.1'
+  spec.add_development_dependency 'engine_cart', '~> 2.2'
+  spec.add_development_dependency "factory_bot", '~> 4.4'
+  spec.add_development_dependency "faraday"
+  spec.add_development_dependency 'fcrepo_wrapper', '~> 0.5', '>= 0.5.1'
+  spec.add_development_dependency 'newspaper_works_fixtures', '~> 0.3', '>=0.3.1'
+  spec.add_development_dependency 'rails-controller-testing', '~> 1'
+  spec.add_development_dependency 'rspec-rails', '~> 3.1'
+  spec.add_development_dependency 'rspec-activemodel-mocks'
+  spec.add_development_dependency 'selenium-webdriver'
+  spec.add_development_dependency 'shoulda-matchers', '~> 3.1'
+  spec.add_development_dependency 'solr_wrapper', '>= 1.1', '< 3.0'
+  spec.add_development_dependency 'webdrivers', '~> 3.0'
+  spec.add_development_dependency 'webmock', '~> 3.6'
+end

data/spec/.keep.txt

@@ -0,0 +1 @@
+spec dir for RSpec