newspaper_works 0.1.0

data/spec/lib/newspaper_works/page_finder_spec.rb

@@ -0,0 +1,53 @@
+require 'spec_helper'
+RSpec.describe NewspaperWorks::PageFinder do
+  # use before(:all) so we only create fixtures once
+  before(:all) do
+    @issue = NewspaperIssue.new
+    @issue.title = ["Yesterday's News: December 7, 1941"]
+
+    @page1 = NewspaperPage.new
+    @page1.title = ['Page 1']
+    @page2 = NewspaperPage.new
+    @page2.title = ['Page 2']
+    @page3 = NewspaperPage.new
+    @page3.title = ['Page 3']
+
+    @issue.ordered_members << @page1
+    @issue.ordered_members << @page2
+    @issue.ordered_members << @page3
+
+    @issue.save!
+    @page1.save!
+    @page2.save!
+    @page3.save!
+
+    @page1_solr_doc = SolrDocument.find(@page1.id)
+    @page2_solr_doc = SolrDocument.find(@page2.id)
+    @page3_solr_doc = SolrDocument.find(@page3.id)
+  end
+
+  let(:controller) { NewspaperWorks::NewspapersController.new }
+
+  let(:ordered_pages_array) { [@page1_solr_doc, @page2_solr_doc, @page3_solr_doc] }
+
+  describe 'pages_for_issue' do
+    subject { controller.pages_for_issue(@issue.id) }
+    it 'returns the pages, in order' do
+      # for some reason, the line below doesn't work, so we compare ids
+      # expect(subject).to eq ordered_pages_array
+      expect(subject[0]['id']).to eq ordered_pages_array[0]['id']
+      expect(subject[1]['id']).to eq ordered_pages_array[1]['id']
+      expect(subject[2]['id']).to eq ordered_pages_array[2]['id']
+    end
+  end
+
+  describe '#ordered_pages' do
+    subject { controller.ordered_pages(ordered_pages_array.shuffle) }
+    it { is_expected.to eq ordered_pages_array }
+  end
+
+  describe '#get_page_index' do
+    subject { controller.get_page_index(@page2.id) }
+    it { is_expected.to eq 1 }
+  end
+end

data/spec/lib/newspaper_works/resource_fetcher_spec.rb

@@ -0,0 +1,65 @@
+require 'spec_helper'
+
+describe NewspaperWorks::ResourceFetcher do
+  describe "cache hit and expiration handling" do
+    cached_time_url1 = 0
+    cached_time_url2 = 0
+
+    let(:url1) { 'https://www.example.com/1' }
+    let(:url2) { 'https://www.example.com/2' }
+
+    before do
+      stub_request(:any, url1)
+        .to_return(body: 'abc', headers: { 'Content-Length' => 3 })
+      stub_request(:any, url2)
+        .to_return(body: 'xyz', headers: { 'Content-Length' => 3 })
+      # populate cache for url1 by getting:
+      record = described_class.new(3600).miss_get(url1)
+      cached_time_url1 = record['cached_time']
+      # populate cache for url2, but...
+      record = described_class.new(3600).miss_get(url2)
+      # set cached time to something old:
+      record['cached_time'] = record['cached_time'] - 3601
+      cached_time_url2 = record['cached_time']
+    end
+
+    it "gets cached record for url" do
+      expect(described_class.include?(url1)).to be true
+      record = described_class.get(url1)
+      expect(record['cached_time']).to eq cached_time_url1
+    end
+
+    it "refreshes resource from origin on stale cached record" do
+      # while it "has" or includes url:
+      expect(described_class.include?(url2)).to be true
+      # on the terms of the default stale_after parameter, it is too old:
+      record = described_class.cache[url2]
+      expect(described_class.new(3600).expired(record)).to be true
+      # ...fetching will get new:
+      record = described_class.get(url2)
+      # new time means fresh request made to origin:
+      expect(record['cached_time']).not_to eq cached_time_url2
+    end
+  end
+
+  describe "cache miss fetch handling" do
+    let(:url) { 'https://www.example.com' }
+
+    before do
+      stub_request(:any, url)
+        .to_return(body: 'abc', headers: { 'Content-Length' => 3 })
+    end
+
+    it "makes request on cache miss" do
+      expect(described_class.include?(url)).to be false
+      record = described_class.get(url)
+      expect(record).to be_a Hash
+      timestamp = record['cached_time']
+      # now cached:
+      expect(described_class.include?(url)).to be true
+      record = described_class.get(url)
+      # same timestamp == effect of cache HIT
+      expect(record['cached_time']).to eq timestamp
+    end
+  end
+end

data/spec/lib/newspaper_works/text_extraction/alto_reader_spec.rb

@@ -0,0 +1,49 @@
+require 'json'
+require 'spec_helper'
+
+RSpec.describe NewspaperWorks::TextExtraction::AltoReader do
+  let(:fixture_path) do
+    File.join(
+      NewspaperWorks::GEM_PATH, 'spec', 'fixtures', 'files'
+    )
+  end
+
+  let(:minimal_path) { File.join(fixture_path, 'minimal-alto.xml') }
+  let(:ndnp_alto_path) { File.join(fixture_path, 'ndnp-alto-sample.xml') }
+  let(:minimal) { File.read(minimal_path) }
+
+  let(:reader_minimal) { described_class.new(minimal) }
+  let(:reader_minimal_path) { described_class.new(minimal_path) }
+  let(:reader_ndnp) { described_class.new(ndnp_alto_path) }
+
+  describe "reads alto" do
+    it "loads ALTO source" do
+      expect(reader_minimal_path.source).to eq reader_minimal.source
+      expect(reader_minimal_path.source.size).to eq 1383
+      expect(reader_ndnp.source.size).to eq 1_050_876
+    end
+
+    it "loads document stream" do
+      expect(reader_minimal_path.doc_stream).to be_kind_of Nokogiri::XML::SAX::Document
+      expect(reader_minimal_path.doc_stream).to respond_to :text
+      expect(reader_minimal_path.doc_stream).to respond_to :words
+    end
+  end
+
+  describe "outputs text derivative formats" do
+    it "outputs plain text" do
+      # try simple flat text input
+      expect(reader_minimal.text).to eq "This is only a test."
+      expect(reader_minimal.text).to eq reader_minimal.doc_stream.text
+      # try more complex input
+      expect(reader_ndnp.text.size).to eq 30_519
+    end
+
+    it "passes args to WordCoordsBuilder and receives output" do
+      parsed = JSON.parse(reader_minimal.json)
+      expect(parsed['coords'].length).to be > 1
+      parsed = JSON.parse(reader_ndnp.json)
+      expect(parsed['coords'].size).to eq 2_125
+    end
+  end
+end

data/spec/lib/newspaper_works/text_extraction/page_ocr_spec.rb

@@ -0,0 +1,84 @@
+require 'json'
+require 'nokogiri'
+require 'spec_helper'
+
+RSpec.describe NewspaperWorks::TextExtraction::PageOCR do
+  let(:fixture_path) do
+    File.join(
+      NewspaperWorks::GEM_PATH, 'spec', 'fixtures', 'files'
+    )
+  end
+
+  let(:altoxsd) do
+    xsdpath = File.join(fixture_path, 'alto-2-0.xsd')
+    Nokogiri::XML::Schema(File.read(xsdpath))
+  end
+
+  # sample "snippet" images for OCR testing:
+  let(:example_gray_tiff) { File.join(fixture_path, 'ocr_gray.tiff') }
+  let(:example_mono_tiff) { File.join(fixture_path, 'ocr_mono.tiff') }
+  let(:example_color_tiff) { File.join(fixture_path, 'ocr_color.tiff') }
+  let(:example_gray_jp2) { File.join(fixture_path, 'ocr_gray.jp2') }
+  let(:ocr_from_gray_tiff) { described_class.new(example_gray_tiff) }
+
+  describe "performs OCR" do
+    def match_ocr_expectations(words)
+      expect(words).to be_an(Array)
+      expect(words).not_to be_empty
+      expect(words[0]).to be_a(Hash)
+      [:word, :x_start, :y_start, :x_end, :y_end].each do |key|
+        expect(words[0].keys).to include key
+      end
+    end
+
+    it "gets words and coordinates from grayscale source" do
+      match_ocr_expectations(ocr_from_gray_tiff.words)
+    end
+
+    it "gets words and coordinates from one-bit source" do
+      ocr = described_class.new(example_mono_tiff)
+      match_ocr_expectations(ocr.words)
+    end
+
+    it "gets words and coordinates from color source" do
+      ocr = described_class.new(example_color_tiff)
+      match_ocr_expectations(ocr.words)
+    end
+
+    it "gets words and coordinates from jp2 source" do
+      ocr = described_class.new(example_gray_jp2)
+      match_ocr_expectations(ocr.words)
+    end
+  end
+
+  describe "turns image into ALTO" do
+    it "takes grayscale tiff, outputs valid ALTO, geometry" do
+      alto = ocr_from_gray_tiff.alto
+      document = Nokogiri::XML(alto)
+      errors = altoxsd.validate(document)
+      expect(errors.length).to eq 0
+      expect(document.at_css('PrintSpace')['WIDTH']).to eq "418"
+      expect(document.at_css('PrintSpace')['HEIGHT']).to eq "1046"
+    end
+  end
+
+  describe "plain text" do
+    it "makes plain text available for image" do
+      plain = ocr_from_gray_tiff.plain
+      expect(plain.class).to be String
+      expect(plain.length).to be > 0
+    end
+  end
+
+  describe "JSON word coordinates" do
+    it "passes properly formatted data to WordCoordsBuilder and receives output" do
+      parsed = JSON.parse(ocr_from_gray_tiff.word_json)
+      expect(parsed['coords'].length).to be > 1
+      word = ocr_from_gray_tiff.words[0]
+      word1 = parsed['coords'][word[:word]]
+      word1_coords = word1[0]
+      expect(word1_coords[2]).to eq word[:x_end] - word[:x_start]
+      expect(word1_coords[3]).to eq word[:y_end] - word[:y_start]
+    end
+  end
+end

data/spec/lib/newspaper_works/text_extraction/render_alto_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+
+RSpec.describe NewspaperWorks::TextExtraction::RenderAlto do
+  let(:fixture_path) do
+    File.join(
+      NewspaperWorks::GEM_PATH, 'spec', 'fixtures', 'files'
+    )
+  end
+
+  let(:altoxsd) do
+    xsdpath = File.join(fixture_path, 'alto-2-0.xsd')
+    Nokogiri::XML::Schema(File.read(xsdpath))
+  end
+
+  let(:page_prefix) { '<Page ID="ID1" PHYSICAL_IMG_NR="1"' }
+
+  let(:words) do
+    [
+      { word: 'If',  x_start: 52, y_start: 13, x_end: 63, y_end: 27 },
+      { word: 'you', x_start: 69, y_start: 17, x_end: 100, y_end: 31 },
+      { word: 'are', x_start: 108, y_start: 17, x_end: 136, y_end: 27 },
+      { word: 'a', x_start: 143, y_start: 17, x_end: 151, y_end: 27 },
+      { word: 'friend,', x_start: 158, y_start: 13, x_end: 214, y_end: 29 },
+      { word: 'you', x_start: 51, y_start: 39, x_end: 82, y_end: 53 },
+      { word: 'speak', x_start: 90, y_start: 35, x_end: 140, y_end: 53 },
+      { word: 'the', x_start: 146, y_start: 35, x_end: 174, y_end: 49 },
+      { word: 'password,', x_start: 182, y_start: 35, x_end: 267, y_end: 53 },
+      { word: 'and', x_start: 51, y_start: 57, x_end: 81, y_end: 71 },
+      { word: 'the', x_start: 89, y_start: 57, x_end: 117, y_end: 71 },
+      { word: 'doors', x_start: 124, y_start: 57, x_end: 172, y_end: 71 },
+      { word: 'will', x_start: 180, y_start: 57, x_end: 208, y_end: 71 },
+      { word: 'open.', x_start: 216, y_start: 61, x_end: 263, y_end: 75 }
+    ]
+  end
+
+  describe "renders alto" do
+    it "creates alto given width, height, words" do
+      renderer = described_class.new(12_000, 9600)
+      output = renderer.to_alto(words)
+      expect(output.class).to be String
+      expect(output).to include '<alto'
+      expect(output).to include '<String'
+      expect(output).to include page_prefix + ' HEIGHT="9600" WIDTH="12000"'
+      expect(Nokogiri::XML(output).errors.empty?).to be true
+    end
+
+    it "makes alto 2.0 that validates" do
+      renderer = described_class.new(12_000, 9600)
+      output = renderer.to_alto(words)
+      document = Nokogiri::XML(output)
+      altoxsd.validate(document)
+    end
+  end
+end

data/spec/lib/newspaper_works/text_extraction/word_coords_builder_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+RSpec.describe NewspaperWorks::TextExtraction::WordCoordsBuilder do
+  let(:words) do
+    [
+      { word: "foo", coordinates: [1, 2, 3, 4] },
+      { word: "bar", coordinates: [5, 6, 7, 8] },
+      { word: "baz", coordinates: [9, 10, 11, 12] },
+      { word: "foo", coordinates: [13, 14, 15, 16] }
+    ]
+  end
+  let(:image_width) { 1_234 }
+  let(:image_height) { 5_678 }
+  let(:wcb) { described_class.new(words, image_width, image_height) }
+
+  describe '#to_json' do
+    let(:wcb_to_json) { JSON.parse(wcb.to_json) }
+
+    it 'has the correct structure' do
+      expect(wcb_to_json['height']).to eq image_height
+      expect(wcb_to_json['width']).to eq image_width
+      expect(wcb_to_json['coords'].length).to eq 3
+      expect(wcb_to_json['coords']['foo']).not_to be_falsey
+    end
+
+    it 'combines coordinates for the same word' do
+      expect(wcb_to_json['coords']['foo']).to eq [[1, 2, 3, 4], [13, 14, 15, 16]]
+    end
+  end
+end

data/spec/lib/tasks/newspaper_works_rake_spec.rb

@@ -0,0 +1,124 @@
+require 'spec_helper'
+require 'rake'
+require 'ndnp_shared'
+require 'lib/newspaper_works/ingest/ingest_shared'
+require 'active_fedora/cleaner'
+
+describe 'newspaper_works rake tasks' do
+  include_context 'ndnp fixture setup'
+  include_context 'ingest test fixtures'
+
+  before(:all) do
+    Rake.application.rake_require '../lib/tasks/newspaper_works_tasks'
+    Rake::Task.define_task(:environment)
+  end
+
+  describe 'ingest tasks' do
+    before do
+      ActiveFedora::Cleaner.clean!
+      Hyrax::PermissionTemplateAccess.destroy_all
+      Hyrax::PermissionTemplate.destroy_all
+    end
+
+    let(:pdf_lccn) { 'sn93059126' }
+
+    let(:pdf_path) { File.join(pdf_fixtures, pdf_lccn, '1853060401.pdf') }
+
+    let(:single_issue_dir) do
+      Hyrax.config.whitelisted_ingest_dirs.push('/tmp')
+      parent_dir = Dir.mktmpdir
+      dir = File.join(parent_dir, pdf_lccn)
+      FileUtils.mkdir(dir)
+      FileUtils.cp(pdf_path, dir)
+      dir
+    end
+
+    let(:run_ndnp_ingest_task) do
+      task = 'newspaper_works:ingest_ndnp'
+      stub_const(
+        'ARGV',
+        [
+          'newspaper_works:ingest_ndnp',
+          '--',
+          "--path=#{batch1}"
+        ]
+      )
+      Rake::Task[task].reenable
+      Rake.application.invoke_task(task)
+    end
+
+    let(:run_pdf_ingest_task) do
+      task = 'newspaper_works:ingest_pdf_issues'
+      stub_const(
+        'ARGV',
+        [
+          'newspaper_works:ingest_pdf_issues',
+          '--',
+          "--path=#{single_issue_dir}"
+        ]
+      )
+      Rake::Task[task].reenable
+      Rake.application.invoke_task(task)
+    end
+
+    def expect_clean_slate
+      expect(NewspaperTitle.all.to_a).to be_empty
+      expect(NewspaperIssue.all.to_a).to be_empty
+      expect(NewspaperPage.all.to_a).to be_empty
+    end
+
+    def expect_generated_issues(publication)
+      batch = NewspaperWorks::Ingest::NDNP::BatchXMLIngest.new(batch1)
+      relevant = batch.select { |i| i.metadata.lccn == publication.lccn }
+      issue_dates = relevant.map(&:publication_date)
+      expect(publication.issues.size).to eq issue_dates.size
+      expect(publication.issues.map(&:publication_date)).to \
+        match_array issue_dates
+    end
+
+    def expect_generated_content(lccn_list)
+      lccn_list.each do |lccn|
+        # expect title work for LCCN
+        publication = NewspaperTitle.where(lccn: lccn).first
+        expect(publication).not_to be_nil
+        # expect title to have issue children
+        issues = publication.issues.to_a
+        expect(issues).not_to be_empty
+      end
+    end
+
+    def check_pages(lccns)
+      # quick verification of pages imported:
+      pages = NewspaperPage.all
+      expect(pages.size).to eq 5
+      pages.each do |page|
+        lccn = page.publication.lccn
+        expect(lccns).to include lccn
+      end
+    end
+
+    def expect_pdf_issues
+      # expect NewspaperTitle for LCCN 'sn93059126'
+      publication = NewspaperTitle.where(lccn: 'sn93059126').first
+      expect(publication).not_to be_nil
+      # expect issue for date:
+      issue = publication.issues[0]
+      expect(issue.publication_date).to eq '1853-06-04'
+    end
+
+    it 'successfully ingests NDNP batch by task' do
+      pub_lccns = ['sn84038814', 'sn85025202']
+      expect_clean_slate
+      run_ndnp_ingest_task
+      # the batch we test has two titles, verify all content for each:
+      expect_generated_content(pub_lccns)
+      check_pages(pub_lccns)
+    end
+
+    it 'ingests a PDF issue batch' do
+      expect_clean_slate
+      run_pdf_ingest_task
+      expect_pdf_issues
+    end
+  end
+end

data/spec/misc_shared.rb

@@ -0,0 +1,109 @@
+RSpec.shared_context "shared setup", shared_context: :metadata do
+  let(:fixture_path) do
+    path = File.join(
+      NewspaperWorks::GEM_PATH, 'spec', 'fixtures', 'files'
+    )
+    whitelist = Hyrax.config.whitelisted_ingest_dirs
+    whitelist.push(path) unless whitelist.include?(path)
+    path
+  end
+
+  # shared date to be invariant across all tests in a run:
+  date_static = Hyrax::TimeService.time_in_utc
+  let(:static_date) { date_static }
+
+  # path fixtures:
+  let(:example_gray_jp2) { File.join(fixture_path, 'ocr_gray.jp2') }
+  let(:txt_path) { File.join(fixture_path, 'credits.md') }
+  let(:sample_thumbnail) { File.join(fixture_path, 'thumbnail.jpg') }
+
+  # sample data:
+  let(:sample_text) { 'even in a mythical Age there must be some enigmas' }
+
+  let(:valid_file_set) do
+    file_set = FileSet.new
+    file_set.save!(validate: false)
+    file_set
+  end
+
+  let(:sample_work) do
+    work = NewspaperPage.new
+    work.title = ['Bombadil']
+    work.members.push(valid_file_set)
+    work.save!
+    work
+  end
+
+  # sample objects:
+  let(:work_with_file) do
+    # we need a work with not just a valid (but empty) fileset, but also
+    #   a persisted file, so we use the shared work sample, and expand
+    #   on it with actual file data/metadata.
+    work = sample_work
+    fileset = work.members.select { |m| m.class == FileSet }[0]
+    file = Hydra::PCDM::File.create
+    fileset.original_file = file
+    # Set binary content on file via ActiveFedora content= mutator method
+    #   which also makes .size method return valid result for content
+    file.content = File.open(txt_path)
+    # Set some metdata we would expect to otherwise be set upon an upload
+    file.original_name = 'credits.md'
+    file.mime_type = 'text/plain'
+    file.date_modified = static_date
+    file.date_created = static_date
+    # saving fileset also saves file content
+    fileset.save!
+    work
+  end
+
+  def path_factory
+    Hyrax::DerivativePath
+  end
+
+  def work_file_set(work)
+    work.members.select { |m| m.class == FileSet }[0]
+  end
+
+  def text_path(work)
+    path_factory.derivative_path_for_reference(work_file_set(work), 'txt')
+  end
+
+  def jp2_path(work)
+    path_factory.derivative_path_for_reference(work_file_set(work), 'jp2')
+  end
+
+  def thumbnail_path(work)
+    path_factory.derivative_path_for_reference(work_file_set(work), 'thumbnail')
+  end
+
+  def mkdir_derivative(work, name)
+    # make shared path for derivatives to live, Hyrax ususally does this
+    #   for thumbnails, and newspaper_works does this in its derivative
+    #   service plugins; here we do same.
+    fsid = work_file_set(work).id
+    path = path_factory.derivative_path_for_reference(fsid, name)
+    dir = File.join(path.split('/')[0..-2])
+    FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+  end
+
+  def mk_jp2_derivative(work)
+    mkdir_derivative(work, 'jp2')
+    dst_path = jp2_path(work)
+    FileUtils.copy(example_gray_jp2, dst_path)
+    expect(File.exist?(dst_path)).to be true
+  end
+
+  def mk_txt_derivative(work)
+    mkdir_derivative(work, 'txt')
+    dst_path = text_path(work)
+    File.open(dst_path, 'w') { |f| f.write(sample_text) }
+    expect(File.exist?(dst_path)).to be true
+  end
+
+  def mk_thumbnail_derivative(work)
+    mkdir_derivative(work, 'thumbnail')
+    dst_path = thumbnail_path(work)
+    FileUtils.copy(sample_thumbnail, dst_path)
+    expect(File.exist?(dst_path)).to be true
+  end
+end