newspaper_works 0.1.0

data/spec/lib/newspaper_works/ingest/ndnp/issue_ingester_spec.rb

@@ -0,0 +1,155 @@
+require 'spec_helper'
+require 'ndnp_shared'
+
+RSpec.describe NewspaperWorks::Ingest::NDNP::IssueIngester do
+  include_context "ndnp fixture setup"
+
+  # Source data:
+  let(:issue_data) do
+    NewspaperWorks::Ingest::NDNP::IssueIngest.new(issue1)
+  end
+
+  let(:metadata) { issue_data.metadata }
+
+  # IssueIngester adapter does the work we are testing:
+  let(:adapter) { described_class.new(issue_data) }
+
+  describe "adapter and asset construction" do
+    def expect_issue_import_logging(adapter)
+      expect(adapter).to receive(:write_log).with(
+        satisfy { |v| v.include?('Saved metadata to new NewspaperIssue') }
+      ).once
+    end
+
+    # remove publication asset from repository for LCCN, when re-creating
+    #   is desired test behavior
+    def clear_publication(lccn)
+      NewspaperTitle.where(lccn: lccn).delete_all
+    end
+
+    it "constructs adapter with issue source" do
+      expect(adapter.issue).to be issue_data
+      expect(adapter.path).to eq issue_data.path
+      # initially nil target:
+      expect(adapter.target).to be_nil
+    end
+
+    it "constructs adapter with hash options" do
+      user = User.batch_user.user_key
+      adapter = described_class.new(
+        issue_data,
+        depositor: user
+      )
+      expect(adapter.opts[:depositor]).to eq user
+    end
+
+    it "constructs NewspaperIssue with adapter" do
+      # construct_issue is only the first part of ingest, create issue
+      #   and find-or-link publication NewspaperTitle;
+      #   this does not trigger creation of child pages.
+      clear_publication(issue_data.metadata.lccn)
+      expect_issue_import_logging(adapter)
+      expect(adapter).to receive(:write_log).with(
+        satisfy do |v|
+          v.include?('Created NewspaperTitle work') ||
+          v.include?('Found existing NewspaperTitle')
+        end
+      ).once
+      expect(adapter).to receive(:write_log).with(
+        satisfy { |v| v.include?('Linked NewspaperIssue') }
+      ).once
+      adapter.construct_issue
+      issue = adapter.target
+      expect(issue).to be_a NewspaperIssue
+      expect(issue.id).not_to be_nil
+      # check parent publication
+      publication = issue.publication
+      expect(publication.lccn).to eq issue_data.metadata.lccn
+      expect(publication.title).to contain_exactly 'The Park Record'
+    end
+
+    it "creates new NewspaperTitle without place of publication" do
+      # clear any existing publications from previous testing
+      lccn = issue_data.metadata.lccn
+      clear_publication(lccn)
+      # construct with title, this time no username set for geonames:
+      Qa::Authorities::Geonames.username = ''
+      adapter.construct_issue
+      expect(adapter.target.publication.place_of_publication).to be_empty
+      Qa::Authorities::Geonames.username = 'newspaper_works'
+    end
+
+    it "creates new NewspaperTitle with place of publication" do
+      # clear any existing publications from previous testing
+      lccn = issue_data.metadata.lccn
+      clear_publication(lccn)
+      # construct with title, this time with username set for geonames:
+      Qa::Authorities::Geonames.username = 'newspaper_works'
+      adapter.construct_issue
+      pop = adapter.target.publication.place_of_publication.map do |v|
+        v.to_uri.to_s
+      end
+      expect(pop).not_to be_empty
+      expect(pop[0]).to start_with 'http://sws.geonames.org/'
+    end
+  end
+
+  describe "metadata access/setting" do
+    def normalized_pubtitle(issue_data)
+      issue_data.metadata.publication_title.strip.split(/ \(/)[0]
+    end
+
+    def expected_title(issue_data)
+      metadata = issue_data.metadata
+      d = DateTime.iso8601(metadata.publication_date).strftime('%B %-d, %Y')
+      "#{normalized_pubtitle(issue_data)}: #{d}"
+    end
+
+    it "copies metadata to NewspaperIssue" do
+      adapter.construct_issue
+      issue = adapter.target
+      metadata = issue_data.metadata
+      expect(issue.title).to contain_exactly expected_title(issue_data)
+      expect(issue.lccn).to eq metadata.lccn
+      expect(issue.volume).to eq metadata.volume
+      expect(issue.publication_date).to eq metadata.publication_date
+      expect(issue.issue_number).to eq metadata.issue_number
+    end
+
+    it "sets default administrative metadata with default construction" do
+      adapter.construct_issue
+      issue_asset = adapter.target
+      expect(issue_asset.depositor).to eq User.batch_user.user_key
+      expect(issue_asset.admin_set).to eq AdminSet.find(AdminSet::DEFAULT_ID)
+      expect(issue_asset.visibility).to eq 'open'
+    end
+
+    it "sets custom administrative metadata for issue" do
+      # test one exemplary/representative option:
+      adapter = described_class.new(issue_data, visibility: 'open')
+      adapter.construct_issue
+      expect(adapter.target.visibility).to eq 'open'
+    end
+
+    it "sets custom administrative metadata for constructed publication" do
+      # test one exemplary/representative option:
+      adapter = described_class.new(issue_data, visibility: 'open')
+      adapter.construct_issue
+      publication_asset = adapter.target.publication
+      expect(publication_asset).not_to be_nil
+      expect(publication_asset.visibility).to eq 'open'
+    end
+  end
+
+  describe "child page creation" do
+    it "creates child pages on ingest of issue" do
+      # calling ingest without invoking the ususal async jobs should
+      #   create child pages without additional work of attaching files
+      #   to them, which we don't need to test here (tested elsewhere).
+      adapter.ingest
+      adapter.target.pages.each do |page|
+        expect(page.issue.id).to eq adapter.target.id
+      end
+    end
+  end
+end

data/spec/lib/newspaper_works/ingest/ndnp/issue_metadata_spec.rb

@@ -0,0 +1,84 @@
+require 'spec_helper'
+require 'ndnp_shared'
+
+RSpec.describe NewspaperWorks::Ingest::NDNP::IssueMetadata do
+  include_context "ndnp fixture setup"
+
+  describe "sample fixture 'batch_local'" do
+    let(:issue) { described_class.new(issue1) }
+
+    it "gets lccn" do
+      expect(issue.lccn).to eq "sn85058233"
+    end
+
+    it "gets volume" do
+      expect(issue.volume).to eq "56"
+    end
+
+    it "gets issue" do
+      expect(issue.issue_number).to eq "27"
+    end
+
+    it "gets edition fields" do
+      expect(issue.edition_name).to eq "Main Edition"
+      expect(issue.edition_number).to eq "1"
+    end
+
+    it "gets publication date" do
+      expect(issue.publication_date).to eq "1935-08-02"
+    end
+
+    it "gets publication title via //mets/@LABEL" do
+      expect(issue.publication_title).to eq 'The Park Record (Park City, UT)'
+    end
+
+    it "gets held_by" do
+      expect(issue.held_by).to eq "University of Utah; Salt Lake City, UT"
+    end
+  end
+
+  describe "sample fixture 'batch_test_ver01" do
+    let(:issue) { described_class.new(issue2) }
+    let(:issue_ingest) do
+      NewspaperWorks::Ingest::NDNP::IssueIngest.new(issue2)
+    end
+
+    it "gets lccn" do
+      expect(issue.lccn).to eq "sn85025202"
+    end
+
+    it "gets volume" do
+      expect(issue.volume).to eq "2"
+    end
+
+    it "gets issue" do
+      expect(issue.issue_number).to eq "4"
+    end
+
+    it "gets edition fields" do
+      expect(issue.edition_name).to be_nil
+      expect(issue.edition_number).to eq "1"
+    end
+
+    it "gets publication date" do
+      expect(issue.publication_date).to eq "1857-02-14"
+    end
+
+    it "gets publication title via label, when reel unavailable" do
+      expect(issue.publication_title).to \
+        eq 'Weekly Trinity journal (Weaverville, Calif.)'
+    end
+
+    # integration test for reel context publication title:
+    it "gets publication title via label, from reel" do
+      expect(issue_ingest.metadata.publication_title).to \
+        eq 'Weekly Trinity journal (Weaverville, Calif.)'
+      expect(issue_ingest.metadata.publication_title).to \
+        eq issue_ingest.container.metadata.title
+    end
+
+    it "gets held_by" do
+      expect(issue.held_by).to eq "University of Utah, Salt Lake City, UT"
+    end
+  end
+end

data/spec/lib/newspaper_works/ingest/ndnp/page_ingest_spec.rb

@@ -0,0 +1,79 @@
+require 'spec_helper'
+require 'ndnp_shared'
+
+RSpec.describe NewspaperWorks::Ingest::NDNP::PageIngest do
+  include_context "ndnp fixture setup"
+
+  def file_type?(path, ext)
+    path.split('/')[-1].split('.')[-1].casecmp(ext).zero?
+  end
+
+  def includes_file_type(files, ext)
+    files.any? { |path| file_type?(path, ext) }
+  end
+
+  def check_expected_files(page, extensions)
+    files = page.files
+    expect(files.size).to eq extensions.size
+    files.each do |filepath|
+      # each path is normalized to absolute path
+      expect(filepath.start_with?('/')).to be true
+    end
+    extensions.each do |ext|
+      expect(includes_file_type(files, ext)).to be true
+    end
+  end
+
+  describe "sample fixture 'batch_local'" do
+    let(:page) { described_class.new(issue1, 'pageModsBib8') }
+
+    it "gets metadata" do
+      expect(page.metadata).to be_a NewspaperWorks::Ingest::NDNP::PageMetadata
+      # uses same Nokogiri document context:
+      expect(page.metadata.doc).to be page.doc
+    end
+
+    it "gets expected files" do
+      check_expected_files(page, ['tif', 'jp2', 'pdf', 'xml'])
+    end
+
+    it "gets nil container for page without reel XML" do
+      reel = page.container
+      expect(reel).to be_nil
+    end
+  end
+
+  describe "sample fixture 'batch_test_ver01'" do
+    let(:page) { described_class.new(issue2, 'pageModsBib1') }
+
+    it "gets metadata" do
+      expect(page.metadata).to be_a NewspaperWorks::Ingest::NDNP::PageMetadata
+      # uses same Nokogiri document context:
+      expect(page.metadata.doc).to be page.doc
+    end
+
+    it "gets expected files" do
+      check_expected_files(page, ['tif', 'jp2', 'pdf', 'xml'])
+    end
+
+    it "gets a ContainerIngest for reel providing page" do
+      reel = page.container
+      expect(reel).to be_a NewspaperWorks::Ingest::NDNP::ContainerIngest
+      expect(reel.path).to end_with '_1.xml'
+    end
+  end
+
+  describe "sample fixture reel xml (for control images)" do
+    let(:page) { described_class.new(reel1, 'targetModsBib1') }
+
+    it "gets metadata" do
+      expect(page.metadata).to be_a NewspaperWorks::Ingest::NDNP::PageMetadata
+      # uses same Nokogiri document context:
+      expect(page.metadata.doc).to be page.doc
+    end
+
+    it "gets expected files" do
+      check_expected_files(page, ['tif', 'jp2', 'pdf', 'xml'])
+    end
+  end
+end

data/spec/lib/newspaper_works/ingest/ndnp/page_ingester_spec.rb

@@ -0,0 +1,184 @@
+require 'spec_helper'
+require 'ndnp_shared'
+require 'misc_shared'
+
+RSpec.describe NewspaperWorks::Ingest::NDNP::PageIngester do
+  include_context "ndnp fixture setup"
+  include_context "shared setup"
+
+  # use FactoryBot issue factory for a NewspaperIssue object for page:
+  let(:issue) { create(:newspaper_issue) }
+
+  # We need page source data as PageIngest
+  let(:page_data) do
+    NewspaperWorks::Ingest::NDNP::PageIngest.new(issue1, 'pageModsBib8')
+  end
+
+  let(:metadata) { page_data.metadata }
+
+  # PageIngester adapter does the work we are testing:
+  let(:adapter) { described_class.new(page_data, issue) }
+
+  describe "adapter and asset construction" do
+    it "constructs adapter with page source, issue context" do
+      expect(adapter.page).to be page_data
+      expect(adapter.issue).to be issue
+      expect(adapter.path).to eq page_data.path
+    end
+
+    it "constructs NewspaperPage with adapter" do
+      # construct_page is ingest of metadata only, without importing files:
+      adapter.construct_page
+      page = adapter.target
+      expect(page).to be_a NewspaperPage
+      expect(page.id).not_to be_nil
+      expect(issue.members).to include page
+      expect(issue.ordered_members.to_a).to include page
+    end
+
+    it "constructs adapter with hash options" do
+      user = User.batch_user.user_key
+      adapter = described_class.new(
+        page_data,
+        issue,
+        depositor: user
+      )
+      expect(adapter.opts[:depositor]).to eq user
+    end
+  end
+
+  describe "metadata access/setting" do
+    let(:expected_title) do
+      "#{issue.title.first}: Page #{metadata.page_number}"
+    end
+
+    it "sets default administrative metadata with default construction" do
+      adapter.construct_page
+      asset = adapter.target
+      expect(asset.depositor).to eq User.batch_user.user_key
+      expect(asset.admin_set).to eq AdminSet.find(AdminSet::DEFAULT_ID)
+      expect(asset.visibility).to eq 'open'
+    end
+
+    it "sets custom administrative metadata" do
+      # test one exemplary/representative option:
+      adapter = described_class.new(page_data, issue, visibility: 'open')
+      adapter.construct_page
+      expect(adapter.target.visibility).to eq 'open'
+    end
+
+    it "copies metadata to NewspaperPage" do
+      adapter.construct_page
+      page = adapter.target
+      expect(page.title).to contain_exactly expected_title
+      expect(page.width).to eq metadata.width
+      expect(page.height).to eq metadata.height
+      expect(page.page_number).to eq metadata.page_number
+      expect(page.identifier).to contain_exactly metadata.identifier
+    end
+  end
+
+  describe "reel/container linking" do
+    # need publication, title, and reel to use for page data context:
+    let(:publication) { create(:newspaper_title) }
+
+    let(:issue) do
+      issue = create(:newspaper_issue)
+      publication.members << issue
+      publication.save!
+      issue
+    end
+
+    let(:issue_data) do
+      NewspaperWorks::Ingest::NDNP::IssueIngest.new(issue2)
+    end
+
+    let(:page_data) do
+      data = issue_data.to_a[0]
+      # some NDNP samples missing TIFF, put dummy in place of missing, as needed
+      data.files = data.files.map do |path|
+        File.exist?(path) ? path : File.join(fixture_path, 'ocr_gray.tiff')
+      end
+      data
+    end
+
+    let(:adapter) { described_class.new(page_data, issue) }
+
+    it "links page to reel" do
+      # construct_page + link_reel ~= ingest without files import:
+      adapter.construct_page
+      adapter.link_reel
+      page = adapter.target
+      page.reload
+      expect(page.container).not_to be_nil
+      expect(page.container.ordered_members.to_a.map(&:id)).to include page.id
+    end
+  end
+
+  describe "file import integration" do
+    do_now_jobs = [IngestLocalFileJob, IngestJob, InheritPermissionsJob]
+
+    let(:issue_data) do
+      NewspaperWorks::Ingest::NDNP::IssueIngest.new(issue2)
+    end
+
+    let(:page_data_minus_tiff) { issue_data.to_a[0] }
+
+    def check_fileset(page)
+      fileset = page.members.select { |m| m.class == FileSet }[0]
+      # Reload fileset because jobs have modified:
+      fileset.reload
+      expect(fileset).not_to be_nil
+      expect(fileset.original_file).not_to be_nil
+      expect(fileset.original_file.mime_type).to eq 'image/tiff'
+      expect(fileset.original_file.size).to be > 0
+    end
+
+    def expect_file_assignment_logging(adapter)
+      expect(adapter).to receive(:write_log).with(
+        satisfy { |v| v.include?('Assigned primary file to work') }
+      ).once
+      expect(adapter).to receive(:write_log).with(
+        satisfy { |v| v.include?('Assigned derivative file to work') }
+      ).exactly(3).times
+      expect(adapter).to receive(:write_log).with(
+        satisfy { |v| v.include?('Beginning file attachment') }
+      ).once
+    end
+
+    def expect_page_import_logging(adapter)
+      expect(adapter).to receive(:write_log).with(
+        satisfy { |v| v.include?('Created NewspaperPage work') }
+      ).once
+      expect(adapter).to receive(:write_log).with(
+        satisfy { |v| v.include?('Saved metadata to NewspaperPage work') }
+      ).once
+      expect(adapter).to receive(:write_log).with(
+        satisfy { |v| v.include?('Linked NewspaperIssue') }
+      ).once
+    end
+
+    it "attaches primary, derivative files", perform_enqueued: do_now_jobs do
+      expect_page_import_logging(adapter)
+      expect_file_assignment_logging(adapter)
+      adapter.ingest
+      page = adapter.target
+      check_fileset(page)
+      derivatives = NewspaperWorks::Data::WorkDerivatives.new(page)
+      expect(derivatives.keys).to match_array ["jp2", "xml", "pdf"]
+    end
+
+    # support this use-case for evaluation purposes
+    it "generates TIFF when missing from page", perform_enqueued: do_now_jobs do
+      adapter = described_class.new(page_data_minus_tiff, issue)
+      expect_page_import_logging(adapter)
+      expect(adapter).to receive(:write_log).with(
+        satisfy { |arg| arg.include?('Creating TIFF') },
+        Logger::WARN
+      ).exactly(1).times
+      expect_file_assignment_logging(adapter)
+      expect { adapter.ingest }.not_to raise_error
+      check_fileset(adapter.target)
+    end
+  end
+end

data/spec/lib/newspaper_works/ingest/ndnp/page_metadata_spec.rb

@@ -0,0 +1,85 @@
+require 'spec_helper'
+require 'ndnp_shared'
+
+RSpec.describe NewspaperWorks::Ingest::NDNP::PageMetadata do
+  include_context "ndnp fixture setup"
+
+  describe "sample fixture 'batch_local'" do
+    let(:page1) { described_class.new(issue1, nil, 'pageModsBib8') }
+    let(:page2) { described_class.new(issue1, nil, 'pageModsBib6') }
+
+    it "gets expected page number as String" do
+      expect(page1.page_number).to eq "1"
+      expect(page2.page_number).to eq "2"
+    end
+
+    it "gets expected sequence number as Integer" do
+      expect(page1.page_sequence_number).to eq 1
+      expect(page2.page_sequence_number).to eq 2
+    end
+
+    it "gets expected width from ALTO as Integer " do
+      expect(page1.width).to eq 18_352
+      expect(page2.width).to eq 18_200
+    end
+
+    it "gets expected height from ALTO as Integer " do
+      expect(page1.height).to eq 28_632
+      expect(page2.height).to eq 28_872
+    end
+
+    it "gets identifier from ALTO as primary file name" do
+      expect(page1.identifier).to eq "0657b"
+      expect(page2.identifier).to eq "0656a"
+    end
+  end
+
+  describe "sample fixture 'batch_test_ver01" do
+    let(:page) { described_class.new(issue2, nil, 'pageModsBib1') }
+
+    it "fallback to sequence number on page without page number" do
+      expect(page.page_number).to eq page.page_sequence_number.to_s
+    end
+
+    it "gets expected sequence number as Integer" do
+      expect(page.page_sequence_number).to eq 1
+    end
+
+    it "gets expected width from ALTO as Integer " do
+      expect(page.width).to eq 21_464
+    end
+
+    it "gets expected height from ALTO as Integer " do
+      expect(page.height).to eq 30_268
+    end
+
+    it "gets identifier from ALTO as primary file name" do
+      expect(page.identifier).to eq "0225"
+    end
+  end
+
+  describe "sample fixture via Reel XML" do
+    let(:page) { described_class.new(reel1, nil, 'targetModsBib1') }
+
+    it "return nil page number when page and sequence missing" do
+      expect(page.page_number).to eq nil
+      expect(page.page_sequence_number).to eq nil
+    end
+
+    it "gets expected sequence number as Integer" do
+      expect(page.page_sequence_number).to eq nil
+    end
+
+    it "gets expected width from ALTO as Integer " do
+      expect(page.width).to eq 30_176
+    end
+
+    it "gets expected height from ALTO as Integer " do
+      expect(page.height).to eq 29_152
+    end
+
+    it "gets identifier from ALTO as primary file name" do
+      expect(page.identifier).to eq "0001"
+    end
+  end
+end

data/spec/lib/newspaper_works/ingest/newspaper_issue_ingest_spec.rb

@@ -0,0 +1,83 @@
+require 'spec_helper'
+
+# test NewspaperIssueIngest against a NewspaperIssue
+RSpec.describe NewspaperWorks::Ingest::NewspaperIssueIngest do
+  # define the path to the file we will use for multiple examples
+  let(:path) do
+    fixtures = File.join(NewspaperWorks::GEM_PATH, 'spec/fixtures/files')
+    Hyrax.config.whitelisted_ingest_dirs.push(fixtures)
+    File.join(fixtures, 'sample-4page-issue.pdf')
+  end
+
+  let(:path2) do
+    fixtures = File.join(NewspaperWorks::GEM_PATH, 'spec/fixtures/files')
+    File.join(fixtures, 'ndnp-sample1.pdf')
+  end
+
+  it_behaves_like('ingest adapter IO')
+
+  describe "file import and attachment" do
+    do_now_jobs = [
+      IngestJob,
+      IngestLocalFileJob,
+      InheritPermissionsJob,
+      VisibilityCopyJob
+    ]
+
+    PERMISSION_METHODS = [
+      :edit_users,
+      :read_users,
+      :discover_users,
+      :edit_groups,
+      :read_groups,
+      :discover_groups
+    ].freeze
+
+    def check_equivalent_permissions(obj1, obj2)
+      PERMISSION_METHODS.each do |m|
+        expect(obj1.send(m)).to match_array obj2.send(m)
+      end
+      expect(obj1.visibility).to eq obj2.visibility
+    end
+
+    def check_page_metadata(page)
+      expect(page.date_uploaded).not_to be nil
+      expect(page.date_modified).not_to be nil
+      # title: issue title plus page qualifier expected:
+      expect(page.title).to contain_exactly "Here and There: Page 1"
+      # page number is sequence number, expressed as String
+      expect(page.page_number).to be_a String
+      expect(page.page_number).to match(/^[0-9]+$/)
+    end
+
+    def assign_custom_permissions(work)
+      # read_groups ['public'] <==> "open" visibility
+      work.read_groups = ['public']
+      # add a permission to issue, that is not default:
+      work.read_users = ['peanutbutter@example.com']
+      work.save!
+    end
+
+    it "ingests work and creates child page works" do
+      adapter = build(:newspaper_issue_ingest)
+      adapter.ingest(path)
+      child_pages = adapter.work.members.select { |w| w.class == NewspaperPage }
+      expect(child_pages.length).to eq 4
+    end
+
+    # For created child pages, date and permission attributes are side-effect
+    #   of file attachment process (`NewspaperWorks::Data::WorkFiles`)
+    #   manipulating the work through the Hyrax actor stack create pipeline.
+    it "sets work attributes on created pages via file attachment",
+       peform_enqueued: do_now_jobs do
+      adapter = build(:newspaper_issue_ingest)
+      assign_custom_permissions(adapter.work)
+      adapter.ingest(path2)
+      child_pages = adapter.work.members.select { |w| w.class == NewspaperPage }
+      page = child_pages[0]
+      check_page_metadata(page)
+      # permissions:
+      check_equivalent_permissions(adapter.work, page)
+    end
+  end
+end