newspaper_works 0.1.0

data/spec/lib/newspaper_works/ingest/lc_publication_info_spec.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+
+RSpec.describe NewspaperWorks::Ingest::LCPublicationInfo do
+  let(:lccn1) { 'sn83021453' }
+  let(:lccn2) { 'sn83045396' }
+  let(:bad_lccn) { 'sn99999999' }
+
+  describe "gets metadata" do
+    it "gets simple metadata" do
+      meta = described_class.new(lccn1)
+      expect(meta.title).to eq 'Salt Lake daily tribune'
+      expect(meta.issn).to be_nil
+      expect(meta.oclcnum).to eq 'ocm10170377'
+      expect(meta.place_name).to eq 'Salt Lake City, Utah'
+      expect(meta.place_of_publication).to eq 'http://sws.geonames.org/5780993/'
+    end
+
+    it "gets related item metadata" do
+      meta1 = described_class.new(lccn1)
+      meta2 = described_class.new(lccn2)
+      # lccn2 succeeds lccn1, favors lccn.loc.gov URL as authoritative:
+      expect(meta1.succeeded_by).to eq "https://lccn.loc.gov/#{lccn2}"
+      # lccn1 precedes lccn2, favors lccn.loc.gov URL as authoritative:
+      expect(meta2.preceded_by).to eq "https://lccn.loc.gov/sn83021453"
+    end
+  end
+
+  describe "error handling" do
+    it "handles unknown LCCN (empty mods)" do
+      meta = described_class.new(bad_lccn)
+      expect(meta.empty?).to be true
+    end
+  end
+end

data/spec/lib/newspaper_works/ingest/ndnp/batch_ingester_spec.rb

@@ -0,0 +1,131 @@
+require 'spec_helper'
+require 'ndnp_shared'
+
+RSpec.describe NewspaperWorks::Ingest::NDNP::BatchIngester do
+  include_context "ndnp fixture setup"
+
+  describe "adapter construction" do
+    it "loads batch to operate on" do
+      adapter = described_class.new(batch1)
+      expect(adapter.batch).to be_a NewspaperWorks::Ingest::NDNP::BatchXMLIngest
+      expect(adapter.batch.path).to eq adapter.path
+    end
+
+    it "finds batch xml, if given path containing batch" do
+      parent_path = File.dirname(batch1)
+      adapter = described_class.new(parent_path)
+      expect(adapter.path).to eq batch1
+      expect(adapter.batch.path).to eq adapter.path
+    end
+
+    it "constructs adapter with hash options" do
+      user = User.batch_user.user_key
+      adapter = described_class.new(
+        batch1,
+        depositor: user
+      )
+      expect(adapter.opts[:depositor]).to eq user
+    end
+  end
+
+  describe "ingests issues" do
+    def expect_start_finish_logging(adapter)
+      expect(adapter).to receive(:write_log).with(
+        satisfy { |v| v.include?('Beginning NDNP batch ingest') }
+      ).once
+      expect(adapter).to receive(:write_log).with(
+        satisfy { |v| v.include?('NDNP batch ingest complete') }
+      ).once
+    end
+
+    it "calls ingest for all issues in batch" do
+      adapter = described_class.new(batch1)
+      issue_ingest_call_count = 0
+      # rubocop:disable RSpec/AnyInstance (we really need to stub this way)
+      allow_any_instance_of(NewspaperWorks::Ingest::NDNP::IssueIngester).to \
+        receive(:ingest) { issue_ingest_call_count += 1 }
+      # rubocop:enable RSpec/AnyInstance
+      expect_start_finish_logging(adapter)
+      adapter.ingest
+      expect(issue_ingest_call_count).to eq 4
+    end
+  end
+
+  describe "command invocation" do
+    def construct(args)
+      described_class.from_command(
+        args,
+        'rake newspaper_works:ingest_ndnp --'
+      )
+    end
+
+    let(:fake_argv) do
+      [
+        'newspaper_works:ingest_ndnp',
+        '--',
+        "--path=#{batch1}"
+      ]
+    end
+
+    let(:fake_argv2) do
+      [
+        'newspaper_works:ingest_ndnp',
+        '--',
+        "--path=#{batch1}",
+        "--admin_set=admin_set/default",
+        "--depositor=#{User.batch_user.user_key}",
+        "--visibility=open"
+      ]
+    end
+
+    it "creates ingester from command arguments" do
+      adapter = construct(fake_argv)
+      expect(adapter).to be_a described_class
+      expect(adapter.path).to eq batch1
+    end
+
+    it "creates ingester from expanded command arguments" do
+      adapter = construct(fake_argv2)
+      expect(adapter).to be_a described_class
+      expect(adapter.path).to eq batch1
+      expect(adapter.opts[:depositor]).to eq User.batch_user.user_key
+      expect(adapter.opts[:visibility]).to eq 'open'
+      expect(adapter.opts[:admin_set]).to eq 'admin_set/default'
+    end
+
+    it "creates ingester from command with dir path" do
+      # command can accept a parent directory for batch:
+      base_path = File.dirname(batch1)
+      fake_argv = ['newspaper_works:ingest_ndnp', '--', "--path=#{base_path}"]
+      adapter = construct(fake_argv)
+      expect(adapter).to be_a described_class
+      # adapter.path is path to actual XML
+      expect(adapter.path).to eq batch1
+    end
+
+    it "exits on file not found for batch" do
+      fake_argv = ['newspaper_works:ingest_ndnp', '--', "--path=123/45/5678"]
+      begin
+        construct(fake_argv)
+      rescue SystemExit => e
+        expect(e.status).to eq(1)
+      end
+    end
+
+    it "exits on missing path for batch" do
+      fake_argv = ['newspaper_works:ingest_ndnp', '--']
+      begin
+        construct(fake_argv)
+      rescue SystemExit => e
+        expect(e.status).to eq(1)
+      end
+    end
+
+    it "exits on unexpected arguments" do
+      fake_argv = ['newspaper_works:ingest_ndnp', '--', '--foo=bar']
+      expect { construct(fake_argv) }.to raise_error(
+        OptionParser::InvalidOption
+      )
+    end
+  end
+end

data/spec/lib/newspaper_works/ingest/ndnp/batch_xml_ingest_spec.rb

@@ -0,0 +1,64 @@
+require 'spec_helper'
+require 'ndnp_shared'
+
+RSpec.describe NewspaperWorks::Ingest::NDNP::BatchXMLIngest do
+  include_context "ndnp fixture setup"
+
+  describe "sample batch" do
+    let(:batch) { described_class.new(batch1) }
+
+    it "gets batch name" do
+      expect(batch.name).to eq 'batch_test'
+    end
+
+    it "gets issue by path" do
+      path = batch.issue_paths[0]
+      issue = batch.get(path)
+      expect(issue).to be_a NewspaperWorks::Ingest::NDNP::IssueIngest
+      expect(issue.path).to eq path
+    end
+
+    it "gets reel/container by path" do
+      path = batch.container_paths[0]
+      container = batch.get(path)
+      expect(container).to be_a NewspaperWorks::Ingest::NDNP::ContainerIngest
+      expect(container.path).to eq path
+    end
+
+    it "enumerates container paths" do
+      reel_ids = batch.container_paths
+      expect(reel_ids).to be_an Array
+      expect(reel_ids.size).to eq 3
+    end
+
+    it "enumerates issue paths" do
+      issue_ids = batch.issue_paths
+      expect(issue_ids).to be_an Array
+      expect(issue_ids.size).to eq 4
+    end
+
+    it "enumerates issues via method" do
+      issues = batch.issues
+      expect(issues).to be_an Array
+      expect(issues.size).to eq 4
+      expect(issues[0]).to be_a NewspaperWorks::Ingest::NDNP::IssueIngest
+    end
+
+    it "makes batch fixed-size enumerable of issues" do
+      expect(batch.size).to eq batch.issue_paths.size
+      issues = batch.to_a # implied .each
+      expect(issues.size).to eq batch.size
+      expect(issues.size).to eq 4
+      issues.each do |issue|
+        expect(issue).to be_a NewspaperWorks::Ingest::NDNP::IssueIngest
+      end
+    end
+
+    it "enumerates containers" do
+      reels = batch.containers
+      expect(reels).to be_an Array
+      expect(reels.size).to eq 3
+      expect(reels[0]).to be_a NewspaperWorks::Ingest::NDNP::ContainerIngest
+    end
+  end
+end

data/spec/lib/newspaper_works/ingest/ndnp/container_ingest_spec.rb

@@ -0,0 +1,44 @@
+require 'spec_helper'
+require 'ndnp_shared'
+
+RSpec.describe NewspaperWorks::Ingest::NDNP::ContainerIngest do
+  include_context "ndnp fixture setup"
+
+  describe "sample fixture 'batch_test_ver01'" do
+    let(:reel) { described_class.new(reel1) }
+
+    it "gets metadata" do
+      expect(reel.metadata).to be_a \
+        NewspaperWorks::Ingest::NDNP::ContainerMetadata
+      # uses same Nokogiri document context:
+      expect(reel.metadata.doc).to be reel.doc
+      # has identifier method equivalent to reel number
+      expect(reel.identifier).to eq reel.metadata.reel_number
+    end
+
+    it "gets control image as PageIngest, by dmdid" do
+      page = reel.page_by_dmdid('targetModsBib1')
+      expect(page).to be_a NewspaperWorks::Ingest::NDNP::PageIngest
+      expect(page.dmdid).to eq 'targetModsBib1'
+    end
+
+    it "shares xml document context with contained pages" do
+      page = reel.page_by_dmdid('targetModsBib1')
+      expect(page.doc).to be reel.doc
+    end
+
+    it "enumerates expected issues" do
+      # enumerate by casting reel to Array
+      issues = reel.to_a
+      expect(issues.size).to eq 2
+      expect(issues[0]).to be_a NewspaperWorks::Ingest::NDNP::IssueIngest
+      expect(issues[0].path).to eq reel.issue_paths[0]
+    end
+
+    it "gets size, in issue count" do
+      issues = reel.to_a
+      expect(reel.size).to eq issues.size
+      expect(reel.size).to eq reel.issue_paths.size
+    end
+  end
+end

data/spec/lib/newspaper_works/ingest/ndnp/container_ingester_spec.rb

@@ -0,0 +1,126 @@
+require 'spec_helper'
+require 'ndnp_shared'
+require 'misc_shared'
+
+RSpec.describe NewspaperWorks::Ingest::NDNP::ContainerIngester do
+  include_context "ndnp fixture setup"
+  include_context "shared setup"
+
+  # use FactoryBot for publication, issue
+  let(:publication) { create(:newspaper_title) }
+  let(:issue) do
+    issue = create(:newspaper_issue)
+    publication.members << issue
+    publication.save!
+    issue
+  end
+  let(:linked_publication) { issue.publication }
+
+  let(:issue_data) do
+    NewspaperWorks::Ingest::NDNP::IssueIngest.new(issue2)
+  end
+
+  let(:page_data) do
+    data = issue_data.to_a[0]
+    # some NDNP samples missing TIFF, put dummy in place of missing, as needed
+    data.files = data.files.map do |path|
+      File.exist?(path) ? path : File.join(fixture_path, 'ocr_gray.tiff')
+    end
+    data
+  end
+
+  let(:page) do
+    ingester = NewspaperWorks::Ingest::NDNP::PageIngester.new(page_data, issue)
+    ingester.ingest
+    ingester.target
+  end
+
+  # reel data via ContainerIngest, ContainerMetadata objects:
+  let(:reel_data) { issue_data.container }
+  let(:metadata) { reel_data.metadata }
+  let(:sn) { metadata.reel_number }
+
+  describe "asset construction and linking" do
+    before do
+      # trick with testing ingest that does find-or-create on reel is
+      #   that we want to clear previous reel assets left over from
+      #   other tests, most of the time.
+      containers = NewspaperContainer.where(identifier: sn)
+      next if containers.size.zero?
+      # first, unlink the reel from publication before deleting:
+      container = containers[0]
+      publication = container.publication
+      publication.members.delete(container) unless publication.nil?
+      # then delete reel
+      containers.delete_all
+    end
+
+    it "constructs adapter wth reel_data, publication asset" do
+      adapter = described_class.new(reel_data, linked_publication)
+      expect(adapter.source).to be reel_data
+      expect(adapter.publication).to be linked_publication
+    end
+
+    it "constructs a publication-linked asset for reel" do
+      adapter = described_class.new(reel_data, linked_publication)
+      expect(adapter.target).to be_nil
+      adapter.ingest
+      expect(adapter.target).to be_a NewspaperContainer
+      expect(adapter.target.publication).to be_a NewspaperTitle
+    end
+
+    it "links pages to reel" do
+      adapter = described_class.new(reel_data, linked_publication)
+      adapter.ingest
+      adapter.link(page)
+      expect(adapter.target.pages.map(&:id)).to include page.id
+      expect(adapter.target.pages.size).to eq 1
+    end
+
+    it "copies reel metadata" do
+      adapter = described_class.new(reel_data, linked_publication)
+      adapter.ingest
+      reel = adapter.target
+      expect(reel.identifier).to contain_exactly sn
+      expect(reel.held_by).to eq metadata.held_by
+      expect(reel.title).to contain_exactly "Microform reel (#{sn})"
+      expect(reel.publication_date_start).to eq metadata.publication_date_start
+      expect(reel.publication_date_end).to eq metadata.publication_date_end
+    end
+
+    it "sets default administrative metadata with default construction" do
+      adapter = described_class.new(reel_data, linked_publication)
+      adapter.ingest
+      asset = adapter.target
+      expect(asset.depositor).to eq User.batch_user.user_key
+      expect(asset.admin_set).to eq AdminSet.find(AdminSet::DEFAULT_ID)
+      expect(asset.visibility).to eq 'open'
+    end
+
+    it "sets custom administrative metadata" do
+      # test one exemplary/representative option:
+      adapter = described_class.new(
+        reel_data,
+        linked_publication,
+        visibility: 'open'
+      )
+      adapter.ingest
+      expect(adapter.target.visibility).to eq 'open'
+    end
+
+    it "finds or creates container asset for reel" do
+      # No initial container for thre reel id, per before block above:
+      expect(NewspaperContainer.where(identifier: sn).size).to eq 0
+      # create it once
+      described_class.new(reel_data, linked_publication).ingest
+      result = NewspaperContainer.where(identifier: sn)
+      expect(result.size).to eq 1
+      expect(result.first.identifier).to contain_exactly sn
+      # now do this again, expecting to find the existing container asset:
+      described_class.new(reel_data, linked_publication).ingest
+      result = NewspaperContainer.where(identifier: sn)
+      expect(result.size).to eq 1 # still just one asset
+      expect(result.first.identifier).to contain_exactly sn
+    end
+  end
+end

data/spec/lib/newspaper_works/ingest/ndnp/container_metadata_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+require 'ndnp_shared'
+
+RSpec.describe NewspaperWorks::Ingest::NDNP::ContainerMetadata do
+  include_context "ndnp fixture setup"
+
+  describe "sample fixture 'batch_test_ver01'" do
+    let(:meta) { described_class.new(reel1) }
+    let(:meta2) { described_class.new(reel2) }
+
+    it "gets reel_number" do
+      expect(meta.reel_number).to eq "00279557177"
+      expect(meta2.reel_number).to eq "00279557281"
+    end
+
+    it "gets held_by" do
+      expect(meta.held_by).to eq "University of Utah, Salt Lake City, UT"
+    end
+
+    it "gets genre" do
+      expect(meta.genre).to eq 'microfilm'
+    end
+
+    it "gets title" do
+      expect(meta.title).to eq 'Daily national Democrat (Marysville, Calif.)'
+    end
+
+    it "gets start date" do
+      expect(meta.publication_date_start).to eq '1858-08-13'
+    end
+
+    it "gets end date" do
+      expect(meta.publication_date_end).to eq '1858-12-31'
+    end
+  end
+end

data/spec/lib/newspaper_works/ingest/ndnp/issue_ingest_spec.rb

@@ -0,0 +1,108 @@
+require 'spec_helper'
+require 'ndnp_shared'
+
+RSpec.describe NewspaperWorks::Ingest::NDNP::IssueIngest do
+  include_context "ndnp fixture setup"
+
+  describe "sample fixture 'batch_local'" do
+    let(:issue) { described_class.new(issue1) }
+
+    it "gets metadata" do
+      expect(issue.metadata).to be_a NewspaperWorks::Ingest::NDNP::IssueMetadata
+      # uses same Nokogiri document context:
+      expect(issue.metadata.doc).to be issue.doc
+      # has identifier method equivalent to lccn
+      expect(issue.identifier).to eq issue.metadata.lccn
+    end
+
+    it "gets nil container for issue without reel XML" do
+      reel = issue.container
+      expect(reel).to be_nil
+    end
+
+    it "gets page by dmdid" do
+      page = issue.page_by_dmdid('pageModsBib8')
+      expect(page).to be_a NewspaperWorks::Ingest::NDNP::PageIngest
+      expect(page.metadata.page_sequence_number).to eq 1
+      expect(page.dmdid).to eq 'pageModsBib8'
+    end
+
+    it "gets page by sequence number" do
+      page = issue.page_by_sequence_number(1)
+      expect(page.metadata.page_sequence_number).to eq 1
+      expect(page.dmdid).to eq 'pageModsBib8'
+      page = issue.page_by_sequence_number(2)
+      expect(page.metadata.page_sequence_number).to eq 2
+      expect(page.dmdid).to eq 'pageModsBib6'
+    end
+
+    it "shares xml document context with contained pages" do
+      page = issue.page_by_sequence_number(1)
+      expect(page.doc).to be issue.doc
+    end
+
+    it "enumerates expected pages" do
+      # enumerate by casting issue to Array
+      pages = issue.to_a
+      expect(pages.size).to eq 2
+      expect(pages[0]).to be_a NewspaperWorks::Ingest::NDNP::PageIngest
+      expect(pages[0].metadata.page_sequence_number).to eq 1
+    end
+
+    it "gets size, in page count" do
+      pages = issue.to_a
+      expect(issue.size).to eq pages.size
+      expect(issue.size).to eq issue.dmdids.size
+    end
+  end
+
+  describe "sample fixture 'batch_test_ver01'" do
+    let(:issue) { described_class.new(issue2) }
+
+    it "gets a ContainerIngest for reel providing issue" do
+      reel = issue.container
+      expect(reel).to be_a NewspaperWorks::Ingest::NDNP::ContainerIngest
+      expect(reel.path).to end_with '_1.xml'
+    end
+
+    it "gets metadata" do
+      expect(issue.metadata).to be_a NewspaperWorks::Ingest::NDNP::IssueMetadata
+      # uses same Nokogiri document context:
+      expect(issue.metadata.doc).to be issue.doc
+      # has identifier method equivalent to lccn
+      expect(issue.identifier).to eq issue.metadata.lccn
+    end
+
+    it "gets page by dmdid" do
+      page = issue.page_by_dmdid('pageModsBib1')
+      expect(page).to be_a NewspaperWorks::Ingest::NDNP::PageIngest
+      expect(page.metadata.page_sequence_number).to eq 1
+      expect(page.dmdid).to eq 'pageModsBib1'
+    end
+
+    it "shares xml document context with contained pages" do
+      page = issue.page_by_sequence_number(1)
+      expect(page.doc).to be issue.doc
+    end
+
+    it "gets page by sequence number" do
+      page = issue.page_by_sequence_number(1)
+      expect(page.metadata.page_sequence_number).to eq 1
+      expect(page.dmdid).to eq 'pageModsBib1'
+    end
+
+    it "enumerates expected pages" do
+      # enumerate by casting issue to Array
+      pages = issue.to_a
+      expect(pages.size).to eq 1
+      expect(pages[0]).to be_a NewspaperWorks::Ingest::NDNP::PageIngest
+      expect(pages[0].metadata.page_sequence_number).to eq 1
+    end
+
+    it "gets size, in page count" do
+      pages = issue.to_a
+      expect(issue.size).to eq pages.size
+      expect(issue.size).to eq issue.dmdids.size
+    end
+  end
+end