newspaper_works 0.1.0 → 1.0.0

data/spec/lib/newspaper_works/text_extraction/hocr_reader_spec.rb

@@ -0,0 +1,45 @@
+require 'json'
+require 'nokogiri'
+require 'spec_helper'
+
+RSpec.describe NewspaperWorks::TextExtraction::HOCRReader do
+  let(:fixture_path) do
+    File.join(
+      NewspaperWorks::GEM_PATH, 'spec', 'fixtures', 'files'
+    )
+  end
+
+  let(:minimal_path) { File.join(fixture_path, 'ocr_mono_text_hocr.html') }
+  let(:minimal) { File.read(minimal_path) }
+
+  let(:reader_minimal) { described_class.new(minimal) }
+  let(:reader_minimal_path) { described_class.new(minimal_path) }
+
+  describe "reads hOCR" do
+    it "loads hOCR either from path or source text" do
+      expect(reader_minimal_path.source).to eq reader_minimal.source
+      # size here is in Unicode characters, not bytes:
+      expect(reader_minimal_path.source.size).to eq 16_590
+    end
+
+    it "loads document stream" do
+      expect(reader_minimal_path.doc_stream).to be_kind_of Nokogiri::XML::SAX::Document
+      expect(reader_minimal_path.doc_stream).to respond_to :text
+      expect(reader_minimal_path.doc_stream).to respond_to :words
+    end
+  end
+
+  describe "outputs text derivative formats" do
+    it "outputs plain text" do
+      plain_text = reader_minimal.text
+      expect(plain_text.slice(0, 40)).to eq "_A  FEARFUL  ADVENTURE.\n‘The  Missouri. "
+      expect(reader_minimal.text).to eq reader_minimal.doc_stream.text
+      expect(reader_minimal.text.size).to eq 831
+    end
+
+    it "passes args to WordCoordsBuilder and receives output" do
+      parsed = JSON.parse(reader_minimal.json)
+      expect(parsed['coords'].length).to be > 1
+    end
+  end
+end

data/spec/lib/newspaper_works/text_extraction/page_ocr_spec.rb

@@ -26,7 +26,7 @@ RSpec.describe NewspaperWorks::TextExtraction::PageOCR do
       expect(words).to be_an(Array)
       expect(words).not_to be_empty
       expect(words[0]).to be_a(Hash)
-      [:word, :x_start, :y_start, :x_end, :y_end].each do |key|
+      [:word, :coordinates].each do |key|
         expect(words[0].keys).to include key
       end
     end
@@ -77,8 +77,8 @@ RSpec.describe NewspaperWorks::TextExtraction::PageOCR do
       word = ocr_from_gray_tiff.words[0]
       word1 = parsed['coords'][word[:word]]
       word1_coords = word1[0]
-      expect(word1_coords[2]).to eq word[:x_end] - word[:x_start]
-      expect(word1_coords[3]).to eq word[:y_end] - word[:y_start]
+      expect(word1_coords[2]).to eq word[:coordinates][2]
+      expect(word1_coords[3]).to eq word[:coordinates][3]
     end
   end
 end

data/spec/lib/newspaper_works/text_extraction/render_alto_spec.rb

@@ -16,20 +16,20 @@ RSpec.describe NewspaperWorks::TextExtraction::RenderAlto do
 
   let(:words) do
     [
-      { word: 'If',  x_start: 52, y_start: 13, x_end: 63, y_end: 27 },
-      { word: 'you', x_start: 69, y_start: 17, x_end: 100, y_end: 31 },
-      { word: 'are', x_start: 108, y_start: 17, x_end: 136, y_end: 27 },
-      { word: 'a', x_start: 143, y_start: 17, x_end: 151, y_end: 27 },
-      { word: 'friend,', x_start: 158, y_start: 13, x_end: 214, y_end: 29 },
-      { word: 'you', x_start: 51, y_start: 39, x_end: 82, y_end: 53 },
-      { word: 'speak', x_start: 90, y_start: 35, x_end: 140, y_end: 53 },
-      { word: 'the', x_start: 146, y_start: 35, x_end: 174, y_end: 49 },
-      { word: 'password,', x_start: 182, y_start: 35, x_end: 267, y_end: 53 },
-      { word: 'and', x_start: 51, y_start: 57, x_end: 81, y_end: 71 },
-      { word: 'the', x_start: 89, y_start: 57, x_end: 117, y_end: 71 },
-      { word: 'doors', x_start: 124, y_start: 57, x_end: 172, y_end: 71 },
-      { word: 'will', x_start: 180, y_start: 57, x_end: 208, y_end: 71 },
-      { word: 'open.', x_start: 216, y_start: 61, x_end: 263, y_end: 75 }
+      { word: "If", coordinates: [52, 13, 11, 14] },
+      { word: "you", coordinates: [69, 17, 31, 14] },
+      { word: "are", coordinates: [108, 17, 28, 10] },
+      { word: "a", coordinates: [143, 17, 8, 10] },
+      { word: "friend,", coordinates: [158, 13, 56, 16] },
+      { word: "you", coordinates: [51, 39, 31, 14] },
+      { word: "speak", coordinates: [90, 35, 50, 18] },
+      { word: "the", coordinates: [146, 35, 28, 14] },
+      { word: "password,", coordinates: [182, 35, 85, 18] },
+      { word: "and", coordinates: [51, 57, 30, 14] },
+      { word: "the", coordinates: [89, 57, 28, 14] },
+      { word: "doors", coordinates: [124, 57, 48, 14] },
+      { word: "will", coordinates: [180, 57, 28, 14] },
+      { word: "open.", coordinates: [216, 61, 47, 14] }
     ]
   end
 

data/spec/services/newspaper_works/jp2_derivative_service_spec.rb

@@ -21,30 +21,27 @@ RSpec.describe NewspaperWorks::JP2DerivativeService do
       Hyrax::DerivativePath.derivative_path_for_reference(file_set, 'jp2')
     end
 
-    def get_res(path)
-      lines = `gm identify -verbose #{path}`.lines
-      lines.select { |line| line.strip.start_with?('Geometry') }[0].strip
-    end
-
-    def check_dpi_match(orig, dest)
-      # check ppi, but skip pdf to avoid ghostscript warnings to stderr
-      expect(get_res(orig)).to eq get_res(dest) unless orig.end_with?('pdf')
+    def metadata_match_checker(source, target)
+      target_meta = NewspaperWorks::ImageTool.new(target).metadata
+      source_meta = NewspaperWorks::ImageTool.new(source).metadata
+      expect(target_meta[:content_type]).to eq 'image/jp2'
+      expect(target_meta[:width]).to eq source_meta[:width]
+      expect(target_meta[:height]).to eq source_meta[:height]
     end
 
     def makes_jp2(filename)
       expected = expected_path(valid_file_set)
       expect(File.exist?(expected)).to be false
       svc = described_class.new(valid_file_set)
-      svc.create_derivatives(source_image(filename))
+      source_path = source_image(filename)
+      svc.create_derivatives(source_path)
       expect(File.exist?(expected)).to be true
-      desc = `gm identify #{expected}`
-      expect(desc).to include 'JP2'
-      check_dpi_match(source_image(filename), expected)
+      metadata_match_checker(source_path, expected)
       svc.cleanup_derivatives
     end
 
     it "creates gray JP2 derivative from one-bit source" do
-      makes_jp2('page1.tiff')
+      makes_jp2('ocr_mono.tiff')
     end
 
     it "creates gray JP2 from grayscale source" do

data/spec/services/newspaper_works/newspaper_page_derivative_service_spec.rb

@@ -96,30 +96,32 @@ RSpec.describe NewspaperWorks::NewspaperPageDerivativeService do
     end
 
     it "identifies a source file using ImageMagick" do
-      expect(service_for_file('4.1.07.tiff').identify).to include 'TIFF'
-      expect(service_for_file('4.1.07.tiff').identify).to include '8-bit'
+      service = service_for_file('4.1.07.tiff')
+      expect(service.identify[:content_type]).to eq 'image/tiff'
+      expect(service.identify[:bits_per_component]).to eq 8
     end
 
     it "identifies jp2 source" do
-      # test/verify jp2 source is identified, which relies on GraphicsMagick
-      expect(service_for_file('4.1.07.jp2').identify).to include 'JP2'
-      expect(service_for_file('4.1.07.jp2').identify).to include '8-bit'
+      # test/verify jp2 source is identified, which relies on JP2 backend
+      service = service_for_file('4.1.07.jp2')
+      expect(service.identify[:content_type]).to eq 'image/jp2'
+      expect(service.identify[:bits_per_component]).to eq 8
     end
 
     it "identifies color and gray sources" do
       expect(service_for_file('4.1.07.tiff').use_color?).to be true
-      expect(service_for_file('page1.tiff').use_color?).to be false
+      expect(service_for_file('ocr_gray.tiff').use_color?).to be false
     end
 
     it "identifies a one-bit source" do
       # 1-bit group4 monochrome TIFF:
-      expect(service_for_file('page1.tiff').one_bit?).to be true
+      expect(service_for_file('ocr_mono.tiff').one_bit?).to be true
       # 8-bit gray TIFF:
       expect(
         service_for_file('lowres-gray-via-ndnp-sample.tiff').one_bit?
       ).to be false
       # color TIFF:
-      expect(service_for_file('4.1.07.tif').one_bit?).to be false
+      expect(service_for_file('4.1.07.tiff').one_bit?).to be false
     end
   end
 end

data/spec/services/newspaper_works/pdf_derivative_service_spec.rb

@@ -23,10 +23,10 @@ RSpec.describe NewspaperWorks::PDFDerivativeService do
 
     # given output file name, check DPI is 150
     def check_dpi(expected)
-      desc = `gm identify #{expected}`
-      # get total width of pdf in points from identify, should be 864x == 12in
-      page_width = 864
-      expect(desc).to include "#{page_width}x"
+      metadata = NewspaperWorks::ImageTool.new(expected).metadata
+      # get width of pdf in points (via imagemagick), should be 864x == 12in
+      page_width = metadata[:width]
+      expect(page_width).to eq 864
       # get total width of image in pixels from pdfimages -list, ==> 1800
       image_width = 1800
       im_list = `pdfimages -list #{expected}`
@@ -41,14 +41,14 @@ RSpec.describe NewspaperWorks::PDFDerivativeService do
       svc = described_class.new(valid_file_set)
       svc.create_derivatives(source_image(filename))
       expect(File.exist?(expected)).to be true
-      desc = `gm identify #{expected}`
-      expect(desc).to include 'PDF'
+      metadata = NewspaperWorks::ImageTool.new(expected).metadata
+      expect(metadata[:content_type]).to eq 'application/pdf'
       check_dpi(expected)
       svc.cleanup_derivatives
     end
 
     it "creates gray PDF derivative from one-bit source" do
-      makes_pdf('page1.tiff')
+      makes_pdf('ocr_mono.tiff')
     end
 
     it "creates gray PDF from grayscale source" do
@@ -58,5 +58,9 @@ RSpec.describe NewspaperWorks::PDFDerivativeService do
     it "creates color PDF from color source" do
       makes_pdf('4.1.07.tiff')
     end
+
+    it "creates color PDF from color JP2 source" do
+      makes_pdf('4.1.07.jp2')
+    end
   end
 end

data/spec/services/newspaper_works/tiff_derivative_service_spec.rb

@@ -22,8 +22,8 @@ RSpec.describe NewspaperWorks::TIFFDerivativeService do
     end
 
     def get_res(path)
-      lines = `gm identify -verbose #{path}`.lines
-      lines.select { |line| line.strip.start_with?('Geometry') }[0].strip
+      tool = NewspaperWorks::ImageTool.new(path)
+      "#{tool.metadata[:width]}x#{tool.metadata[:height]}"
     end
 
     def check_dpi_match(orig, dest)
@@ -32,23 +32,30 @@ RSpec.describe NewspaperWorks::TIFFDerivativeService do
     end
 
     def makes_tiff(filename)
+      path = source_image(filename)
       expected = expected_path(valid_file_set)
       expect(File.exist?(expected)).to be false
       svc = described_class.new(valid_file_set)
-      svc.create_derivatives(source_image(filename))
+      svc.create_derivatives(path)
       expect(File.exist?(expected)).to be true
-      desc = `gm identify #{expected}`
-      expect(desc).to include 'TIFF'
-      check_dpi_match(source_image(filename), expected)
+      mime = NewspaperWorks::ImageTool.new(expected).metadata[:content_type]
+      expect(mime).to eq 'image/tiff'
+      check_dpi_match(path, expected)
       svc.cleanup_derivatives
     end
 
-    it "creates gray TIFF derivative from one-bit source" do
-      makes_tiff('page1.tiff')
+    # for cases where primary file is TIFF already
+    def avoids_duplicative_creation(filename)
+      expected = expected_path(valid_file_set)
+      expect(File.exist?(expected)).to be false
+      svc = described_class.new(valid_file_set)
+      svc.create_derivatives(source_image(filename))
+      expect(File.exist?(expected)).not_to be true
     end
 
-    it "creates gray TIFF from grayscale source" do
-      makes_tiff('lowres-gray-via-ndnp-sample.tiff')
+    it "Does not make TIFF derivatives when primary is TIFF" do
+      avoids_duplicative_creation('ocr_mono.tiff')
+      avoids_duplicative_creation('ocr_gray.tiff')
     end
 
     it "creates TIFF from PDF source, robust to multi-page" do

data/spec/spec_helper.rb

@@ -64,6 +64,22 @@ module EngineRoutes
   end
 end
 
+class CSVLoggingFormatter < RSpec::Core::Formatters::JsonFormatter
+  RSpec::Core::Formatters.register self
+
+  def close(_notification)
+    with_headers = {
+      write_headers: true,
+      headers: ['Example', 'Status', 'Run Time', 'Exception']
+    }
+    CSV.open(output.path, 'w', with_headers) do |csv|
+      @output_hash[:examples].map do |ex|
+        csv << [ex[:full_description], ex[:status], ex[:run_time], ex[:exception]]
+      end
+    end
+  end
+end
+
 RSpec.configure do |config|
   # enable FactoryBot:
   require 'factory_bot'
@@ -226,6 +242,9 @@ RSpec.configure do |config|
   #  config.default_formatter = "doc"
   # end
 
+  # opt-in CSV logging formatter, set SPEC_CSV environment variable to use:
+  config.add_formatter(CSVLoggingFormatter, 'spec_log.csv') unless ENV['SPEC_CSV'].nil?
+
   # Print the 10 slowest examples and example groups at the
   # end of the spec run, to help surface which specs are running
   # particularly slow.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: newspaper_works
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Sean Upton
@@ -11,22 +11,22 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-27 00:00:00.000000000 Z
+date: 2019-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: blacklight_iiif_search
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: blacklight_advanced_search
   requirement: !ruby/object:Gem::Requirement
@@ -83,20 +83,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.1'
-- !ruby/object:Gem::Dependency
-  name: rtesseract
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 2.2.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 2.2.0
 - !ruby/object:Gem::Dependency
   name: sass-rails
   requirement: !ruby/object:Gem::Requirement
@@ -371,6 +357,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".coveralls.yml"
 - ".fcrepo_wrapper"
 - ".gitignore"
 - ".rubocop.yml"
@@ -577,6 +564,7 @@ files:
 - config/test-fixture/solr-config/xslt/example_atom.xsl
 - config/test-fixture/solr-config/xslt/example_rss.xsl
 - config/test-fixture/solr-config/xslt/luke.xsl
+- config/vendor/fits.xml
 - config/vendor/imagemagick-6-policy.xml
 - db/migrate/20181214181358_create_newspaper_works_derivative_attachments.rb
 - db/migrate/20190107165909_create_newspaper_works_ingest_file_relations.rb
@@ -603,6 +591,7 @@ files:
 - lib/newspaper_works/data/work_files.rb
 - lib/newspaper_works/engine.rb
 - lib/newspaper_works/errors.rb
+- lib/newspaper_works/image_tool.rb
 - lib/newspaper_works/ingest.rb
 - lib/newspaper_works/ingest/base_ingest.rb
 - lib/newspaper_works/ingest/base_publication_info.rb
@@ -639,11 +628,13 @@ files:
 - lib/newspaper_works/ingest/pub_finder.rb
 - lib/newspaper_works/ingest/publication_info.rb
 - lib/newspaper_works/issue_pdf_composer.rb
+- lib/newspaper_works/jp2_image_metadata.rb
 - lib/newspaper_works/logging.rb
 - lib/newspaper_works/page_finder.rb
 - lib/newspaper_works/resource_fetcher.rb
 - lib/newspaper_works/text_extraction.rb
 - lib/newspaper_works/text_extraction/alto_reader.rb
+- lib/newspaper_works/text_extraction/hocr_reader.rb
 - lib/newspaper_works/text_extraction/page_ocr.rb
 - lib/newspaper_works/text_extraction/render_alto.rb
 - lib/newspaper_works/text_extraction/word_coords_builder.rb
@@ -689,6 +680,7 @@ files:
 - spec/fixtures/files/ocr_gray.jp2
 - spec/fixtures/files/ocr_gray.tiff
 - spec/fixtures/files/ocr_mono.tiff
+- spec/fixtures/files/ocr_mono_text_hocr.html
 - spec/fixtures/files/page1.tiff
 - spec/fixtures/files/resource_mocks/chronam/http404-expected
 - spec/fixtures/files/resource_mocks/chronam/sn84038814.rdf
@@ -732,6 +724,7 @@ files:
 - spec/lib/newspaper_works/data/work_derivatives_spec.rb
 - spec/lib/newspaper_works/data/work_file_spec.rb
 - spec/lib/newspaper_works/data/work_files_spec.rb
+- spec/lib/newspaper_works/image_tool_spec.rb
 - spec/lib/newspaper_works/ingest/batch_issue_ingester_spec.rb
 - spec/lib/newspaper_works/ingest/chronam_publication_info_spec.rb
 - spec/lib/newspaper_works/ingest/from_command_spec.rb
@@ -761,10 +754,12 @@ files:
 - spec/lib/newspaper_works/ingest/publication_info_spec.rb
 - spec/lib/newspaper_works/ingest_spec.rb
 - spec/lib/newspaper_works/issue_pdf_composer_spec.rb
+- spec/lib/newspaper_works/jp2_image_metadata_spec.rb
 - spec/lib/newspaper_works/logging_spec.rb
 - spec/lib/newspaper_works/page_finder_spec.rb
 - spec/lib/newspaper_works/resource_fetcher_spec.rb
 - spec/lib/newspaper_works/text_extraction/alto_reader_spec.rb
+- spec/lib/newspaper_works/text_extraction/hocr_reader_spec.rb
 - spec/lib/newspaper_works/text_extraction/page_ocr_spec.rb
 - spec/lib/newspaper_works/text_extraction/render_alto_spec.rb
 - spec/lib/newspaper_works/text_extraction/word_coords_builder_spec.rb
@@ -830,7 +825,7 @@ files:
 - test/newspaper_works_test.rb
 - test/test_helper.rb
 - tmp/.keep
-homepage: https://github.com/marriott-library/newspaper_works
+homepage: https://github.com/samvera-labs/newspaper_works
 licenses:
 - Apache-2.0
 metadata: {}
@@ -850,7 +845,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14.3
+rubygems_version: 2.7.6.2
 signing_key: 
 specification_version: 4
 summary: newspaper_works is a Rails Engine gem providing model and administrative
@@ -896,6 +891,7 @@ test_files:
 - spec/fixtures/files/ocr_gray.jp2
 - spec/fixtures/files/ocr_gray.tiff
 - spec/fixtures/files/ocr_mono.tiff
+- spec/fixtures/files/ocr_mono_text_hocr.html
 - spec/fixtures/files/page1.tiff
 - spec/fixtures/files/resource_mocks/chronam/http404-expected
 - spec/fixtures/files/resource_mocks/chronam/sn84038814.rdf
@@ -939,6 +935,7 @@ test_files:
 - spec/lib/newspaper_works/data/work_derivatives_spec.rb
 - spec/lib/newspaper_works/data/work_file_spec.rb
 - spec/lib/newspaper_works/data/work_files_spec.rb
+- spec/lib/newspaper_works/image_tool_spec.rb
 - spec/lib/newspaper_works/ingest/batch_issue_ingester_spec.rb
 - spec/lib/newspaper_works/ingest/chronam_publication_info_spec.rb
 - spec/lib/newspaper_works/ingest/from_command_spec.rb
@@ -968,10 +965,12 @@ test_files:
 - spec/lib/newspaper_works/ingest/publication_info_spec.rb
 - spec/lib/newspaper_works/ingest_spec.rb
 - spec/lib/newspaper_works/issue_pdf_composer_spec.rb
+- spec/lib/newspaper_works/jp2_image_metadata_spec.rb
 - spec/lib/newspaper_works/logging_spec.rb
 - spec/lib/newspaper_works/page_finder_spec.rb
 - spec/lib/newspaper_works/resource_fetcher_spec.rb
 - spec/lib/newspaper_works/text_extraction/alto_reader_spec.rb
+- spec/lib/newspaper_works/text_extraction/hocr_reader_spec.rb
 - spec/lib/newspaper_works/text_extraction/page_ocr_spec.rb
 - spec/lib/newspaper_works/text_extraction/render_alto_spec.rb
 - spec/lib/newspaper_works/text_extraction/word_coords_builder_spec.rb