newspaper_works 0.1.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: d0003c7935fbee94e245007e986fcdf7a085bfa5
-  data.tar.gz: 9609ad4d442b7d7630e0231c09bd0592f9828933
+SHA256:
+  metadata.gz: 6e9b9c0ca19656d5b0654d96b5e3f8f31303dfe3a9df388643702a871b64ccbb
+  data.tar.gz: 485b605170e6740250ef85b0eac19880c84a4c2472486bffb1adc66beb548915
 SHA512:
-  metadata.gz: 1522953bfcacedaea668eaf9b59790a6db6ffe3be317ed0e7388a51fa44b4cf3518f463c422f65b5bfedce77ec9731df49f49ea9a0611d5f44041566d596c453
-  data.tar.gz: ddda11224cc5a059c0aa13546ac77805ec1e4e94eae7da2cd9f95eacf4fec8abcc78ec8b68d460939bceed0ce4904cb9b596beca5a20535fa0f83c69554756d4
+  metadata.gz: a180e466f1e80b83e538f7e3687e982f9a30892901f23044bcbb3cc2a89248686a717d097163fa5c4771c5a726bd2d86c7d9c3ab6ce3ae0c9a2d2404cf4ce7ff
+  data.tar.gz: 74794d3456d451b094d83f75378e506383ed0f8da876a9cb5882c696e594fb0a89df1cf300240651cbe3317bffa766672398950133b62b62949a684d08f12f7c

data/.coveralls.yml

@@ -0,0 +1,2 @@
+service_name: travis-ci
+

data/.gitignore

@@ -41,3 +41,7 @@ Gemfile.lock
 
 # Ignore wrapper downloads
 fcrepo-webapp-*
+
+# ignore gem builds
+*.gem
+pkg/

data/.travis.yml

@@ -30,8 +30,8 @@ before_install:
   - sudo unzip /opt/install/fits-1.4.1.zip -d /opt/install/fits-1.4.1
   - sudo chmod +x /opt/install/fits-1.4.1/*.sh
   - sudo ln -s /opt/install/fits-1.4.1/fits.sh /usr/local/bin/fits.sh
-  - sudo wget -q -O /etc/ImageMagick-6/policy.xml https://raw.githubusercontent.com/marriott-library/newspaper_works/7052ff3bf58022572870a0720d8c5a705090a833/config/vendor/imagemagick-6-policy.xml
-  - sudo wget -q -O /opt/install/fits-1.4.1/xml/fits.xml https://gist.githubusercontent.com/brianmcbride/af3fb21fb50a1331d84eca9a9a170f8f/raw/ebb1a285726e4f5c2309b4f6eceed29fa1369538/fits.xml
+  - sudo wget -q -O /etc/ImageMagick-6/policy.xml https://raw.githubusercontent.com/samvera-labs/newspaper_works/master/config/vendor/imagemagick-6-policy.xml
+  - sudo wget -q -O /opt/install/fits-1.4.1/xml/fits.xml https://raw.githubusercontent.com/samvera-labs/newspaper_works/82d0c18d1679696a762fd7083db83dac49de9b35/config/vendor/fits.xml
 rvm:
   - 2.5.5
 

data/README.md

@@ -1,7 +1,7 @@
 NewspaperWorks
 ===================================================
 Code:
-[![Build Status](https://travis-ci.org/marriott-library/newspaper_works.svg?branch=master)](https://travis-ci.org/marriott-library/newspaper_works) [![Coverage Status](https://coveralls.io/repos/github/marriott-library/newspaper_works/badge.svg)](https://coveralls.io/github/marriott-library/newspaper_works)
+[![Build Status](https://travis-ci.org/samvera-labs/newspaper_works.svg?branch=master)](https://travis-ci.org/samvera-labs/newspaper_works) [![Coverage Status](https://coveralls.io/repos/github/samvera-labs/newspaper_works/badge.svg?branch=master)](https://coveralls.io/github/samvera-labs/newspaper_works?branch=master)
 
 Docs:
 [![Apache 2.0 License](http://img.shields.io/badge/APACHE2-license-blue.svg)](./LICENSE)
@@ -21,7 +21,7 @@ Jump in: [![Slack Status](http://slack.samvera.org/badge.svg)](http://slack.samv
     - [Configuration changes you should make after running the installer](#configuration-changes-you-should-make-after-running-the-installer)
 - [Ingesting Content](#ingesting-content)
 - [Developing, Testing, and Contributing](#developing-testing-and-contributing)
-  - [Contributing](#contributing) 
+  - [Contributing](#contributing)
 - [Acknowledgements](#acknowledgements)
   - [Sponsoring Organizations](#sponsoring-organizations)
   - [More Information](#more-information)
@@ -45,10 +45,10 @@ NewspaperWorks supports:
 * OCR keyword match highlighting
 * viewer with page navigation and deep zooming
 
-A complete list of features can be found [here](wiki/Features-List).
+A complete list of features can be found [here](https://github.com/samvera-labs/newspaper_works/wiki/Features-List).
 
 ## Documentation
-A set of helpful documents to help you learn more and deploy NewspaperWorks can be found on the [Project Wiki](./newspaper_works/wiki), including a PCDM model diagram, metadata schema, batch ingest instructions, and more details on installing, developing, and testing the code. 
+A set of helpful documents to help you learn more and deploy NewspaperWorks can be found on the [Project Wiki](https://github.com/samvera-labs/newspaper_works/wiki), including a PCDM model diagram, metadata schema, batch ingest instructions, and more details on installing, developing, and testing the code.
 
 ## Requirements
 
@@ -66,10 +66,11 @@ A set of helpful documents to help you learn more and deploy NewspaperWorks can
   * [LibreOffice](https://www.libreoffice.org/)
   * [ghostscript](https://www.ghostscript.com/)
   * [poppler-utils](https://poppler.freedesktop.org/)
-  * [GraphicsMagick](http://www.graphicsmagick.org/)
+  * [ImageMagick](https://github.com/ImageMagick/ImageMagick6)
+    - _ImageMagick policy XML may need to be more permissive in both resources and source media types allowed.  See template [policy.xml](config/vendor/imagemagick-6-policy.xml)._
   * [libcurl3](https://packages.ubuntu.com/search?keywords=libcurl3)
 
-See the [wiki](./newspaper_works/wiki/Installing,-Developing,-and-Testing) for more details on how to install and configure dependencies.
+See the [wiki](https://github.com/samvera-labs/newspaper_works/wiki/Installing,-Developing,-and-Testing) for more details on how to install and configure dependencies.
 
 # Installation
 NewspaperWorks easily integrates with your Hyrax 2.5.x applications.
@@ -91,7 +92,7 @@ NewspaperWorks easily integrates with your Hyrax 2.5.x applications.
 ### Configuration changes you should make after running the installer:
 
 #### in config/intitializers/hyrax.rb:
-* set `config.geonames_username` 
+* set `config.geonames_username`
   * Enables geolocation tagging of content
   * [how to create a Geonames username](http://www.geonames.org/login)
 * set `config.work_requires_files = false`
@@ -106,9 +107,9 @@ NewspaperWorks easily integrates with your Hyrax 2.5.x applications.
 
 NewspaperWorks supports a range of different ingest workflows:
 * single-item ingest via the UI
-* batch ingest of [NDNP materials](./newspaper_works/wiki/NDNP-Batch-Ingest-Guide) (page-level digitization) via command line
-* batch ingest of [PDF issues](./newspaper_works/wiki/PDF-Batch-Ingest-Guide) via command line
-* batch ingest of [TIFF or JP2 master files](./newspaper_works/wiki/TIFF-or-JP2-Batch-Ingest-Guide) via command line
+* batch ingest of [NDNP materials](https://github.com/samvera-labs/newspaper_works/wiki/NDNP-Batch-Ingest-Guide) (page-level digitization) via command line
+* batch ingest of [PDF issues](https://github.com/samvera-labs/newspaper_works/wiki/PDF-Batch-Ingest-Guide) via command line
+* batch ingest of [TIFF or JP2 master files](https://github.com/samvera-labs/newspaper_works/wiki/TIFF-or-JP2-Batch-Ingest-Guide) via command line
 
 The ingest process creates a full complement of derivatives for each Page object, including:
 * TIFF
@@ -117,13 +118,13 @@ The ingest process creates a full complement of derivatives for each Page object
 * OCR text
 * word-coordinate JSON
 
-For more information on derivatives, see the [wiki](./newspaper_works/wiki/Image-Format-and-Derivative-Notes).
+For more information on derivatives, see the [wiki](https://github.com/samvera-labs/newspaper_works/wiki/Image-Format-and-Derivative-Notes).
 
 # Developing, Testing, and Contributing
 
-Detailed information regarding development and testing environments setup and configuration can be found [here](./newspaper_works/wiki/Installing,-Developing,-and-Testing)
+Detailed information regarding development and testing environments setup and configuration can be found [here](https://github.com/samvera-labs/newspaper_works/wiki/Installing,-Developing,-and-Testing)
 
-A Vagrant VM is available for users and developers to quickly and easily deploy the latest NewspaperWorks codebase using Vagrant and VirtualBox. See [samvera-newspapers-vagrant](https://github.com/marriott-library/samvera-newspapers-vagrant) for more.
+A Vagrant VM is available for users and developers to quickly and easily deploy the latest NewspaperWorks codebase using Vagrant and VirtualBox. See [samvera-newspapers-vagrant](https://github.com/samvera-labs/samvera-newspapers-vagrant) for more.
 
 Additionally, the [NewspaperWorks Demo Site](https://newspaperworks.digitalnewspapers.org/) is available for those interested in testing out NewspaperWorks as deployed in a vanilla Hyrax application. (**NOTE:** The demo site may not be running the latest release of NewspaperWorks.)
 

data/app/services/newspaper_works/jp2_derivative_service.rb

@@ -22,13 +22,11 @@ module NewspaperWorks
     # Target file extension of this service plugin:
     TARGET_EXT = 'jp2'.freeze
 
-    attr_accessor :source_meta
     attr_reader :file_set
     delegate :uri, :mime_type, to: :file_set
 
     def initialize(file_set)
       # cached result string for imagemagick `identify` command
-      @source_meta = nil
       @command = nil
       @unlink_after_creation = []
       super(file_set)
@@ -66,7 +64,7 @@ module NewspaperWorks
       # source introspection:
 
       def tiff_source?
-        identify.include?('TIFF')
+        identify[:content_type] == 'image/tiff'
       end
 
       def make_symlink

data/app/services/newspaper_works/newspaper_page_derivative_service.rb

@@ -2,7 +2,7 @@ module NewspaperWorks
   # Base type for derivative services specific to NewspaperPage only
   class NewspaperPageDerivativeService
     attr_reader :file_set, :master_format
-    delegate :uri, :mime_type, to: :file_set
+    delegate :uri, to: :file_set
 
     TARGET_EXT = nil
 
@@ -47,27 +47,21 @@ module NewspaperWorks
     end
 
     def identify
-      if @source_meta.nil?
-        path = @source_path
-        cmd = "identify #{path}"
-        # fallback to graphicsmagick if source is jp2, as Ubuntu 16.10
-        #   ImageMagick has no jp2 support.
-        cmd = 'gm ' + cmd if path.ends_with?('jp2')
-        Open3.popen3(cmd) do |_stdin, stdout, _stderr, _wait_thr|
-          @source_meta = stdout.read
-        end
-      end
-      @source_meta
+      return @source_meta unless @source_meta.nil?
+      @source_meta = NewspaperWorks::ImageTool.new(@source_path).metadata
+    end
+
+    def mime_type
+      identify[:content_type]
     end
 
     def use_color?
-      # imagemagick `identify` output describes color space:
-      !(identify.include?('Gray') || one_bit?)
+      identify[:color] == 'color'
     end
 
     # is source one-bit monochrome?
     def one_bit?
-      identify.include?('1-bit')
+      identify[:color] == 'monochrome'
     end
 
     def create_derivatives(filename)
@@ -86,6 +80,34 @@ module NewspaperWorks
       end
     end
 
+    def jp2_to_intermediate
+      intermediate_path = File.join(Dir.mktmpdir, 'intermediate.tif')
+      jp2_cmd = "opj_decompress -i #{@source_path} -o #{intermediate_path}"
+      # make intermediate, then...
+      `#{jp2_cmd}`
+      intermediate_path
+    end
+
+    def convert_cmd
+      raise NotImplementedError, 'Calling subclass missing convert_cmd method'
+    end
+
+    # convert non-JP2 source/primary file to PDF derivative with ImageMagick6
+    #   calls convert_cmd on calling subclasses
+    def im_convert
+      `#{convert_cmd}`
+    end
+
+    # convert JP2 source/primary file to PDF derivative, via
+    #   opj_decompress to intermediate TIFF, then ImageMagick6 convert
+    def jp2_convert
+      # jp2 source -> intermediate
+      intermediate_path = jp2_to_intermediate
+      @source_path = intermediate_path
+      # intermediate -> PDF
+      im_convert
+    end
+
     # def cleanup_derivatives; end
   end
 end

data/app/services/newspaper_works/pdf_derivative_service.rb

@@ -16,9 +16,6 @@ module NewspaperWorks
                     '-depth 8 ' \
                     '-compress jpeg %<out_file>s'.freeze
 
-    # graphicsmagick prefix, may be needed for jp2 source on Ubuntu
-    GM_PREFX = 'gm '.freeze
-
     def initialize(file_set)
       super(file_set)
     end
@@ -27,8 +24,7 @@ module NewspaperWorks
     #   JP2 source, and whether we have color or grayscale material.
     def convert_cmd
       template = use_color? ? COLOR_PDF_CMD : GRAY_PDF_CMD
-      cmd = format(template, source_file: @source_path, out_file: @dest_path)
-      @source_path.ends_with?('jp2') ? GM_PREFIX + cmd : cmd
+      format(template, source_file: @source_path, out_file: @dest_path)
     end
 
     def create_derivatives(filename)
@@ -38,8 +34,9 @@ module NewspaperWorks
       # no creation if pdf master
       return if mime_type == 'application/pdf'
 
-      # Get and run imagemagick or graphicsmagick command
-      `#{convert_cmd}`
+      # Get and run conversion command
+      return jp2_convert if mime_type == 'image/jp2'
+      im_convert
     end
   end
 end

data/app/services/newspaper_works/tiff_derivative_service.rb

@@ -21,9 +21,6 @@ module NewspaperWorks
                 '-depth 24 ' \
                 '-compress lzw %<out_file>s'.freeze
 
-    # graphicsmagick prefix, may be needed for jp2 source on Ubuntu
-    GM_PREFX = 'gm '.freeze
-
     def initialize(file_set)
       super(file_set)
     end
@@ -35,20 +32,19 @@ module NewspaperWorks
       source_path += '[0]' if @source_path.ends_with?('pdf')
       template = use_color? ? COLOR_CMD : GRAY_CMD
       template = MONO_CMD if one_bit?
-      cmd = format(template, source_file: source_path, out_file: @dest_path)
-      # normalization of command based on source
-      @source_path.ends_with?('jp2') ? GM_PREFIX + cmd : cmd
+      format(template, source_file: source_path, out_file: @dest_path)
     end
 
     def create_derivatives(filename)
       # Base class takes care of loading @source_path, @dest_path
       super(filename)
 
-      # no creation if pdf master
+      # no creation of TIFF deriviative if primary is TIFF
       return if mime_type == 'image/tiff'
 
-      # Get and run imagemagick or graphicsmagick command
-      `#{convert_cmd}`
+      return jp2_convert if mime_type == 'image/jp2'
+      # Otherwise, get, run imagemagick command to convert
+      im_convert
     end
   end
 end

data/app/views/newspaper_works/base/_attribute_rows.html.erb

@@ -1,34 +1,82 @@
 <%# newspaper_core_metadata attributes %>
 <%= presenter.attribute_to_html(:date_modified, label: t('hyrax.base.show.last_modified'), html_dl: true) %>
-<%= presenter.attribute_to_html(:alternative_title, html_dl: true) %>
-<%= presenter.attribute_to_html(:genre, render_as: :faceted, html_dl: true) %>
+<%= presenter.attribute_to_html(:alternative_title,
+                                label: t('newspaper_works.attributes.alternative_title.label'),
+                                html_dl: true) %>
+<%= presenter.attribute_to_html(:genre,
+                                label: t('newspaper_works.attributes.genre.label'),
+                                render_as: :faceted, html_dl: true) %>
+<%= presenter.attribute_to_html(:publication_title,
+                                label: t('newspaper_works.attributes.publication_title.label'),
+                                render_as: :faceted, html_dl: true) %>
 <%= presenter.attribute_to_html(:place_of_publication_label,
-                                label: I18n.t('newspaper_works.attributes.place_of_publication.label'),
+                                label: t('newspaper_works.attributes.place_of_publication.label'),
+                                render_as: :faceted, html_dl: true) %>
+<%= presenter.attribute_to_html(:issn,
+                                label: t('newspaper_works.attributes.issn.label'),
+                                render_as: :faceted, html_dl: true) %>
+<%= presenter.attribute_to_html(:lccn,
+                                label: t('newspaper_works.attributes.lccn.label'),
+                                render_as: :faceted, html_dl: true) %>
+<%= presenter.attribute_to_html(:oclcnum,
+                                label: t('newspaper_works.attributes.oclcnum.label'),
+                                render_as: :faceted, html_dl: true) %>
+<%= presenter.attribute_to_html(:held_by,
+                                label: t('newspaper_works.attributes.held_by.label'),
                                 render_as: :faceted, html_dl: true) %>
-<%= presenter.attribute_to_html(:issn, render_as: :faceted, html_dl: true) %>
-<%= presenter.attribute_to_html(:lccn, render_as: :faceted, html_dl: true) %>
-<%= presenter.attribute_to_html(:oclcnum, render_as: :faceted, html_dl: true) %>
-<%= presenter.attribute_to_html(:held_by, render_as: :faceted, html_dl: true) %>
 <%# scanned_media_metadata attributes %>
-<%= presenter.attribute_to_html(:text_direction, html_dl: true) %>
-<%= presenter.attribute_to_html(:page_number, html_dl: true) %>
-<%= presenter.attribute_to_html(:section, html_dl: true) %>
+<%= presenter.attribute_to_html(:text_direction,
+                                label: t('newspaper_works.attributes.text_direction.label'),
+                                html_dl: true) %>
+<%= presenter.attribute_to_html(:page_number,
+                                label: t('newspaper_works.attributes.page_number.label'),
+                                html_dl: true) %>
+<%= presenter.attribute_to_html(:section,
+                                label: t('newspaper_works.attributes.section.label'),
+                                html_dl: true) %>
 <%# newspaper_article attributes %>
-<%= presenter.attribute_to_html(:author, render_as: :faceted, html_dl: true) %>
-<%= presenter.attribute_to_html(:photographer, render_as: :faceted, html_dl: true) %>
-<%= presenter.attribute_to_html(:volume, html_dl: true) %>
-<%= presenter.attribute_to_html(:issue_number, html_dl: true) %>
-<%= presenter.attribute_to_html(:geographic_coverage, render_as: :faceted, html_dl: true) %>
-<%= presenter.attribute_to_html(:extent, html_dl: true) %>
-<%= presenter.attribute_to_html(:publication_date, render_as: :date, html_dl: true) %>
+<%= presenter.attribute_to_html(:author,
+                                label: t('newspaper_works.attributes.author.label'),
+                                render_as: :faceted, html_dl: true) %>
+<%= presenter.attribute_to_html(:photographer,
+                                label: t('newspaper_works.attributes.photographer.label'),
+                                render_as: :faceted, html_dl: true) %>
+<%= presenter.attribute_to_html(:volume,
+                                label: t('newspaper_works.attributes.volume.label'),
+                                html_dl: true) %>
+<%= presenter.attribute_to_html(:issue_number,
+                                label: t('newspaper_works.attributes.issue_number.label'),
+                                html_dl: true) %>
+<%= presenter.attribute_to_html(:geographic_coverage,
+                                label: t('newspaper_works.attributes.geographic_coverage.label'),
+                                render_as: :faceted, html_dl: true) %>
+<%= presenter.attribute_to_html(:extent,
+                                label: t('newspaper_works.attributes.extent.label'),html_dl: true) %>
+<%= presenter.attribute_to_html(:publication_date,
+                                label: t('newspaper_works.attributes.publication_date.label'),
+                                render_as: :date, html_dl: true) %>
 <%# newspaper_title attributes %>
-<%= presenter.attribute_to_html(:edition_number, html_dl: true) %>
-<%= presenter.attribute_to_html(:edition_name, html_dl: true) %>
-<%= presenter.attribute_to_html(:frequency, html_dl: true) %>
-<%= presenter.attribute_to_html(:preceded_by, render_as: :faceted, html_dl: true) %>
-<%= presenter.attribute_to_html(:succeeded_by, render_as: :faceted, html_dl: true) %>
-<%= presenter.attribute_to_html(:publication_date_start, render_as: :date, html_dl: true) %>
-<%= presenter.attribute_to_html(:publication_date_end, render_as: :date, html_dl: true) %>
+<%= presenter.attribute_to_html(:edition_number,
+                                label: t('newspaper_works.attributes.edition_number.label'),
+                                html_dl: true) %>
+<%= presenter.attribute_to_html(:edition_name,
+                                label: t('newspaper_works.attributes.edition_name.label'),
+                                html_dl: true) %>
+<%= presenter.attribute_to_html(:frequency,
+                                label: t('newspaper_works.attributes.frequency.label'),
+                                html_dl: true) %>
+<%= presenter.attribute_to_html(:preceded_by,
+                                label: t('newspaper_works.attributes.preceded_by.label'),
+                                render_as: :faceted, html_dl: true) %>
+<%= presenter.attribute_to_html(:succeeded_by,
+                                label: t('newspaper_works.attributes.succeeded_by.label'),
+                                render_as: :faceted, html_dl: true) %>
+<%= presenter.attribute_to_html(:publication_date_start,
+                                label: t('newspaper_works.attributes.publication_date_start.label'),
+                                render_as: :date, html_dl: true) %>
+<%= presenter.attribute_to_html(:publication_date_end,
+                                label: t('newspaper_works.attributes.publication_date_end.label'),
+                                render_as: :date, html_dl: true) %>
 <%# hyrax %>
 <%= presenter.attribute_to_html(:creator, render_as: :faceted, html_dl: true) %>
 <%= presenter.attribute_to_html(:contributor, render_as: :faceted, html_dl: true) %>

data/config/locales/newspaper_article.de.yml

@@ -5,7 +5,7 @@ de:
     select_type:
       newspaper_article:
         name:              "Zeitungsartikel"
-        description:       "Zeitungsartikel Werke"
+        description:       "Ein Artikel aus einer Zeitungsausgabe"
   simple_form:
     labels:
       newspaper_article:

data/config/locales/newspaper_article.en.yml

@@ -5,7 +5,7 @@ en:
     select_type:
       newspaper_article:
         name:              "Newspaper Article"
-        description:       "Newspaper article works"
+        description:       "An article from a newspaper issue"
   simple_form:
     labels:
       newspaper_article:

data/config/locales/newspaper_article.es.yml

@@ -5,7 +5,7 @@ es:
     select_type:
       newspaper_article:
         name:              "Artículo de periódico"
-        description:       "Trabajos de artículos periodísticos"
+        description:       "Un artículo de un número de periódico."
   simple_form:
     labels:
       newspaper_article:

data/config/locales/newspaper_article.fr.yml

@@ -5,7 +5,7 @@ fr:
     select_type:
       newspaper_article:
         name:              "Article de journal"
-        description:       "Article de journal fonctionne"
+        description:       "Un article d'un numéro de journal"
   simple_form:
     labels:
       newspaper_article:

data/config/locales/newspaper_article.it.yml

@@ -5,7 +5,7 @@ it:
     select_type:
       newspaper_article:
         name:              "Articolo di giornale"
-        description:       "Articoli per articoli di giornale"
+        description:       "Un articolo tratto da un numero di giornale"
   simple_form:
     labels:
       newspaper_article:

data/config/locales/newspaper_article.pt-BR.yml

@@ -5,7 +5,7 @@ pt-BR:
     select_type:
       newspaper_article:
         name:              "Artigo de jornal"
-        description:       "Trabalhos de artigo de jornal"
+        description:       "Um artigo de uma edição de jornal"
   simple_form:
     labels:
       newspaper_article: