newspaper_works 0.1.0

data/lib/newspaper_works/ingest/lc_publication_info.rb

@@ -0,0 +1,144 @@
+require 'faraday'
+require 'nokogiri'
+require 'uri'
+
+module NewspaperWorks
+  module Ingest
+    class LCPublicationInfo < BasePublicationInfo
+      attr_accessor :place_of_publication, :full_title, :lccn, :place_name, :doc
+
+      XML_NS = {
+        mods: 'http://www.loc.gov/mods/v3',
+        MODS: 'http://www.loc.gov/mods/v3'
+      }.freeze
+
+      BASE_URL = 'https://lccn.loc.gov'.freeze
+
+      def initialize(lccn)
+        super(lccn)
+        @doc = nil
+        @full_title = nil
+        @place_of_publication = nil
+        @place_name = nil
+        load
+      end
+
+      def inspect
+        format(
+          "<#{self.class}:0x000000000%<oid>x " \
+            "\tlccn: '#{@lccn}'>",
+          oid: object_id << 1
+        )
+      end
+
+      def url
+        "#{BASE_URL}/#{@lccn}/mods"
+      end
+
+      def load_lc
+        resp = NewspaperWorks::ResourceFetcher.get url
+        @doc = Nokogiri.XML(resp['body'])
+        return if empty?
+        # try title[@type="uniform"] first:
+        title = find('//mods:titleInfo[@type="uniform"]/mods:title').first
+        # if no type="uniform" title, try non-alternate bare titleInfo:
+        #   -- in either case, should omit any non-sorted article (e.g. "The")
+        title = find('//mods:titleInfo[count(@type)=0]/mods:title').first if title.nil?
+        @full_title = title.text unless title.nil?
+      end
+
+      def mods_place_name
+        # prefer geographic subject hierarchy for place name construction:
+        city = find('//mods:hierarchicalGeographic/mods:city').first
+        # State (e.g. "Utah"), Province (e.g. "Ontario"), other (e.g. "England")
+        state = find('//mods:hierarchicalGeographic/mods:state').first
+        # if state is nil, fallback to country in its place
+        state = find('//mods:hierarchicalGeographic/mods:country').first if state.nil?
+        return "#{city.text}, #{state.text}" if city && state
+        # fallback to placeTerm text, which may be abbreviated in such a
+        #   way that geonames struggles to find on search; for a list of
+        #   abbreviations, see:
+        #   https://www.loc.gov/aba/publications/FreeSHM/H0810.pdf
+        name = find('//mods:originInfo//mods:placeTerm[@type="text"]').first
+        name.nil? ? nil : name.text
+      end
+
+      def load_place
+        @place_name = mods_place_name || place_name_from_title(@full_title)
+        return if @place_name.nil?
+        uri = NewspaperWorks::Ingest.geonames_place_uri(@place_name)
+        @place_of_publication = uri
+      end
+
+      def empty?
+        @doc.nil? || @doc.root.children.empty?
+      end
+
+      def load
+        load_lc
+        load_place unless @full_title.nil?
+      end
+
+      def title
+        return if empty?
+        NewspaperWorks::Ingest.normalize_title(@full_title.split(/ [\(]/)[0])
+      end
+
+      # ISO-639-2 three character language code, default is 'eng' (English)
+      def language(default = 'eng')
+        return if empty?
+        v = find('//mods:language/mods:languageTerm').first
+        v.nil? ? default : v.text
+      end
+
+      def issn
+        return if empty?
+        v = find('//mods:mods/mods:identifier[@type="issn"]').first
+        v.nil? ? nil : v.text
+      end
+
+      def oclcnum
+        return if empty?
+        v = find('//mods:mods/mods:identifier[@type="oclc"]').first
+        v.nil? ? nil : oclc_prefixed(v.text)
+      end
+
+      def preceded_by
+        related_by('preceding')
+      end
+
+      def succeeded_by
+        related_by('succeeding')
+      end
+
+      private
+
+        def related_by(key)
+          return if empty?
+          v = find("//mods:relatedItem[@type='#{key}']")
+          return nil if v.empty?
+          lccn = lccn_for(v[0])
+          return "#{BASE_URL}/#{lccn}" unless lccn.nil?
+          # No LCCN, ergo no URL, but a related item with a literal title?
+          titles = find('mods:title', v[0])
+          titles.empty? ? nil : titles[0].text
+        end
+
+        def lccn_for(related_item)
+          identifiers = find('mods:identifier[@type="local"]', related_item)
+          selected = identifiers.select { |v| v.text.start_with?('(DLC)') }
+          return if selected.size.zero?
+          selected.first.text.split(')')[-1].sub(' ', '')
+        end
+
+        def find(expr, context = nil)
+          context ||= @doc
+          return if context.nil? && empty?
+          context.xpath(
+            expr,
+            **XML_NS
+          )
+        end
+    end
+  end
+end

data/lib/newspaper_works/ingest/named_issue_metadata.rb

@@ -0,0 +1,60 @@
+require 'date'
+
+module NewspaperWorks
+  module Ingest
+    # Mixin for deducing issue metadata from path, publication info.
+    # precondition: consuming class has accessor for:
+    #   - `path`: full path to issue
+    #   - `publication`: a `NewspaperWorks::Ingest::PublicationInfo object.
+    module NamedIssueMetadata
+      # Memoized filename from path:
+      # @return [String]
+      def filename
+        return @filename unless @filename.nil?
+        @filename = File.basename(path)
+      end
+
+      def validate_path
+        # expect path to exist:
+        raise ArgumentError unless File.exist?(path)
+        # `YYYYMMDDEE` with valid date digits, optional `EE` edition
+        ptn = /^([0-9]{4})(1[012]|[0][1-9])(3[01]|[12][0-9]|0[1-9])([0-9]{2})?/
+        raise ArgumentError unless ptn.match(filename)
+      end
+
+      # Publication date stamp
+      # @return [String] ISO 8601 date stamp
+      def publication_date
+        year = filename.slice(0, 4).to_i
+        month = filename.slice(4, 2).to_i
+        day = filename.slice(6, 2).to_i
+        DateTime.new(year, month, day).iso8601[0..9]
+      end
+
+      # Issue edition number
+      # @return [Integer] number of issue edition
+      def edition_number
+        # use file name minus file extension (if applicable, e.g. PDF):
+        base = filename.split('.')[0..-2].join('.')
+        # default for PDF or issue dir not specifying edition value in
+        #   name (before file extension, if applicable):
+        return 1 if base.size < 10
+        # ...otherwise use explicitly provided edition number in filename
+        base.slice(8, 2).to_i
+      end
+
+      # rubocop:disable Rails/Delegate
+      def lccn
+        publication.lccn
+      end
+      # rubocop:enable Rails/Delegate
+
+      def title
+        title_date = DateTime.iso8601(publication_date).strftime('%B %-d, %Y')
+        v = "#{publication.title}: #{title_date}"
+        v = "#{v} (#{edition_number})" if edition_number.to_i > 1
+        [v]
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/ndnp/batch_ingester.rb

@@ -0,0 +1,64 @@
+require 'date'
+require 'find'
+require 'optparse'
+
+module NewspaperWorks
+  module Ingest
+    module NDNP
+      class BatchIngester
+        extend NewspaperWorks::Ingest::FromCommand
+        include NewspaperWorks::Logging
+
+        attr_accessor :path, :batch, :opts
+
+        # normalize path, possibly from directory, to contained batch
+        #   manifest XML path:
+        # @param path [String]
+        def self.normalize_path(path)
+          return path unless File.directory?(path)
+          batch_xml_path = Find.find(path).select do |f|
+            f.downcase.end_with?('batch_1.xml', 'batch.xml')
+          end
+          batch_xml_path.find { |f| f.end_with?('_1.xml') } || batch_xml_path[0]
+        end
+
+        # @param path [String] path to batch xml or directory
+        # @param opts [Hash]
+        #   global ingest options, to be passed to ingester components,
+        #   may include administrative metadata.
+        def initialize(path, opts = {})
+          @path = self.class.normalize_path(path)
+          raise IOError, "No batch file found: #{path}" if @path.empty?
+          @opts = opts
+          @batch = batch_enumerator
+          configure_logger('ingest')
+        end
+
+        def ingest
+          write_log("Beginning NDNP batch ingest for #{@path}")
+          batch.each do |issue|
+            issue_ingester(issue).ingest
+          end
+          write_log(
+            "NDNP batch ingest complete!"
+          )
+        end
+
+        private
+
+          # Return BatchIngest object as enumerable of issues:
+          def batch_enumerator
+            NewspaperWorks::Ingest::NDNP::BatchXMLIngest.new(path)
+          end
+
+          def issue_ingester(issue)
+            NewspaperWorks::Ingest::NDNP::IssueIngester.new(issue, @opts)
+          end
+
+          def normalize_date(v)
+            (v.is_a?(String) ? Date.parse(v) : v).to_s
+          end
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/ndnp/batch_xml_ingest.rb

@@ -0,0 +1,72 @@
+require 'nokogiri'
+
+module NewspaperWorks
+  module Ingest
+    module NDNP
+      class BatchXMLIngest
+        include Enumerable
+        include NewspaperWorks::Ingest::NDNP::NDNPMetsHelper
+
+        attr_accessor :container_paths, :issue_paths, :path
+
+        delegate :size, to: :issue_paths
+
+        def initialize(path)
+          @path = path
+          load_doc
+          @container_paths = xpath('//ndnp:batch//ndnp:reel').map do |e|
+            normalize_path(e.text)
+          end
+          @issue_paths = xpath('//ndnp:batch//ndnp:issue').map do |e|
+            normalize_path(e.text)
+          end
+        end
+
+        def name
+          xpath('//ndnp:batch').first.attributes['name'].value
+        end
+
+        def get(path)
+          return get_issue(path) if issue_paths.include?(path)
+          get_container(path)
+        end
+
+        def issues
+          issue_paths.map { |path| get(path) }
+        end
+
+        def containers
+          container_paths.map { |path| get(path) }
+        end
+
+        def each
+          @issue_paths.each do |path|
+            yield get_issue(path)
+          end
+        end
+
+        private
+
+          def get_issue(path)
+            NewspaperWorks::Ingest::NDNP::IssueIngest.new(path)
+          end
+
+          def get_container(path)
+            NewspaperWorks::Ingest::NDNP::ContainerIngest.new(path)
+          end
+
+          def xpath(expr)
+            ns = {
+              ndnp: 'http://www.loc.gov/ndnp',
+              NDNP: 'http://www.loc.gov/ndnp'
+            }
+            @doc.xpath(expr, **ns)
+          end
+
+          def load_doc
+            @doc = Nokogiri::XML(File.open(path))
+          end
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/ndnp/container_ingest.rb

@@ -0,0 +1,99 @@
+module NewspaperWorks
+  module Ingest
+    module NDNP
+      class ContainerIngest
+        # Enumerable of IssueIngest objects for issues in pages
+        include Enumerable
+        include NewspaperWorks::Ingest::NDNP::NDNPMetsHelper
+
+        attr_accessor :path, :doc, :dmdids, :issue_paths
+
+        def initialize(path)
+          @path = path
+          @doc = nil
+          @metadata = nil
+          # identifiers of control images, which we make accessible, but are
+          #   not the primary focus of enumeration:
+          @dmdids = nil
+          @issue_paths = []
+          load_doc
+        end
+
+        def inspect
+          format(
+            "<#{self.class}:0x000000000%<oid>x\n" \
+              "\tpath: '#{path}',\n",
+            oid: object_id << 1
+          )
+        end
+
+        def identifier
+          metadata.reel_number
+        end
+
+        # Return control image as PageIngest object.
+        #   These objects will not have pagination/sequence data, but
+        #   will provide an equivalent programmatic interface for file access
+        #   of control images, as one would access normal page files.
+        # @return [NewspaperWorks::Ingest::NDNP::PageIngest]
+        def page_by_dmdid(dmdid)
+          NewspaperWorks::Ingest::NDNP::PageIngest.new(@path, dmdid, self)
+        end
+
+        # Get IssueIngest object, given path to its XML
+        # return [NewspaperWorks::Ingest::NDNP::IssueIngest]
+        def issue_by_path(path)
+          NewspaperWorks::Ingest::NDNP::IssueIngest.new(path)
+        end
+
+        def each
+          @issue_paths.each do |path|
+            yield issue_by_path(path)
+          end
+        end
+
+        def size
+          @issue_paths.size
+        end
+
+        def metadata
+          return @metadata unless @metadata.nil?
+          @metadata = NewspaperWorks::Ingest::NDNP::ContainerMetadata.new(
+            path,
+            self
+          )
+        end
+
+        private
+
+          def load_doc
+            @doc = Nokogiri::XML(File.open(path)) if @doc.nil?
+            page_divs = doc.xpath(
+              "//mets:structMap/mets:div[@TYPE='np:reel']/" \
+                "mets:div[@TYPE='np:target']",
+              mets: 'http://www.loc.gov/METS/'
+            )
+            # identifiers for reel control images:
+            @dmdids = page_divs.map { |div| div.attr('DMDID') }
+            load_issue_paths
+          end
+
+          # Load instance attribute for issue paths,
+          #   based on listing of directory in which reel XML is present.
+          #   This is done without context of batch xml,
+          #   with file name expectations based on convention,
+          #   as expressed in NDNP technical guidelines,
+          #   which presume that the issue XML file name will (sans extension)
+          #   match directory name for the issue, in date+edition syntax.
+          def load_issue_paths
+            issue_dir_paths = Dir["#{File.dirname(path)}/*/"].select do |v|
+              !File.basename(v).match(/^[0-9]+$/).nil?
+            end
+            @issue_paths = issue_dir_paths.map do |path|
+              File.join(path, "#{File.basename(path)}.xml")
+            end
+          end
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/ndnp/container_ingester.rb

@@ -0,0 +1,84 @@
+module NewspaperWorks
+  module Ingest
+    module NDNP
+      # Ingester for reel/container, given reel source data
+      #   and required publication (NewspaperTitle) asset.
+      #   Responsibile for creating/finding container, linking
+      #   to (parent) publication and (child) pages.
+      class ContainerIngester
+        include NewspaperWorks::Ingest::NDNP::NDNPAssetHelper
+
+        attr_accessor :source, :target, :publication, :opts
+
+        # Create ingester in context of source reel data, NewspaperTitle
+        # @param source [NewspaperWorks::Ingest::NDNP::ContainerIngest]
+        # @param publication [NewspaperTitle] Required publication to link to
+        # @param opts [Hash]
+        #   ingest options, e.g. administrative metadata
+        def initialize(source, publication, opts = {})
+          @source = source
+          @publication = publication
+          @opts = opts
+          # initially nil, populate w/ NewspaperContainer object via .ingest
+          @target = nil
+        end
+
+        def ingest
+          find_or_create_container
+          link_publication
+        end
+
+        # Link a page to target container
+        # @param page [NewspaperPage]
+        def link(page)
+          @target.ordered_members << page
+          # save each link attempt (for now no deferring/bundling)
+          @target.save!
+        end
+
+        def find_or_create_container
+          @target = find_container
+          create_container if @target.nil?
+        end
+
+        private
+
+          def metadata
+            @source.metadata
+          end
+
+          def find_container
+            NewspaperContainer.where(identifier: metadata.reel_number).first
+          end
+
+          def create_container
+            @target = NewspaperContainer.create
+            copy_metadata
+            assign_administrative_metadata
+            @target.save!
+          end
+
+          def copy_metadata
+            reel_number = metadata.reel_number
+            @target.identifier = [reel_number]
+            @target.title = ["Microform reel (#{reel_number})"]
+            copy_fields = [
+              :held_by,
+              :publication_date_start,
+              :publication_date_end
+            ]
+            copy_fields.each do |fieldname|
+              value = metadata.send(fieldname.to_s)
+              @target.send("#{fieldname}=", value)
+            end
+          end
+
+          def link_publication
+            return unless @target.publication.nil?
+            @publication.members << @target
+            @publication.save!
+          end
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/ndnp/container_metadata.rb

@@ -0,0 +1,87 @@
+module NewspaperWorks
+  module Ingest
+    module NDNP
+      class ContainerMetadata
+        include NewspaperWorks::Ingest::NDNP::NDNPMetsHelper
+
+        attr_accessor :path, :doc
+
+        def initialize(path, parent = nil)
+          @path = path
+          @parent = parent
+          @doc = nil
+          load_doc
+        end
+
+        def inspect
+          format(
+            "<#{self.class}:0x000000000%<oid>x\n" \
+              "\tpath: '#{path}',\n",
+            oid: object_id << 1
+          )
+        end
+
+        # Reel Number (NDNP-mandatory)
+        # @return [String] a serial number string for reel, may correspond
+        #   to an issued barcode
+        def reel_number
+          v = xpath("//mods:identifier[@type='reel number']").first
+          return v.text unless v.nil?
+          xpath('//mets:mets/@LABEL').first.value
+        end
+
+        # Original Source Repository (NDNP-mandatory)
+        # @return [String]
+        def held_by
+          v = xpath("//mods:physicalLocation").first
+          return v['displayLabel'] unless v.nil?
+          # fallback to look at mods:note/@displayLabel, when the
+          #   @type="agencyResponsibleForReproduction"
+          xpath(
+            '//mods:note[@type="agencyResponsibleForReproduction"]' \
+            '/@displayLabel'
+          ).first.value
+        end
+
+        # Media genre/form (Page Physical Description, e.g. "microform")
+        #   NDNP Mandatory.
+        # @return [String]
+        def genre
+          form = xpath('//mods:physicalDescription/MODS:form').first
+          form.attributes['type'].value
+        end
+
+        # Titles (on Reel) (optional)
+        # @return [String] title
+        def title
+          techmd('ndnp:titles')
+        end
+
+        # Start Date (optional)
+        # @return [String] ISO 8601 formatted date
+        def publication_date_start
+          techmd('ndnp:startDate')
+        end
+
+        # End Date (optional)
+        # @return [String] ISO 8601 formatted date
+        def publication_date_end
+          techmd('ndnp:endDate')
+        end
+
+        private
+
+          def load_doc
+            @doc = @parent.doc unless @parent.nil?
+            @doc = Nokogiri::XML(File.open(path)) if @doc.nil?
+          end
+
+          def techmd(spec = nil)
+            base = xpath('//ndnp:reelTechMD')
+            return base if spec.nil?
+            base.xpath(spec).first.text
+          end
+      end
+    end
+  end
+end

data/lib/newspaper_works/ingest/ndnp/issue_ingest.rb

@@ -0,0 +1,81 @@
+module NewspaperWorks
+  module Ingest
+    module NDNP
+      class IssueIngest
+        include Enumerable
+        include NewspaperWorks::Ingest::NDNP::NDNPMetsHelper
+
+        attr_accessor :path, :doc, :dmdids
+
+        def initialize(path)
+          @path = path
+          @doc = nil
+          @metadata = nil
+          # Enumeration based on list of DMDID loaded by load_doc
+          @dmdids = nil
+          load_doc
+          # cache dmdid -> PageIngest
+          @page_cache = {}
+        end
+
+        def inspect
+          format(
+            "<#{self.class}:0x000000000%<oid>x\n" \
+              "\tpath: '#{path}',\n",
+            oid: object_id << 1
+          )
+        end
+
+        def identifier
+          metadata.lccn
+        end
+
+        def page_by_dmdid(dmdid)
+          return @page_cache[dmdid] if @page_cache.key?(dmdid)
+          p = NewspaperWorks::Ingest::NDNP::PageIngest.new(@path, dmdid, self)
+          @page_cache[dmdid] = p
+          p
+        end
+
+        def page_by_sequence_number(n)
+          page_by_dmdid(
+            doc.xpath(
+              "//mods:extent//mods:start[text()='#{n}']",
+              mets: 'http://www.loc.gov/METS/',
+              mods: 'http://www.loc.gov/mods/v3'
+            ).first.ancestors('dmdSec').first['ID']
+          )
+        end
+
+        def each
+          @dmdids.each do |dmdid|
+            yield page_by_dmdid(dmdid)
+          end
+        end
+
+        def size
+          @dmdids.size
+        end
+
+        def metadata
+          return @metadata unless @metadata.nil?
+          @metadata = NewspaperWorks::Ingest::NDNP::IssueMetadata.new(
+            path,
+            self
+          )
+        end
+
+        private
+
+          def load_doc
+            @doc = Nokogiri::XML(File.open(path)) if @doc.nil?
+            page_divs = doc.xpath(
+              "//mets:structMap//mets:div[@TYPE='np:page']",
+              mets: 'http://www.loc.gov/METS/'
+            )
+            @dmdids = page_divs.map { |div| div.attr('DMDID') }
+          end
+      end
+    end
+  end
+end