traject 2.3.4 → 3.0.0.alpha.1

data/lib/traject/macros/nokogiri_macros.rb

@@ -0,0 +1,43 @@
+module Traject
+  module Macros
+    module NokogiriMacros
+
+      def default_namespaces
+        @default_namespaces ||= (settings["nokogiri.namespaces"] || {}).tap { |ns|
+          unless ns.kind_of?(Hash)
+            raise ArgumentError, "nokogiri.namespaces must be a hash, not: #{ns.inspect}"
+          end
+        }
+      end
+
+      def extract_xpath(xpath, ns: {}, to_text: true)
+        if ns && ns.length > 0
+          namespaces = default_namespaces.merge(ns)
+        else
+          namespaces = default_namespaces
+        end
+
+        lambda do |record, accumulator|
+          result = record.xpath(xpath, namespaces)
+
+          if to_text
+            # take all matches, for each match take all
+            # text content, join it together separated with spaces
+            # Make sure to avoid text content that was all blank, which is "between the children"
+            # whitespace.
+            result = result.collect do |n|
+              n.xpath('.//text()').collect(&:text).tap do |arr|
+                arr.reject! { |s| s =~ (/\A\s+\z/) }
+              end.join(" ")
+            end
+          else
+            # just put all matches in accumulator as Nokogiri::XML::Node's
+            result = result.to_a
+          end
+
+          accumulator.concat result
+        end
+      end
+    end
+  end
+end

data/lib/traject/macros/transformation.rb

@@ -0,0 +1,162 @@
+module Traject
+  module Macros
+    # Macros intended to be mixed into an Indexer and used in config
+    # as second or further args to #to_field, to transform existing accumulator values.
+    #
+    # They have the same form as any proc/block passed to #to_field, but
+    # operate on an existing accumulator, intended to be used as non-first-step
+    # transformations.
+    #
+    # Some of these are extracted from extract_marc options, so they can be
+    # used with any first-step extract methods.  Some informed by current users.
+    module Transformation
+
+      # Maps all values on accumulator through a  Traject::TranslationMap.
+      #
+      # A Traject::TranslationMap is hash-like mapping from input to output, usually
+      # defined in a yaml or dot-properties file, which can be looked up in load path
+      # with a file name as arg. See [Traject::TranslationMap](../translation_map.rb)
+      # header coments for details.
+      #
+      # Using this macro, you can pass in one TranslationMap initializer arg, but you can
+      # also pass in multiple, and they will be merged into each other (last one last), so
+      # you can use this to apply over-rides: Either from another on-disk map, or even from
+      # an inline hash (since a Hash is a valid TranslationMap initialization arg too).
+      #
+      # @example
+      #     to_field("something"), to_field "cataloging_agency", extract_marc("040a"), translation_map("marc_040a")
+      #
+      # @example with override
+      #     to_field("something"), to_field "cataloging_agency", extract_marc("040a"), translation_map("marc_040a", "local_marc_040a")
+      #
+      # @example with multiple overrides, including local hash
+      #     to_field("something"), to_field "cataloging_agency", extract_marc("040a"), translation_map("marc_040a", "local_marc_040a", {"DLC" => "U.S. LoC"})
+      def translation_map(*translation_map_specifier)
+        translation_map = translation_map_specifier.
+          collect { |spec| Traject::TranslationMap.new(spec) }.
+          reduce(:merge)
+
+        lambda do |rec, acc|
+          translation_map.translate_array! acc
+        end
+      end
+
+      # Pass in a proc/lambda arg or a block (or both), that will be called on each
+      # value already in the accumulator, to transform it. (Ie, with `#map!`/`#collect!` on your proc(s)).
+      #
+      # Due to how ruby syntax precedence works, the block form is probably not too useful
+      # in traject config files, except with the `&:` trick.
+      #
+      # The "stabby lambda" may be convenient for passing an explicit proc argument.
+      #
+      # You can pass both an explicit proc arg and a block, in which case the proc arg
+      # will be applied first.
+      #
+      # @example
+      #    to_field("something"), extract_marc("something"), transform(&:upcase)
+      #
+      # @example
+      #    to_field("something"), extract_marc("something"), transform(->(val) { val.tr('^a-z', "\uFFFD") })
+      def transform(a_proc=nil, &block)
+        unless a_proc || block
+          raise ArgumentError, "Needs a transform proc arg or block arg"
+        end
+
+        transformer_callable = if a_proc && block
+          # need to make a combo wrapper.
+          ->(val) { block.call(a_proc.call(val)) }
+        elsif a_proc
+          a_proc
+        else
+          block
+        end
+
+        lambda do |rec, acc|
+          acc.collect! do |value|
+            transformer_callable.call(value)
+          end
+        end
+      end
+
+      # Adds a literal to accumulator if accumulator was empty
+      #
+      # @example
+      #      to_field "title", extract_marc("245abc"), default("Unknown Title")
+      def default(default_value)
+        lambda do |rec, acc|
+          if acc.empty?
+            acc << default_value
+          end
+        end
+      end
+
+      # Removes all but the first value from accumulator, if more values were present.
+      #
+      # @example
+      #     to_field "main_author", extract_marc("100"), first_only
+      def first_only
+        lambda do |rec, acc|
+          # kind of esoteric, but slice used this way does mutating first, yep
+          acc.slice!(1, acc.length)
+        end
+      end
+
+
+      # calls ruby `uniq!` on accumulator, removes any duplicate values
+      #
+      # @example
+      #     to_field "something", extract_marc("245:240"), unique
+      def unique
+        lambda do |rec, acc|
+          acc.uniq!
+        end
+      end
+
+
+      # For each value in accumulator, remove all leading or trailing whitespace
+      # (unique aware). Like ruby #strip, but whitespace-aware
+      #
+      # @example
+      #     to_field "title", extract_marc("245"), strip
+      def strip
+        lambda do |rec, acc|
+          acc.collect! do |v|
+            # unicode whitespace class aware
+            v.sub(/\A[[:space:]]+/,'').sub(/[[:space:]]+\Z/, '')
+          end
+        end
+      end
+
+      # Run ruby `split` on each value in the accumulator, with separator
+      # given, flatten all results into single array as accumulator.
+      # Will generally result in more individual values in accumulator as output than were
+      # there in input, as input values are split up into multiple values.
+      def split(separator)
+        lambda do |rec, acc|
+          acc.replace( acc.flat_map { |v| v.split(separator) } )
+        end
+      end
+
+      # Append argument to end of each value in accumulator.
+      def append(suffix)
+        lambda do |rec, acc|
+          acc.collect! { |v| v + suffix }
+        end
+      end
+
+      # prepend argument to beginning of each value in accumulator.
+      def prepend(prefix)
+        lambda do |rec, acc|
+          acc.collect! { |v| prefix + v }
+        end
+      end
+
+      # Run ruby `gsub` on each value in accumulator, with pattern and replace value given.
+      def gsub(pattern, replace)
+        lambda do |rec, acc|
+          acc.collect! { |v| v.gsub(pattern, replace) }
+        end
+      end
+    end
+  end
+end

data/lib/traject/marc_extractor.rb

@@ -151,6 +151,8 @@ module Traject
       if options[:alternate_script] != false
         @fetch_alternate_script = true
         show_interest_in_tag(ALTERNATE_SCRIPT_TAG)
+      else
+        @fetch_alternate_script = false
       end
 
       @interesting_tags_list = @interesting_tags_hash.keys

data/lib/traject/ndj_reader.rb

@@ -12,7 +12,7 @@ class Traject::NDJReader
   def initialize(input_stream, settings)
     @settings = settings
     @input_stream = input_stream
-    if /\.gz\Z/.match(@settings['command_line.filename'])
+    if input_stream.respond_to?(:path) && /\.gz\Z/.match(input_stream.path)
       @input_stream = Zlib::GzipReader.new(@input_stream, :external_encoding => "UTF-8")
     end
   end

data/lib/traject/nokogiri_reader.rb

@@ -0,0 +1,179 @@
+module Traject
+  # A Trajet reader which reads XML, and yields zero to many Nokogiri::XML::Document
+  # objects as source records in the traject pipeline.
+  #
+  # It does process the entire input document with Nokogiri::XML.parse, DOM-parsing,
+  # so will take RAM for the entire input document, until iteration completes.
+  # (There is a separate half-finished `ExperimentalStreamingNokogiriReader` available, but it is
+  # experimental, half-finished, may disappear or change in backwards compat at any time, problematic,
+  # not recommended for production use, etc.)
+  #
+  # You can have it yield the _entire_ input XML as a single traject source record
+  # (default), or you can use setting `nokogiri.each_record_xpath` to split
+  # the source up into separate records to yield into traject pipeline -- each one
+  # will be it's own Nokogiri::XML::Document.
+  #
+  # ## Settings
+  # * nokogiri.default_namespaces: Set namespace prefixes that can be used in
+  #   other settings, including `extract_xpath` from NokogiriMacros.
+  # * nokogiri.each_record_xpath: if set to a string xpath, will take all matching nodes
+  #   from the input doc, and yield the individually as source records to the pipeline.
+  #   If you need to use namespaces here, you need to have them registered with
+  #   `nokogiri.default_namespaces`. If your source docs use namespaces, you DO need
+  #   to use them in your each_record_xpath.
+  # * nokogiri_reader.extra_xpath_hooks: Experimental in progress, see below.
+  #
+  # ## nokogiri_reader.extra_xpath_hooks: For handling nodes outside of your each_record_xpath
+  #
+  # What if you want to use each_record_xpath to yield certain nodes as source documents, but
+  # there is additional some other info in other parts of the input document you need? This came up
+  # when developing the OaiPmhNokogiriReader, which yields "//oai:record" as pipeline source documents,
+  # but also needed to look at "//oai:resumptionToken" to scrape the entire results.
+  #
+  # There is a semi-finished/in-progress feature that meets that use case -- unclear if it will meet
+  # all use cases for this general issue.
+  #
+  # Setting `nokogiri_reader.extra_xpath_hooks` can be set to a Hash where the keys are xpaths (if using
+  # namespaces must be must be registered with `nokogiri.default_namespaces`), and the value is a lambda/
+  # proc/callable object, taking two arguments.
+  #
+  #     provide "nokogiri_reader.extra_xpath_hooks", {
+  #       "//oai:resumptionToken" =>
+  #         lambda do |node, clipboard|
+  #           clipboard[:resumption_token] = node.text
+  #         end"
+  #     }
+  #
+  # The first arg is the matching node. What's this clipboard? Well, what are you
+  # gonna _do_ with what you get out of there, that you can do in a thread-safe way
+  # in the middle of nokogiri processing? The second arg is a thread-safe Hash "clipboard"
+  # that you can store things in, and later access via reader.clipboard.
+  #
+  # There's no great thread-safe way to get reader.clipboard in a normal nokogiri pipeline though,
+  # (the reader can change in multi-file handling so there can be a race condition if you try naively,
+  # don't!) Which is why this feature needs some work for general applicability. The OaiPmhReader
+  # manually creates it's readers outside the usual nokogiri flow, so can use it.
+  class NokogiriReader
+    include Enumerable
+
+    attr_reader :settings, :input_stream, :clipboard, :path_tracker
+
+    def initialize(input_stream, settings)
+      @settings = Traject::Indexer::Settings.new settings
+      @input_stream = input_stream
+      @clipboard = Traject::Util.is_jruby? ? Concurrent::Map.new : Concurrent::Hash.new
+
+      default_namespaces # trigger validation
+      validate_xpath(each_record_xpath, key_name: "each_record_xpath") if each_record_xpath
+      extra_xpath_hooks.each_pair do |xpath, _callable|
+        validate_xpath(xpath, key_name: "extra_xpath_hooks")
+      end
+    end
+
+    def each_record_xpath
+      @each_record_xpath ||= settings["nokogiri.each_record_xpath"]
+    end
+
+    def extra_xpath_hooks
+      @extra_xpath_hooks ||= settings["nokogiri_reader.extra_xpath_hooks"] || {}
+    end
+
+    def default_namespaces
+      @default_namespaces ||= (settings["nokogiri.namespaces"] || {}).tap { |ns|
+        unless ns.kind_of?(Hash)
+          raise ArgumentError, "nokogiri.namespaces must be a hash, not: #{ns.inspect}"
+        end
+      }
+    end
+
+    def each
+      whole_input_doc = Nokogiri::XML.parse(input_stream)
+
+      if each_record_xpath
+        whole_input_doc.xpath(each_record_xpath, default_namespaces).each do |matching_node|
+          # We want to take the matching node, and make it into root in a new Nokogiri document.
+          # This is tricky to do as performant as possible (we want to re-use the existing libxml node),
+          # while preserving namespaces properly (especially in jruby). Some uses of noko api that seem
+          # like they should work don't, esp in jruby.
+          child_doc = Nokogiri::XML::Document.new
+
+          reparent_node_to_root(child_doc, matching_node)
+
+          yield child_doc
+
+          child_doc = nil # hopefully make things easier on the GC.
+        end
+      else
+        # caller wants whole doc as a traject source record
+        yield whole_input_doc
+      end
+
+      run_extra_xpath_hooks(whole_input_doc)
+
+    ensure
+      # hopefully make things easier on the GC.
+      whole_input_doc = nil
+    end
+
+    private
+
+
+    # In MRI Nokogiri, this is as simple as `new_parent_doc.root = node`
+    # It seemed maybe safer to dup the node as well as remove the original from the original doc,
+    # but I believe this will result in double memory usage, as unlinked nodes aren't GC'd until
+    # their doc is.  I am hoping this pattern results in less memory usage.
+    # https://github.com/sparklemotion/nokogiri/issues/1703
+    #
+    # However, in JRuby it's a different story, JRuby doesn't properly preserve namespaces
+    # when re-parenting a node.
+    # https://github.com/sparklemotion/nokogiri/issues/1774
+    #
+    # The nodes within the tree re-parented _know_ they are in the correct namespaces,
+    # and xpath queries require that namespace, but the appropriate xmlns attributes
+    # aren't included in the serialized XML. This JRuby-specific code seems to get
+    # things back to a consistent state.
+    def reparent_node_to_root(new_parent_doc, node)
+      if Traject::Util.is_jruby?
+        original_ns_scopes = node.namespace_scopes
+      end
+
+      new_parent_doc.root = node
+
+      if Traject::Util.is_jruby?
+        original_ns_scopes.each do |ns|
+          if new_parent_doc.at_xpath("//#{ns.prefix}:*", ns.prefix => ns.href)
+            new_parent_doc.root.add_namespace(ns.prefix, ns.href)
+          end
+        end
+      end
+
+      return new_parent_doc
+    end
+
+    def validate_xpath(xpath, key_name:)
+      components = each_record_xpath.split('/')
+      components.each do |component|
+        prefix, element = component.split(':')
+        unless element
+          # there was no namespace
+          prefix, element = nil, prefix
+        end
+
+        if prefix
+          ns_uri = default_namespaces[prefix]
+          if ns_uri.nil?
+            raise ArgumentError, "#{key_name}: Can't find namespace prefix '#{prefix}' in '#{each_record_xpath}'. To use a namespace in each_record_xpath, it has to be registered with nokogiri.namespaces: #{default_namespaces.inspect}"
+          end
+        end
+      end
+    end
+
+    def run_extra_xpath_hooks(noko_doc)
+      extra_xpath_hooks.each_pair do |xpath, callable|
+        noko_doc.xpath(xpath, default_namespaces).each do |matching_node|
+          callable.call(matching_node, clipboard)
+        end
+      end
+    end
+  end
+end

data/lib/traject/oai_pmh_nokogiri_reader.rb

@@ -0,0 +1,159 @@
+require 'uri'
+require 'cgi'
+require 'http'
+
+module Traject
+  # Reads an OAI feed via HTTP and feeds it directly to a traject pipeline. You don't HAVE to use
+  # this to read oai-pmh, you might choose to fetch and store OAI-PMH responses to disk yourself,
+  # and then process as ordinary XML.
+  #
+  # Example command line:
+  #
+  #         traject -i xml -r Traject::OaiPmhNokogiriReader -s oai_pmh.start_url="http://example.com/oai?verb=ListRecords&metadataPrefix=oai_dc" -c your_config.rb
+  #
+  # ## Settings
+  #
+  # * oai_pmh.start_url: Required, eg "http://example.com/oai?verb=ListRecords&metadataPrefix=oai_dc"
+  # * oai_pmh.timeout: (default 10) timeout for http.rb in seconds
+  # * oai_pmh.try_gzip: (default true). Ask server for gzip response if available
+  # * oai_pmh.http_persistent: (default true). Use persistent HTTP connections.
+  #
+  # ## JRUBY NOTES:
+  #  * Does not work with jruby 9.2 until http.rb does: https://github.com/httprb/http/issues/475
+  #  * JRuby version def reads whole http response into memory before parsing; MRI version might do this too, but might not?
+  #
+  # ## TO DO
+  #
+  # This would be a lot more useful with some sort of built-in HTTP caching.
+  class OaiPmhNokogiriReader
+    include Enumerable
+
+    attr_reader :settings, :input_stream
+
+    def initialize(input_stream, settings)
+      namespaces = (settings["nokogiri.namespaces"] || {}).merge(
+        "oai" => "http://www.openarchives.org/OAI/2.0/"
+      )
+
+
+      @settings = Traject::Indexer::Settings.new(
+          "nokogiri_reader.extra_xpath_hooks" => extra_xpath_hooks,
+          "nokogiri.each_record_xpath" => "/oai:OAI-PMH/oai:ListRecords/oai:record",
+          "nokogiri.namespaces" => namespaces
+        ).with_defaults(
+          "oai_pmh.timeout" => 10,
+          "oai_pmh.try_gzip" => true,
+          "oai_pmh.http_persistent" => true
+        ).fill_in_defaults!.merge(settings)
+
+      @input_stream = input_stream
+    end
+
+    def start_url
+      settings["oai_pmh.start_url"] or raise ArgumentError.new("#{self.class.name} needs a setting 'oai_pmh.start_url'")
+    end
+
+    def start_url_verb
+      @start_url_verb ||= (array = CGI.parse(URI.parse(start_url).query)["verb"]) && array.first
+    end
+
+    def extra_xpath_hooks
+      @extra_xpath_hooks ||= {
+        "//oai:resumptionToken" =>
+          lambda do |doc, clipboard|
+            token = doc.text
+            if token && token != ""
+              clipboard[:resumption_token] = token
+            end
+          end
+      }
+    end
+
+    def each
+      url = start_url
+
+      resumption_token = nil
+      last_resumption_token = nil
+      pages_fetched = 0
+
+      until url == nil
+        resumption_token = read_and_parse_response(url) do |record|
+          yield record
+        end
+        url = resumption_url(resumption_token)
+        (last_resumption_token = resumption_token) if resumption_token
+        pages_fetched += 1
+      end
+
+      logger.info("#{self.class.name}: fetched #{pages_fetched} pages; last resumptionToken found: #{last_resumption_token.inspect}")
+    end
+
+    def resumption_url(resumption_token)
+      return nil if resumption_token.nil? || resumption_token == ""
+
+      # resumption URL is just original verb with resumption token, that seems to be
+      # the oai-pmh spec.
+      parsed_uri = URI.parse(start_url)
+      parsed_uri.query = "verb=#{CGI.escape start_url_verb}&resumptionToken=#{CGI.escape resumption_token}"
+      parsed_uri.to_s
+    end
+
+    def timeout
+      settings["oai_pmh.timeout"]
+    end
+
+    def logger
+      @logger ||= (@settings[:logger] || Yell.new(STDERR, :level => "gt.fatal")) # null logger)
+    end
+
+    private
+
+    # re-use an http-client for subsequent requests, to get http.rb's persistent connection re-use
+    # Note this means this is NOT thread safe, which is fine for now, but we'd have to do something
+    # different if we tried to multi-thread reading multiple files or something.
+    #
+    # @returns [HTTP::Client] from http.rb gem
+    def http_client
+      @http_client ||= begin
+        # timeout setting on http.rb seems to be a mess.
+        # https://github.com/httprb/http/issues/488
+        client = HTTP.timeout(:global, write: timeout / 3, connect: timeout / 3, read: timeout / 3)
+
+        if settings["oai_pmh.try_gzip"]
+          client = client.use(:auto_inflate).headers("accept-encoding" => "gzip;q=1.0, identity;q=0.5")
+        end
+
+        if settings["oai_pmh.http_persistent"]
+          parsed_uri = URI.parse(start_url)
+          client = client.persistent("#{parsed_uri.scheme}://#{parsed_uri.host}")
+        end
+
+        client
+      end
+    end
+
+    def read_and_parse_response(url)
+      http_response = http_client.get(url)
+
+      #File.write("our_oai/#{Time.now.to_i}.xml", body)
+
+      # Not sure why JRuby Nokogiri requires us to call #to_s on it first;
+      # not sure if this has perf implications. In either case, not sure
+      # if we are reading a separate copy of response into memory, or if Noko
+      # consumes it streaming. Trying to explicitly stream it to nokogiri, using
+      # http.rb#readpartial, just gave us a big headache.
+      noko_source_arg = if Traject::Util.is_jruby?
+        http_response.body.to_s
+      else
+        http_response.body
+      end
+
+      reader = Traject::NokogiriReader.new(noko_source_arg, settings)
+
+      reader.each { |d| yield d }
+
+      return reader.clipboard[:resumption_token]
+    end
+
+  end
+end