traject 0.0.1

data/lib/traject/marc_extractor.rb

@@ -0,0 +1,206 @@
+
+
+module Traject
+  # MarcExtractor is a class for extracting lists of strings from a MARC::Record,
+  # according to specifications. See #parse_string_spec for description of string
+  # string arguments used to specify extraction. See #initialize for options
+  # that can be set controlling extraction.
+  #
+  # Examples:
+  #
+  #    array_of_stuff = MarcExtractor.new(marc_record, "001:245abc:700a").extract
+  #    values         = MarcExtractor.new(marc_record, "040a", :seperator => nil).extract
+  #
+  class MarcExtractor
+    attr_accessor :options, :marc_record, :spec_hash
+
+
+    # Convenience method to construct a MarcExtractor object and
+    # run extract on it.
+    #
+    # First arg is a marc record.
+    #
+    # Second arg is either a string that will be given to parse_string_spec,
+    # OR a hash that's the return value of parse_string_spec.
+    #
+    # Third arg is an optional options hash that will be passed as
+    # third arg of MarcExtractor constructor.
+    def self.extract_by_spec(marc_record, specification, options = {})
+      (raise IllegalArgument, "first argument must not be nil") if marc_record.nil?
+
+      unless specification.kind_of? Hash
+        specification = self.parse_string_spec(specification)
+      end
+
+      Traject::MarcExtractor.new(marc_record, specification, options).extract
+    end
+
+    # Take a hash that's the output of #parse_string_spec, return
+    # an array of strings extracted from a marc record accordingly
+    #
+    # options:
+    #
+    # [:seperator]  default ' ' (space), what to use to seperate
+    #               subfield values when joining strings
+    #
+    # [:alternate_script] default :include, include linked 880s for tags
+    #                     that match spec. Also:
+    #                     * false => do not include.
+    #                     * :only => only include linked 880s, not original
+    def initialize(marc_record, spec_hash, options = {})
+      self.options = {
+        :seperator => ' ',
+        :alternate_script => :include
+      }.merge(options)
+
+      raise IllegalArgumentException("second arg to MarcExtractor.new must be a Hash specification object") unless spec_hash.kind_of? Hash
+
+      self.marc_record = marc_record
+      self.spec_hash = spec_hash
+    end
+
+    # Converts from a string marc spec like "245abc:700a" to a nested hash used internally
+    # to represent the specification.
+    #
+    # a String specification is a string of form:
+    #  {tag}{|indicators|}{subfields} seperated by colons
+    # tag is three chars (usually but not neccesarily numeric),
+    # indicators are optional two chars prefixed by hyphen,
+    # subfields are optional list of chars (alphanumeric)
+    #
+    # indicator spec must be two chars, but one can be * meaning "don't care".
+    # space to mean 'blank'
+    #
+    # "245|01|abc65:345abc:700|*5|:800"
+    #
+    # Or, for control (fixed) fields (ordinarily fields 001-010), you can include a byte slice specification,
+    # but can NOT include subfield or indicator specifications. Plus can use special tag "LDR" for
+    # the marc leader. (TODO)
+    #
+    #  "008[35-37]:LDR[5]"
+    #  => bytes 35-37 inclusive of field 008, and byte 5 of the marc leader.
+    #
+    # Returns a nested hash keyed by tags.
+    # { tag => {
+    #     :subfields => ['a', 'b', '2'] # actually, a SET. may be empty or nil
+    #     :indicators => ['1', '0'] # An array. may be empty or nil; duple, either one can be nil
+    #    }
+    #}
+    # For byte offsets, :bytes => 12 or :bytes => (7..10)
+    #
+    # * subfields and indicators can only be provided for marc data/variable fields
+    # * byte slice can only be provided for marc control fields (generally tags less than 010)
+    #
+    # See tests for more examples.
+    def self.parse_string_spec(spec_string)
+      hash = {}
+
+      spec_string.split(":").each do |part|
+        if (part =~ /\A([a-zA-Z0-9]{3})(\|([a-z0-9\ \*]{2})\|)?([a-z0-9]*)?\Z/)
+          # variable field
+          tag, indicators, subfields = $1, $3, $4
+
+          hash[tag] ||= {}
+
+          if subfields
+            subfields.each_char do |subfield|
+              hash[tag][:subfields] ||= Array.new
+              hash[tag][:subfields] << subfield
+            end
+          end
+          if indicators
+            hash[tag][:indicators] = [ (indicators[0] if indicators[0] != "*"), (indicators[1] if indicators[1] != "*") ]
+          end
+        elsif (part =~ /\A([a-zA-Z0-9]{3})(\[(\d+)(-(\d+))?\])\Z/) # "005[4-5]"
+          tag, byte1, byte2 = $1, $3, $5
+          hash[tag] ||= {}
+
+          if byte1 && byte2
+            hash[tag][:bytes] = ((byte1.to_i)..(byte2.to_i))
+          elsif byte1
+            hash[tag][:bytes] = byte1.to_i
+          end
+        else
+          raise ArgumentError.new("Unrecognized marc extract specification: #{part}")
+        end
+      end
+
+      return hash
+    end
+
+
+    # Returns array of strings, extracted values
+    def extract
+      results = []
+
+      self.each_matching_line do |field, spec|
+        if control_field?(field)
+          results << (spec[:bytes] ? field.value.byteslice(spec[:bytes]) : field.value)
+        else
+          results.concat collect_subfields(field, spec)
+        end
+      end
+
+      return results
+    end
+
+    # Yields a block for every line in source record that matches
+    # spec. First arg to block is MARC::Field (control or data), second
+    # is the hash specification that it matched on. May take account
+    # of options such as :alternate_script
+    def each_matching_line
+      self.marc_record.each do |field|
+        if (spec = spec_covering_field(field)) && matches_indicators(field, spec)
+          yield(field, spec)
+        end
+      end
+    end
+
+    # Pass in a marc data field and a hash spec, returns
+    # an ARRAY of one or more strings, subfields extracted
+    # and processed per spec. Takes account of options such
+    # as :seperator
+    def collect_subfields(field, spec)
+      subfields = field.subfields.collect do |subfield|
+        subfield.value if spec[:subfields].nil? || spec[:subfields].include?(subfield.code)
+      end.compact
+
+      return options[:seperator] ? [ subfields.join( options[:seperator]) ] : subfields
+    end
+
+    # Is there a spec covering extraction from this field?
+    # May return true on 880's matching other tags depending
+    # on value of :alternate_script
+    # if :alternate_script is :only, will return original spec when field is an 880.
+    # otherwise will always return nil for 880s, you have to handle :alternate_script :include
+    # elsewhere, to add in the 880 in the right order
+    def spec_covering_field(field)
+      #require 'pry'
+      #binding.pry if field.tag == "880"
+
+      if field.tag == "880" && options[:alternate_script] != false
+        # pull out the spec for corresponding original marc tag this 880 corresponds to
+        # Due to bug in jruby https://github.com/jruby/jruby/issues/886 , we need
+        # to do this weird encode gymnastics, which fixes it for mysterious reasons. 
+        orig_field = field["6"].encode(field["6"].encoding).byteslice(0,3)
+        field["6"] && self.spec_hash[  orig_field  ]
+      elsif options[:alternate_script] != :only
+        self.spec_hash[field.tag]
+      end
+    end
+
+    def control_field?(field)
+      # should the MARC gem have a more efficient way to do this,
+      # define #control_field? on both ControlField and DataField?
+      return field.kind_of? MARC::ControlField
+    end
+
+    # a marc field, and an individual spec hash, {:subfields => array, :indicators => array}
+    def matches_indicators(field, spec)
+      return true if spec[:indicators].nil?
+
+      return (spec[:indicators][0].nil? || spec[:indicators][0] == field.indicator1) &&
+        (spec[:indicators][1].nil? || spec[:indicators][1] == field.indicator2)
+    end
+  end
+end

data/lib/traject/marc_reader.rb

@@ -0,0 +1,61 @@
+require 'marc'
+
+# A Reader class that can be used with Traject::Indexer.reader, to read
+# MARC records. 
+#
+# Includes Enumerable for convenience. 
+#
+# Reads in Marc records using ruby marc. Depends on config variables to
+# determine what serialization type to expect, and other parameters controlling
+# de-serialization. 
+#
+# Settings:
+#   ["marc_source.type"]  serialization type. default 'binary'
+#                 * "binary". Actual marc. 
+#                 * "xml", MarcXML
+#                 * "json". (NOT YET IMPLEMENTED) The "marc-in-json" format, encoded as newline-seperated
+#                   json. A simplistic newline-seperated json, with no comments
+#                   allowed, and no unescpaed internal newlines allowed in the json
+#                   objects -- we just read line by line, and assume each line is a
+#                   marc-in-json. http://dilettantes.code4lib.org/blog/2010/09/a-proposal-to-serialize-marc-in-json/
+#   ["marc_source.xml_parser"] For XML type, which XML parser to tell Marc::Reader
+#                              to use. Anything recognized by Marc::Reader :parser
+#                              argument. By default, asks Marc::Reader to take
+#                              it's best guess as to highest performance available
+#                              installed option. 
+#
+#
+# Can NOT yet read Marc8, input is always assumed UTF8. 
+class Traject::MarcReader
+  include Enumerable
+
+  attr_reader :settings, :input_stream
+
+  @@best_xml_parser = MARC::XMLReader.best_available
+
+  def initialize(input_stream, settings)
+    @settings = settings
+    @input_stream = input_stream
+  end
+
+  # Creates proper kind of ruby MARC reader, depending
+  # on settings or guesses.
+  def internal_reader
+    unless defined? @internal_reader      
+      @internal_reader = 
+        case settings["marc_source.type"]
+        when "xml"
+          parser = settings["marc_source.xml_parser"] || @@best_xml_parser
+          MARC::XMLReader.new(self.input_stream, :parser=> parser)
+        else
+          MARC::Reader.new(self.input_stream)
+        end
+    end
+    return @internal_reader
+  end
+
+  def each(*args, &block)
+    self.internal_reader.each(*args, &block)
+  end
+
+end

data/lib/traject/qualified_const_get.rb

@@ -0,0 +1,30 @@
+# From http://redcorundum.blogspot.com/2006/05/kernelqualifiedconstget.html
+# Adapted into a module, rather than monkey patching it into Kernel
+#
+# Method to take a string constant name, including :: qualifications, and
+# look up the actual constant. Looks up relative to current file.
+# REspects leading ::. Etc.
+module Traject::QualifiedConstGet
+
+
+  def qualified_const_get(str)
+    path = str.to_s.split('::')
+    from_root = path[0].empty?
+    if from_root
+      from_root = []
+      path = path[1..-1]
+    else
+      start_ns = ((Class === self)||(Module === self)) ? self : self.class
+      from_root = start_ns.to_s.split('::')
+    end
+    until from_root.empty?
+      begin
+        return (from_root+path).inject(Object) { |ns,name| ns.const_get(name) }
+      rescue NameError
+        from_root.delete_at(-1)
+      end
+    end
+    path.inject(Object) { |ns,name| ns.const_get(name) }
+  end
+
+end

data/lib/traject/solrj_writer.rb

@@ -0,0 +1,120 @@
+require 'traject'
+require 'traject/qualified_const_get'
+
+#
+# Writes to a Solr using SolrJ, and the SolrJ HttpSolrServer.
+#  (sub-class later for the ConcurrentUpdate server?)
+#
+# settings:
+#   [solr.url] Your solr url (required)
+#   [solrj_writer.server_class_name]  Defaults to "HttpSolrServer". You can specify
+#                                   another Solr Server sub-class, but it has
+#                                   to take a one-arg url constructor. Maybe
+#                                   subclass this writer class and overwrite
+#                                   instantiate_solr_server! otherwise
+#   [solrj.jar_dir] Custom directory containing all of the SolrJ jars. All
+#                   jars in this dir will be loaded. Otherwise,
+#                   we load our own packaged solrj jars. This setting
+#                   can't really be used differently in the same app instance,
+#                   since jars are loaded globally.
+#   [solrj_writer.parser_class_name] A String name of a class in package
+#                                    org.apache.solr.client.solrj.impl,
+#                                    we'll instantiate one with a zero-arg
+#                                    constructor, and pass it as an arg to setParser on
+#                                    the SolrServer instance, if present.
+#                                    NOTE: For contacting a Solr 1.x server, with the
+#                                    recent version of SolrJ used by default, set to
+#                                    "XMLResponseParser"
+#   [solrj_writer.commit_on_close]  If true (or string 'true'), send a commit to solr
+#                                   at end of #process.
+class Traject::SolrJWriter
+  include Traject::QualifiedConstGet
+
+  attr_reader :settings
+
+  def initialize(argSettings)
+    @settings = argSettings
+    settings_check!(settings)
+
+    ensure_solrj_loaded!
+
+    solr_server # init
+  end
+
+  # Loads solrj if not already loaded. By loading all jars found
+  # in settings["solrj.jar_dir"]
+  def ensure_solrj_loaded!
+    unless defined?(HttpSolrServer) && defined?(SolrInputDocument)
+      require 'java'
+
+      tries = 0
+      begin
+        tries += 1
+        java_import org.apache.solr.client.solrj.impl.HttpSolrServer
+        java_import org.apache.solr.common.SolrInputDocument
+      rescue NameError  => e
+        # /Users/jrochkind/code/solrj-gem/lib"
+
+        included_jar_dir = File.expand_path("../../vendor/solrj/lib", File.dirname(__FILE__))
+
+        jardir = settings["solrj.jar_dir"] || included_jar_dir
+        Dir.glob("#{jardir}/*.jar") do |x|
+          require x
+        end
+        if tries > 1
+          raise LoadError.new("Can not find SolrJ java classes")
+        else
+          retry
+        end
+      end
+    end
+  end
+
+  def put(hash)
+    doc = SolrInputDocument.new
+
+    hash.each_pair do |key, value_array|
+      value_array.each do |value|
+        doc.addField( key, value )
+      end
+    end
+
+    # TODO: Buffer docs internally, add in arrays, one http
+    # transaction per array. Is what solrj wiki recommends.
+    solr_server.add(doc)
+  end
+
+  def close
+    solr_server.commit if settings["solrj_writer.commit_on_close"].to_s == "true"
+
+    solr_server.shutdown
+    @solr_server = nil
+  end
+
+
+  def solr_server
+    @solr_server ||= instantiate_solr_server!
+  end
+  attr_writer :solr_server # mainly for testing
+
+  # Instantiates a solr server of class settings["solrj_writer.server_class_name"] or "HttpSolrServer"
+  # and initializes it with settings["solr.url"]
+  def instantiate_solr_server!
+    server_class  = qualified_const_get( settings["solrj_writer.server_class_name"] || "HttpSolrServer" )
+    server        = server_class.new( settings["solr.url"].to_s );
+
+    if parser_name = settings["solrj_writer.parser_class_name"]
+      parser = org.apache.solr.client.solrj.impl.const_get(parser_name).new
+      server.setParser( parser )
+    end
+
+    server
+  end
+
+  def settings_check!(settings)
+    unless settings.has_key?("solr.url") && ! settings["solr.url"].nil?
+      raise ArgumentError.new("SolrJWriter requires a 'solr.url' solr url in settings")
+    end
+  end
+
+end

data/lib/traject/translation_map.rb

@@ -0,0 +1,184 @@
+require 'traject'
+
+require 'yaml'
+
+
+module Traject
+  # A TranslationMap is basically just something that has a hash-like #[]
+  # method to map from input strings to output strings:
+  #
+  #   translation_map["some_input"] #=> some_output
+  #
+  # Input is assumed to always be string, output is either string
+  # or array of strings.
+  #
+  # What makes it more useful than a stunted hash is it's ability to load
+  # the hash definitions from configuration files, either pure ruby or
+  # yaml.
+  #
+  # TranslationMap.new("dir/some_file")
+  #
+  # Will look through the entire ruby $LOAD_PATH, for a translation_maps subdir
+  # that contains either some_file.rb OR  some_file.yaml 
+  # * Looks for "/translation_maps" subdir in load paths, so
+  #   for instance you can have a gem that keeps translation maps
+  #   in ./lib/translation_maps, and it Just Works. 
+  # * Note you do NOT supply the ".rb" or ".yaml" suffix yourself,
+  # it'll use whichever it finds (allows calling code to not care which is used).
+  #
+  # Ruby files just need to have their last line eval to a hash. They file
+  # will be run through `eval`, don't do it with untrusted content (naturally)
+  #
+  # You can also pass in a Hash for consistency to TranslationMap.new, although
+  # I don't know why you'd want to.
+  #
+  # == Special default handling
+  #
+  # The key "__default__" in the hash is treated specially. If set to a string,
+  # that string will be returned by the TranslationMap for any input not otherwise
+  # included. If set to the special string "__passthrough__", then for input not
+  # mapped, the original input string will be returned.
+  #
+  # This is most useful for YAML definition files, if you are using an actual ruby
+  # hash, you could just set the hash to do what you want using Hash#default_proc
+  # etc.
+  #
+  # Or, when calling TranslationMap.new(), you can pass in options over-riding special
+  # key too:
+  #
+  #  TranslationMap.new("something", :default => "foo")
+  #  TranslationMap.new("something", :default => :passthrough)
+  #
+  # == Output: String or array of strings
+  #
+  # The output can be a string or an array of strings, or nil.  It should not be anything
+  # When used with the #translate_array! method, one string can be replaced by multiple values
+  # (array of strings) or removed (nil)
+  #
+  # == Caching
+  # Lookup and loading of configuration files will be cached, for efficiency.
+  # You can reset with `TranslationMap.reset_cache!`
+  #
+  # == YAML example:
+  #
+  #     key: value
+  #     key2: value2 multiple words fine
+  #     key2b: "Although you can use quotes if you want: Or need."
+  #     key3:
+  #       - array
+  #       - of
+  #       - values look like this
+  class TranslationMap
+    class Cache
+      def initialize
+        @cached = Hash.new 
+      end
+
+      # Returns an actual Hash -- or nil if none found. 
+      def lookup(path)
+        unless @cached.has_key?(path)
+          @cached[path] = _lookup!(path)
+        end
+        return @cached[path]
+      end
+
+      # force lookup, without using cache.
+      # used by cache. Returns the actual hash. 
+      # Returns nil if none found. 
+      # May raise on syntax error in file being loaded. 
+      def _lookup!(path)
+        found = nil
+
+        $LOAD_PATH.each do |base|
+          rb_file = File.join( base,  "translation_maps",  "#{path}.rb"  )
+          yaml_file = File.join( base, "translation_maps", "#{path}.yaml"  )
+
+          if File.exists? rb_file
+            found = eval( File.open(rb_file).read , binding, rb_file )
+            break
+          elsif File.exists? yaml_file
+            found = YAML.load_file(yaml_file)
+          end
+        end
+
+        return found
+      end
+
+      def reset_cache!
+        @cached.clear
+      end
+
+    end
+
+    attr_reader :hash
+    attr_reader :default
+
+    class << self
+      attr_accessor :cache
+      def reset_cache!
+        cache.reset_cache!
+      end
+    end
+    self.cache = Cache.new
+
+
+    def initialize(defn, options = {})
+      if defn.kind_of? Hash
+        @hash = defn
+      else
+        @hash = self.class.cache.lookup(defn)
+        raise NotFound.new(defn) if @hash.nil?
+      end
+
+      if options[:default]
+        @default = options[:default]
+      elsif @hash.has_key? "__default__"
+        @default = @hash.delete("__default__")
+      end
+    end
+
+    def [](key)
+      if self.default && (! @hash.has_key?(key))
+        if self.default == "__passthrough__"
+          return key
+        else
+          return self.default
+        end
+      end
+
+      @hash[key]
+    end
+    alias_method :map, :[]
+
+    # Run every element of an array through this translation map,
+    # return the resulting array. If translation map returns nil,
+    # original element will be missing from output. 
+    #
+    # If an input maps to an array, each element of the array will be flattened
+    # into the output. 
+    #
+    # If an input maps to nil, it will cause the input element to be removed
+    # entirely. 
+    def translate_array(array)
+      array.each_with_object([]) do |input_element, output_array|
+        output_element = self.map(input_element)
+        if output_element.kind_of? Array
+          output_array.concat output_element
+        elsif ! output_element.nil?
+          output_array << output_element
+        end
+      end
+    end
+
+    def translate_array!(array)
+      array.replace( self.translate_array(array))
+    end
+
+    class NotFound < Exception
+      def initialize(path)
+        super("No translation map definition file found at '#{path}[.rb|.yaml]' in load path")
+      end
+    end
+
+  end
+end