traject 2.1.0 → 2.2.0

data/lib/traject/marc_extractor_spec.rb

@@ -0,0 +1,229 @@
+# Represents a single specification for extracting data
+# from a marc field, like "600abc" or "600|1*|x".
+#
+# Includes the tag for reference, although this is redundant and not actually used
+# in logic, since the tag is also implicit in the overall spec_hash
+# with tag => [spec1, spec2]
+
+
+module Traject
+  class MarcExtractor
+
+    # A set of specs
+    class SpecSet
+
+      attr_accessor :hash
+
+      def self.new(seedset = {})
+
+        case seedset
+          when String
+            s      = allocate
+            s.hash = Spec.hash_from_string(seedset)
+            s
+          when Hash
+            s    = allocate
+            hash = Hash.new
+            seedset.each_pair do |k, v|
+              hash[k] = Array(v)
+            end
+            s.hash = hash
+            s
+          when SpecSet
+            seedset
+          else
+            raise ArgumentError.new, "SpecSet can only be constructed from a string, a hash, or another SpecSet"
+        end
+      end
+
+      def add(spec)
+        @hash[spec.tag] << spec
+      end
+
+      def tags
+        @hash.keys
+      end
+
+      def specs_for_tag(tag)
+        @hash[tag] || []
+      end
+
+      def specs_matching_field(field, use_alternate_script = false)
+
+        tag = if use_alternate_script
+                effective_tag(field)
+              else
+                field.tag
+              end
+        specs_for_tag(tag).select { |s| s.matches_indicators?(field) }
+      end
+
+      def effective_tag(field)
+        if field.tag == ALTERNATE_SCRIPT_TAG and field['6']
+          field["6"].encode(field["6"].encoding).byteslice(0, 3)
+        else
+          field.tag
+        end
+      end
+
+    end
+
+    class Spec
+      attr_accessor :tag, :subfields
+      attr_reader :indicator1, :indicator2, :byte1, :byte2, :bytes
+
+      # Allow use of a hash to initialize. Should ditch this and use
+      # optional keyword args once folks move to 2.x syntax
+      def initialize(hash = nil)
+        if hash
+          hash.each_pair do |key, value|
+            self.send("#{key}=", value)
+          end
+        end
+      end
+
+      #  Should subfields extracted by joined, if we have a seperator?
+      #  * '630' no subfields specified => join all subfields
+      #  * '630abc' multiple subfields specified = join all subfields
+      #  * '633a' one subfield => do not join, return one value for each $a in the field
+      #  * '633aa' one subfield, doubled => do join after all, will return a single string joining all the values of all the $a's.
+      #
+      # Last case is handled implicitly at the moment when subfields == ['a', 'a']
+      def joinable?
+        (self.subfields.nil? || self.subfields.size != 1)
+      end
+
+      def indicator1=(ind1)
+        ind1 == '*' ? @indicator1 = nil : @indicator1 = ind1
+      end
+
+      def indicator2=(ind2)
+        ind2 == '*' ? @indicator2 = nil : @indicator2 = ind2
+      end
+
+      def byte1=(byte1)
+        @byte1 = byte1.to_i if byte1
+        set_bytes(@byte1, @byte2)
+      end
+
+      def byte2=(byte2)
+        @byte2 = byte2.to_i if byte2
+        set_bytes(@byte1, @byte2)
+      end
+
+      def set_bytes(byte1, byte2)
+        if byte1 && byte2
+          @bytes = ((byte1.to_i)..(byte2.to_i))
+        elsif byte1
+          @bytes = byte1.to_i
+        end
+      end
+
+      # Pass in a MARC field, do it's indicators match indicators
+      # in this spec? nil indicators in spec mean we don't care, everything
+      # matches.
+      def matches_indicators?(field)
+        return (indicator1.nil? || indicator1 == field.indicator1) &&
+            (indicator2.nil? || indicator2 == field.indicator2)
+      end
+
+      # Pass in a string subfield code like 'a'; does this
+      # spec include it?
+      def includes_subfield_code?(code)
+        # subfields nil means include them all
+        self.subfields.nil? || self.subfields.include?(code)
+      end
+
+      # Simple equality definition
+      def ==(spec)
+        return false unless spec.kind_of?(Spec)
+
+        return (self.tag == spec.tag) &&
+            (self.subfields == spec.subfields) &&
+            (self.indicator1 == spec.indicator1) &&
+            (self.indicator2 == spec.indicator2) &&
+            (self.bytes == spec.bytes)
+      end
+
+
+      # Converts from a string marc spec like "008[35]:245abc:700a" to a hash used internally
+      # to represent the specification. See comments at head of class for
+      # documentation of string specification format.
+      #
+      #
+      # ## Return value
+      #
+      # The hash returned is keyed by tag, and has as values an array of 0 or
+      # or more MarcExtractor::Spec objects representing the specified extraction
+      # operations for that tag.
+      #
+      # It's an array of possibly more than one, because you can specify
+      # multiple extractions on the same tag: for instance "245a:245abc"
+      #
+      # See tests for more examples.
+
+      DATAFIELD_PATTERN    = /\A([a-zA-Z0-9]{3})(\|([a-z0-9\ \*])([a-z0-9\ \*])\|)?([a-z0-9]*)?\Z/
+      CONTROLFIELD_PATTERN = /\A([a-zA-Z0-9]{3})(\[(\d+)(-(\d+))?\])\Z/
+
+      def self.hash_from_string(spec_string)
+        # hash defaults to []
+        hash         = Hash.new
+
+        # Split the string(s) given on colon
+        spec_strings = spec_string.is_a?(Array) ? spec_string.map { |s| s.split(/\s*:\s*/) }.flatten : spec_string.split(/s*:\s*/)
+
+        spec_strings.each do |part|
+          if m = DATAFIELD_PATTERN.match(part)
+
+            tag, ind1, ind2, subfields = m[1], m[3], m[4], m[5]
+
+            spec = create_datafield_spec(tag, ind1, ind2, subfields)
+
+            hash[spec.tag] ||= []
+            hash[spec.tag] << spec
+
+          elsif m = CONTROLFIELD_PATTERN.match(part)
+            tag, byte1, byte2 = m[1], m[3], m[5]
+
+            spec = create_controlfield_spec(tag, byte1, byte2)
+
+            hash[spec.tag] ||= []
+            hash[spec.tag] << spec
+          else
+            raise ArgumentError.new("Unrecognized marc extract specification: #{part}")
+          end
+        end
+
+        return hash
+      end
+
+
+      # Create a new datafield spec. Most of the logic about how to deal
+      # with special characters is built into the Spec class.
+
+      def self.create_datafield_spec(tag, ind1, ind2, subfields)
+        spec            = Spec.new(:tag => tag)
+        spec.indicator1 = ind1
+        spec.indicator2 = ind2
+
+        if subfields and !subfields.empty?
+          spec.subfields = subfields.split('')
+        end
+
+        spec
+
+      end
+
+      # Create a new controlfield spec
+      def self.create_controlfield_spec(tag, byte1, byte2)
+        spec = Spec.new(:tag => tag)
+        spec.set_bytes(byte1, byte2)
+        spec
+      end
+
+
+    end
+  end
+
+end
+

data/lib/traject/version.rb

@@ -1,3 +1,3 @@
 module Traject
-  VERSION = "2.1.0"
+  VERSION = "2.2.0"
 end

data/test/debug_writer_test.rb

@@ -32,6 +32,47 @@ describe 'Simple output' do
 
   end
 
+  it "deals ok with a missing ID" do
+    context      = Traject::Indexer::Context.new(:output_hash => @indexer.map_record(@record))
+    logger_strio = StringIO.new
+    idfield      = 'id'
+
+    context.logger   = Logger.new(logger_strio)
+    context.position = 1
+
+    context.output_hash.delete(idfield)
+    @writer.put context
+    expected = [
+        "record_num_1 title #{@title}",
+    ]
+    assert_equal expected.join("\n").gsub(/\s/, ''), @io.string.gsub(/\s/, '')
+    assert_match /At least one record \(\#1\) doesn't define field 'id'/, logger_strio.string
+    @writer.close
+
+  end
+
+  it "sets the idfield correctly" do
+    bad_rec_id_field = 'iden'
+    writer           = Traject::DebugWriter.new("output_stream" => @io, "debug_writer.idfield" => bad_rec_id_field)
+
+    context = Traject::Indexer::Context.new(:output_hash => @indexer.map_record(@record))
+
+    logger_strio = StringIO.new
+
+    context.logger   = Logger.new(logger_strio)
+    context.position = 1
+
+    writer.put context
+    expected = [
+        "record_num_1 id #{@id }",
+        "record_num_1 title #{@title}",
+    ]
+    assert_equal expected.join("\n").gsub(/\s/, ''), @io.string.gsub(/\s/, '')
+    assert_match /At least one record \(\#1\) doesn't define field 'iden'/, logger_strio.string
+    writer.close
+
+  end
+
 end
 
 

data/test/marc_extractor_test.rb

@@ -9,14 +9,14 @@ describe "Traject::MarcExtractor" do
   it "is frozen read-only" do
     extractor = Traject::MarcExtractor.new("100abcde", :seperator => ";")
     assert extractor.frozen?
-    assert extractor.spec_hash.frozen?
+    assert extractor.spec_set.frozen?
     assert extractor.options.frozen?
   end
 
 
   describe "#parse_marc_spec" do
     it "parses single spec with all elements" do
-      parsed = Traject::MarcExtractor.parse_string_spec("245|1*|abcg")
+      parsed = Traject::MarcExtractor::Spec.hash_from_string("245|1*|abcg")
 
       assert_kind_of Hash, parsed
       assert_equal 1, parsed.keys.length
@@ -30,7 +30,7 @@ describe "Traject::MarcExtractor" do
     end
 
     it "parses a mixed bag" do
-      parsed = Traject::MarcExtractor.parse_string_spec("245abcde:810:700|*4|bcd")
+      parsed  = Traject::MarcExtractor::Spec.hash_from_string("245abcde:810:700|*4|bcd")
       spec245 = parsed['245'].first
       spec810 = parsed['810'].first
       spec700 = parsed['700'].first
@@ -57,14 +57,14 @@ describe "Traject::MarcExtractor" do
     end
 
     it "parses fixed field byte offsets" do
-      parsed = Traject::MarcExtractor.parse_string_spec("005[5]:008[7-10]")
+      parsed = Traject::MarcExtractor::Spec.hash_from_string("005[5]:008[7-10]")
 
       assert_equal 5, parsed["005"].first.bytes
       assert_equal 7..10, parsed["008"].first.bytes
     end
 
     it "allows arrays of specs" do
-      parsed = Traject::MarcExtractor.parse_string_spec %w(
+      parsed = Traject::MarcExtractor::Spec.hash_from_string %w(
         245abcde
         810
         700|*4|bcd
@@ -73,7 +73,7 @@ describe "Traject::MarcExtractor" do
     end
 
     it "allows mixture of array and colon-delimited specs" do
-      parsed = Traject::MarcExtractor.parse_string_spec %w(
+      parsed = Traject::MarcExtractor::Spec.hash_from_string %w(
         245abcde
         100:110:111
         810
@@ -127,13 +127,13 @@ describe "Traject::MarcExtractor" do
 
   describe "#extract_by_spec" do
     before do
-      @record = MARC::Reader.new(support_file_path  "manufacturing_consent.marc").to_a.first
+      @record = MARC::Reader.new(support_file_path "manufacturing_consent.marc").first
     end
 
     describe "extracts a basic case" do
       before do
-        parsed_spec = Traject::MarcExtractor.parse_string_spec("700abcdef:856|*2|:505|1*|:245ba")
-        @values = Traject::MarcExtractor.new(parsed_spec).extract(@record)
+        @parsed_spec = Traject::MarcExtractor::Spec.hash_from_string("700abcdef:856|*2|:505|1*|:245ba")
+        @values      = Traject::MarcExtractor.new(@parsed_spec).extract(@record)
       end
 
       it "returns an array" do
@@ -150,7 +150,7 @@ describe "Traject::MarcExtractor" do
       end
 
       it "does not have 505, due to non-matching indicators" do
-        assert ! @values.find {|s| s.include? "propaganda model"}
+        assert !@values.find { |s| s.include? "propaganda model" }, @values
       end
 
 
@@ -166,20 +166,20 @@ describe "Traject::MarcExtractor" do
 
     describe "extracts fixed fields" do
       it ", complete" do
-        parsed_spec = Traject::MarcExtractor.parse_string_spec("001")
-        values = Traject::MarcExtractor.new(parsed_spec).extract(@record)
+        parsed_spec = Traject::MarcExtractor::Spec.hash_from_string("001")
+        values      = Traject::MarcExtractor.new(parsed_spec).extract(@record)
 
         assert_equal ["2710183"], values
       end
       it ", single byte offset" do
-        parsed_spec = Traject::MarcExtractor.parse_string_spec("008[5]")
-        values = Traject::MarcExtractor.new(parsed_spec).extract(@record)
+        parsed_spec = Traject::MarcExtractor::Spec.hash_from_string("008[5]")
+        values      = Traject::MarcExtractor.new(parsed_spec).extract(@record)
 
         assert_equal ["1"], values
       end
       it ", byte range" do
-        parsed_spec = Traject::MarcExtractor.parse_string_spec("008[7-10]")
-        values = Traject::MarcExtractor.new(parsed_spec).extract(@record)
+        parsed_spec = Traject::MarcExtractor::Spec.hash_from_string("008[7-10]")
+        values      = Traject::MarcExtractor.new(parsed_spec).extract(@record)
 
         assert_equal ["2002"], values
       end
@@ -187,15 +187,15 @@ describe "Traject::MarcExtractor" do
 
     describe "separator argument" do
       it "causes non-join when nil" do
-        parsed_spec = Traject::MarcExtractor.parse_string_spec("245")
-        values = Traject::MarcExtractor.new(parsed_spec, :separator => nil).extract(@record)
+        parsed_spec = Traject::MarcExtractor::Spec.hash_from_string("245")
+        values      = Traject::MarcExtractor.new(parsed_spec, :separator => nil).extract(@record)
 
         assert_length 3, values
       end
 
       it "can be non-default" do
-        parsed_spec = Traject::MarcExtractor.parse_string_spec("245")
-        values = Traject::MarcExtractor.new(parsed_spec, :separator => "!! ").extract(@record)
+        parsed_spec = Traject::MarcExtractor::Spec.hash_from_string("245")
+        values      = Traject::MarcExtractor.new(parsed_spec, :separator => "!! ").extract(@record)
 
         assert_length 1, values
         assert_equal "Manufacturing consent :!! the political economy of the mass media /!! Edward S. Herman and Noam Chomsky ; with a new introduction by the authors.", values.first
@@ -204,8 +204,8 @@ describe "Traject::MarcExtractor" do
 
     describe "extracts alternate script" do
       before do
-        @record = MARC::Reader.new(support_file_path  "hebrew880s.marc").to_a.first
-        @parsed_spec = Traject::MarcExtractor.parse_string_spec("245b")
+        @record      = MARC::Reader.new(support_file_path  "hebrew880s.marc").to_a.first
+        @parsed_spec = Traject::MarcExtractor::Spec.hash_from_string("245b")
       end
       it "from default :include" do
 
@@ -301,10 +301,10 @@ describe "Traject::MarcExtractor" do
   describe "MarcExtractor.cached" do
     it "creates" do
       extractor = Traject::MarcExtractor.cached("245abc", :separator => nil)
-      spec_hash = extractor.spec_hash
+      spec_set  = extractor.spec_set
 
       assert extractor.options[:separator].nil?, "extractor options[:separator] is nil"
-      assert_equal({"245"=>[Traject::MarcExtractor::Spec.new(:tag => "245", :subfields=>["a", "b", "c"])]}, spec_hash)
+      assert_equal([Traject::MarcExtractor::Spec.new(:tag => "245", :subfields => ["a", "b", "c"])], spec_set.specs_for_tag('245'))
     end
     it "caches" do
       ext1 = Traject::MarcExtractor.cached("245abc", :separator => nil)

data/test/test_support/demo_config.rb

@@ -1,4 +1,4 @@
-# A sample traject configration, save as say `traject_config.rb`, then
+# A sample traject configuration, save as say `traject_config.rb`, then
 # run `traject -c traject_config.rb marc_file.marc` to index to
 # solr specified in config file, according to rules specified in
 # config file

data/traject.gemspec

@@ -29,19 +29,19 @@ Gem::Specification.new do |spec|
   spec.add_dependency "dot-properties", ">= 0.1.1"  # reading java style .properties
   spec.add_dependency "httpclient", "~> 2.5"
   spec.add_dependency 'marc-fastxmlwriter', '~>1.0' # fast marc->xml
-  
-  # If we're building the package under JRuby, add in the 
+
+  # If we're building the package under JRuby, add in the
   # jruby-only gems and specify the platform.
-  
+
   if defined? JRUBY_VERSION
     spec.platform = 'java'
     spec.add_dependency "traject-marc4j_reader", "~> 1.0"
   else
     spec.platform = "ruby"
   end
-  
 
-  spec.add_development_dependency "bundler", "~> 1.3"
+
+  spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "minitest"
 end