traject 0.9.1 → 0.10.0

data/lib/traject/version.rb

@@ -1,3 +1,3 @@
 module Traject
-  VERSION = "0.9.1"
+  VERSION = "0.10.0"
 end

data/lib/traject/yaml_writer.rb

@@ -0,0 +1,9 @@
+require 'traject/line_writer'
+require 'yaml'
+
+class Traject::YamlWriter < Traject::LineWriter
+  def serialize(context)
+    context.output_hash.to_yaml(:indentation=>3, :line_width => 78) + "\n\n"
+  end
+end
+  

data/test/debug_writer_test.rb

@@ -0,0 +1,38 @@
+require 'test_helper'
+require 'stringio'
+
+require 'traject/debug_writer'
+require 'traject'
+require 'marc'
+
+describe 'Simple output' do
+  before do
+    @record = MARC::Reader.new(support_file_path  "manufacturing_consent.marc").to_a.first
+    @indexer = Traject::Indexer.new
+    @indexer.instance_eval do
+      to_field "id", extract_marc("001", :first => true)
+      to_field "title", extract_marc("245ab")
+    end
+    @io = StringIO.new
+    @writer = Traject::DebugWriter.new("output_stream" => @io)
+    
+    @id = "2710183"
+    @title = "Manufacturing consent : the political economy of the mass media /"
+  end
+  
+  it "does a simple output" do
+    @writer.put Traject::Indexer::Context.new(:output_hash => @indexer.map_record(@record))
+    expected = [
+      "#{@id} id #{@id}",
+      "#{@id} title #{@title}",
+      "\n"
+    ]
+    assert_equal expected.join("\n").gsub(/\s/, ''), @io.string.gsub(/\s/, '')
+    @writer.close
+    
+  end
+  
+end
+    
+    
+    

data/test/indexer/each_record_test.rb

@@ -7,13 +7,13 @@ describe "Traject::Indexer#each_record" do
 
   describe "checks arguments" do
     it "rejects no-arg block" do
-      assert_raises(ArgumentError) do
+      assert_raises(Traject::Indexer::ArityError) do
         @indexer.each_record do
         end
       end
     end
     it "rejects three-arg block" do
-      assert_raises(ArgumentError) do
+      assert_raises(Traject::Indexer::ArityError) do
         @indexer.each_record do |one, two, three|
         end
       end
@@ -30,5 +30,30 @@ describe "Traject::Indexer#each_record" do
       @indexer.each_record do |*variable|
       end
     end
+
+    it "finds first (only) field on each_record error" do
+      begin
+        @indexer.to_field('foo') {|one, two| }
+        @indexer.each_record {|one, two, three| }   # bad arity
+        flunk("Should have rejected bad arity ")
+      rescue Traject::Indexer::ArityError => e
+        assert_match(/foo/, e.message)
+      rescue 
+        flunk("Should only fail with a ArityError")
+      end
+    end
+    
+    it "rejects each_record with a name (e.g., using a to_field syntax)" do
+      assert_raises(Traject::Indexer::NamingError) do
+        @indexer.each_record('bad_name') {|one, two| }
+      end
+    end
+    
+    it "reject each_record with no arguments/blocks at all" do
+      assert_raises(ArgumentError) do
+        @indexer.each_record()
+      end
+    end
+
   end
 end

data/test/indexer/macros_marc21_semantics_test.rb

@@ -25,7 +25,18 @@ describe "Traject::Macros::Marc21Semantics" do
     end
     output = @indexer.map_record(@record)
 
-    assert_equal %w{2710183 47971712},  output["oclcnum"]
+    assert_equal %w{47971712},  output["oclcnum"]
+  end
+
+  it "#marc_series_facet" do
+    @record = MARC::Reader.new(support_file_path  "louis_armstrong.marc").to_a.first
+
+    @indexer.instance_eval do
+      to_field "series_facet", marc_series_facet
+    end
+    output = @indexer.map_record(@record)
+
+    assert_equal ["Big bands."], output["series_facet"]
   end
 
   describe "marc_sortable_author" do

data/test/indexer/settings_test.rb

@@ -114,8 +114,15 @@ describe "Traject::Indexer#settings" do
     assert_equal "new", settings["c"]
   end
 
-  describe "defaults" do
-
+  describe "inspect" do
+    it "keeps keys ending in 'password' out of inspect" do
+      settings = Traject::Indexer::Settings.new("a" => "a", 
+        "password" => "password", "some_password" => "password",
+        "some.password" => "password")
+
+      parsed = eval( settings.inspect )
+      assert_equal( {"a" => "a", "password" => "[hidden]", "some_password" => "[hidden]", "some.password" => "[hidden]"}, parsed)
+    end
   end
 
 end

data/test/indexer/to_field_test.rb

@@ -6,20 +6,23 @@ describe "Traject::Indexer.to_field" do
   end
   describe "checks it's arguments" do
     it "rejects nil first arg" do
-      assert_raises(ArgumentError) { @indexer.to_field(nil) }
+      assert_raises(Traject::Indexer::NamingError) { @indexer.to_field(nil) }
     end
     it "rejects empty string first arg" do
-      assert_raises(ArgumentError) {@indexer.to_field("")}
+      assert_raises(Traject::Indexer::NamingError) {@indexer.to_field("")}
     end
+    it "rejects non-string first arg" do
+      assert_raises(Traject::Indexer::NamingError) {@indexer.to_field(:symbol)}
+    end
+    
     it "rejects one-arg lambda" do
-      assert_raises(ArgumentError) do
+      assert_raises(Traject::Indexer::ArityError) do
         @indexer.to_field("foo") do |one_arg|
-
         end
       end
     end
     it "rejects four-arg lambda" do
-      assert_raises(ArgumentError) do 
+      assert_raises(Traject::Indexer::ArityError) do 
         @indexer.to_field("foo") do |one_arg, two_arg, three_arg, four_arg|
         end
       end
@@ -36,4 +39,31 @@ describe "Traject::Indexer.to_field" do
       end
     end
   end
+  
+  describe "gives location in error message" do
+
+    it "finds no previous field on initial error" do
+      begin
+        @indexer.to_field('') {|one, two| }   # bad field name
+        flunk("Should have rejected empty field name")
+      rescue Traject::Indexer::NamingError => e
+        assert_match(/no previous named fields/, e.message)
+      rescue 
+        flunk("Should only fail with a NamingError")
+      end
+    end
+
+    it "finds first (only) field on error" do
+      begin
+        @indexer.to_field('foo') {|one, two| }
+        @indexer.to_field('') {|one, two| }   # bad field name
+        flunk("Should have rejected empty field name")
+      rescue Traject::Indexer::NamingError => e
+        assert_match(/foo/, e.message)
+      rescue 
+        flunk("Should only fail with a NamingError")
+      end
+    end
+  end
+  
 end

data/test/marc4j_reader_test.rb

@@ -54,6 +54,9 @@ describe "Marc4JReader" do
     # it's legal, it probably looks weird as a string literal
     # below, depending on your editor.
     assert_equal "Por uma outra globalização :", a245a
+
+    # Set leader byte to proper for unicode
+    assert_equal 'a', array.first.leader[9]
   end
 
 

data/test/marc_extractor_test.rb

@@ -50,6 +50,67 @@ describe "Traject::MarcExtractor" do
       assert_equal 5, parsed["005"][:bytes]
       assert_equal 7..10, parsed["008"][:bytes]
     end
+    
+    it "allows arrays of specs" do
+      parsed = Traject::MarcExtractor.parse_string_spec %w(
+        245abcde
+        810
+        700|*4|bcd
+      )
+      assert_length 3, parsed
+    end
+    
+    it "allows mixture of array and colon-delimited specs" do
+      parsed = Traject::MarcExtractor.parse_string_spec %w(
+        245abcde
+        100:110:111
+        810
+        700|*4|bcd
+      )
+      assert_length 6, parsed
+    end
+      
+    
+  end
+
+  # Mostly an internal method, not neccesarily API, but
+  # an important one, so we unit test some parts of it.
+  describe "#spec_covering_field" do
+    describe "for alternate script tags" do
+      before do
+        @record = MARC::Reader.new(support_file_path  "hebrew880s.marc").to_a.first
+        @extractor = Traject::MarcExtractor.new("245")
+
+        @a245 = @record.fields.find {|f| f.tag == "245"}
+        assert ! @a245.nil?, "Found a 245 to test"
+
+        @a880_245 = @record.fields.find do |field|
+          (field.tag == "880") && field['6'] &&
+          "245" == field['6'].slice(0,3)
+        end
+        assert ! @a880_245.nil?, "Found an 880-245 to test"
+
+        @a880_100 = @record.fields.find do |field|
+          (field.tag == "880") && field['6'] &&
+          "100" == field['6'].slice(0,3)
+        end
+
+        assert ! @a880_100.nil?, "Found an 880-100 to test"
+      end
+      it "finds spec for relevant 880" do
+        assert_equal( {}, @extractor.spec_covering_field(@a880_245) )
+        assert_nil        @extractor.spec_covering_field(@a880_100)
+      end
+      it "does not find spec for 880 if disabled" do
+        @extractor = Traject::MarcExtractor.new("245", :alternate_script => false)
+        assert_nil @extractor.spec_covering_field(@a880_245) 
+      end
+      it "finds only 880 if so configured" do
+        @extractor = Traject::MarcExtractor.new("245", :alternate_script => :only)
+        assert_nil @extractor.spec_covering_field(@a245) 
+        assert_equal({},  @extractor.spec_covering_field(@a880_245))
+      end
+    end
   end
 
   describe "#extract_by_spec" do
@@ -60,7 +121,7 @@ describe "Traject::MarcExtractor" do
     describe "extracts a basic case" do
       before do
         parsed_spec = Traject::MarcExtractor.parse_string_spec("700abcdef:856|*2|:505|1*|:245ba")
-        @values = Traject::MarcExtractor.extract_by_spec(@record, parsed_spec)
+        @values = Traject::MarcExtractor.new(parsed_spec).extract(@record)
       end
 
       it "returns an array" do
@@ -94,19 +155,19 @@ describe "Traject::MarcExtractor" do
     describe "extracts fixed fields" do
       it ", complete" do
         parsed_spec = Traject::MarcExtractor.parse_string_spec("001")
-        values = Traject::MarcExtractor.extract_by_spec(@record, parsed_spec)
+        values = Traject::MarcExtractor.new(parsed_spec).extract(@record)
 
         assert_equal ["2710183"], values
       end
       it ", single byte offset" do
         parsed_spec = Traject::MarcExtractor.parse_string_spec("008[5]")
-        values = Traject::MarcExtractor.extract_by_spec(@record, parsed_spec)
+        values = Traject::MarcExtractor.new(parsed_spec).extract(@record)
 
         assert_equal ["1"], values
       end
       it ", byte range" do
         parsed_spec = Traject::MarcExtractor.parse_string_spec("008[7-10]")
-        values = Traject::MarcExtractor.extract_by_spec(@record, parsed_spec)
+        values = Traject::MarcExtractor.new(parsed_spec).extract(@record)
 
         assert_equal ["2002"], values
       end
@@ -115,14 +176,14 @@ describe "Traject::MarcExtractor" do
     describe "seperator argument" do
       it "causes non-join when nil" do
         parsed_spec = Traject::MarcExtractor.parse_string_spec("245")
-        values = Traject::MarcExtractor.extract_by_spec(@record, parsed_spec, :seperator => nil)
+        values = Traject::MarcExtractor.new(parsed_spec, :seperator => nil).extract(@record)
 
         assert_length 3, values
       end
 
       it "can be non-default" do
         parsed_spec = Traject::MarcExtractor.parse_string_spec("245")
-        values = Traject::MarcExtractor.extract_by_spec(@record, parsed_spec, :seperator => "!! ")
+        values = Traject::MarcExtractor.new(parsed_spec, :seperator => "!! ").extract(@record)
 
         assert_length 1, values
         assert_equal "Manufacturing consent :!! the political economy of the mass media /!! Edward S. Herman and Noam Chomsky ; with a new introduction by the authors.", values.first
@@ -136,19 +197,19 @@ describe "Traject::MarcExtractor" do
       end
       it "from default :include" do
 
-        values = Traject::MarcExtractor.extract_by_spec(@record, @parsed_spec)
+        values = Traject::MarcExtractor.new(@parsed_spec).extract(@record)
 
         assert_length 2, values # both the original and the 880
         assert_equal ["ben Marṭin Buber le-Aharon Daṿid Gordon /", "בין מרטין בובר לאהרן דוד גורדון /"], values
       end
       it "with :only" do
-        values = Traject::MarcExtractor.extract_by_spec(@record, @parsed_spec, :alternate_script => :only)
+        values = Traject::MarcExtractor.new(@parsed_spec, :alternate_script => :only).extract(@record)
 
         assert_length 1, values
         assert_equal ["בין מרטין בובר לאהרן דוד גורדון /"], values
       end
       it "with false" do
-        values = Traject::MarcExtractor.extract_by_spec(@record, @parsed_spec, :alternate_script => false)
+        values = Traject::MarcExtractor.new(@parsed_spec, :alternate_script => false).extract(@record)
 
         assert_length 1, values
         assert_equal ["ben Marṭin Buber le-Aharon Daṿid Gordon /"], values
@@ -156,22 +217,22 @@ describe "Traject::MarcExtractor" do
     end
 
     it "works with string second arg too" do
-      values = Traject::MarcExtractor.extract_by_spec(@record, "245abc")
+      values = Traject::MarcExtractor.new("245abc").extract(@record)
 
       assert_length 1, values
       assert values.first.include?("Manufacturing consent"), "Extracted value includes title"
     end
 
     it "returns empty array if no matching tags" do
-      values = Traject::MarcExtractor.extract_by_spec(@record, "999abc")
+      values = Traject::MarcExtractor.new("999abc").extract(@record)
       assert_equal [], values
 
-      values = Traject::MarcExtractor.extract_by_spec(@record, "999")
+      values = Traject::MarcExtractor.new("999").extract(@record)
       assert_equal [], values
     end
 
-    it "returns empty array if matching tag but no subfield" do 
-      values = Traject::MarcExtractor.extract_by_spec(@record, "245xyz")
+    it "returns empty array if matching tag but no subfield" do
+      values = Traject::MarcExtractor.new("245xyz").extract(@record)
       assert_equal [], values
     end
 
@@ -180,7 +241,7 @@ describe "Traject::MarcExtractor" do
   describe "with bad data" do
     it "can ignore an 880 with no $6" do
       @record = MARC::Reader.new(support_file_path  "880_with_no_6.utf8.marc").to_a.first
-      values = Traject::MarcExtractor.extract_by_spec(@record, "001")
+      values = Traject::MarcExtractor.new("001").extract(@record)
       assert_equal ["3468569"], values
     end
   end
@@ -188,11 +249,11 @@ describe "Traject::MarcExtractor" do
   describe "#each_matching_line" do
     before do
       @record = MARC::Reader.new(support_file_path  "manufacturing_consent.marc").to_a.first
-      @extractor = Traject::MarcExtractor.new(@record, "245abc")
+      @extractor = Traject::MarcExtractor.new("245abc")
     end
     it "yields two args" do
       called = false
-      @extractor.each_matching_line do |field, spec|
+      @extractor.each_matching_line(@record) do |field, spec|
         called = true
         assert_kind_of MARC::DataField, field
         assert_kind_of Hash, spec
@@ -201,7 +262,7 @@ describe "Traject::MarcExtractor" do
     end
     it "yields three args" do
       called = false
-      @extractor.each_matching_line do |field, spec, extractor|
+      @extractor.each_matching_line(@record) do |field, spec, extractor|
         called = true
         assert_kind_of MARC::DataField, field
         assert_kind_of Hash, spec
@@ -215,16 +276,29 @@ describe "Traject::MarcExtractor" do
   describe "#collect_matching_lines" do
     before do
       @record = MARC::Reader.new(support_file_path  "manufacturing_consent.marc").to_a.first
-      @extractor = Traject::MarcExtractor.new(@record, "245abc")
+      @extractor = Traject::MarcExtractor.new("245abc")
     end
     it "collects with custom block" do
-      results = @extractor.collect_matching_lines do |field, spec, extractor|
+      results = @extractor.collect_matching_lines(@record) do |field, spec, extractor|
         extractor.collect_subfields(field, spec)
       end
       assert_equal ["Manufacturing consent : the political economy of the mass media / Edward S. Herman and Noam Chomsky ; with a new introduction by the authors."], results
     end
   end
 
+  describe "MarcExtractor.cached" do
+    it "creates" do
+      ext = Traject::MarcExtractor.cached("245abc", :seperator => nil)
+      assert_equal({"245"=>{:subfields=>["a", "b", "c"]}}, ext.spec_hash)
+      assert ext.options[:seperator].nil?, "extractor options[:seperator] is nil"
+    end
+    it "caches" do
+      ext1 = Traject::MarcExtractor.cached("245abc", :seperator => nil)
+      ext2 = Traject::MarcExtractor.cached("245abc", :seperator => nil)
+
+      assert_same ext1, ext2
+    end
+  end
 
 
 end

data/test/test_support/demo_config.rb

@@ -105,11 +105,14 @@ to_field "pub_date",          marc_publication_date
 
 # LCC to broad class, start with built-in from marc record, but then do our own for local
 # call numbers.
-lcc_map = Traject::TranslationMap.new("lcc_top_level")
+lcc_map             = Traject::TranslationMap.new("lcc_top_level")
+holdings_extractor  = Traject::MarcExtractor.new("991:937")
+sudoc_extractor     = Traject::MarcExtractor.new("086a", :seperator =>nil)
+
 to_field "discipline_facet",  marc_lcc_to_broad_category(:default => nil) do |record, accumulator|
   # add in our local call numbers
   accumulator.concat(
-    Traject::MarcExtractor.new(record, "991:937").collect_matching_lines do |field, spec, extractor|
+    holdings_extractor.collect_matching_lines(record) do |field, spec, extractor|
         # we output call type 'processor' in subfield 'f' of our holdings
         # fields, that sort of maybe tells us if it's an LCC field.
         # When the data is right, which it often isn't.
@@ -130,7 +133,7 @@ to_field "discipline_facet",  marc_lcc_to_broad_category(:default => nil) do |re
 
   # If it's got an 086, we'll put it in "Government Publication", to be
   # consistent with when we do that from a local SuDoc call #.
-  if Traject::MarcExtractor.extract_by_spec(record, "086a", :seperator =>nil).length > 0
+  if sudoc_extractor.extract(record).length > 0
     accumulator << "Government Publication"
   end