traject 0.0.2 → 0.9.1

data/test/indexer/macros_marc21_test.rb

@@ -45,7 +45,16 @@ describe "Traject::Macros::Marc21" do
 
       output = @indexer.map_record(@record)
 
-      assert ! output["title"].first.end_with?("/"), "does not end with /"
+      assert_equal ["Manufacturing consent : the political economy of the mass media"], output["title"]
+    end
+
+    it "respects :default option" do
+      @indexer.instance_eval do
+        to_field "only_default", extract_marc("9999", :default => "DEFAULT VALUE")
+      end
+      output = @indexer.map_record(@record)
+
+      assert_equal ["DEFAULT VALUE"], output["only_default"]
     end
 
     it "Marc21::trim_punctuation class method" do

data/test/indexer/map_record_test.rb

@@ -1,9 +1,9 @@
 require 'test_helper'
 
-describe "Traject::Indexer#map_record" do 
+describe "Traject::Indexer#map_record" do
   before do
     @indexer = Traject::Indexer.new
-    @record = MARC::Reader.new(support_file_path  "manufacturing_consent.marc").to_a.first    
+    @record = MARC::Reader.new(support_file_path  "manufacturing_consent.marc").to_a.first
   end
 
 
@@ -16,7 +16,7 @@ describe "Traject::Indexer#map_record" do
     end
   end
 
-  describe "#to_field" do    
+  describe "#to_field" do
     it "works with block" do
       called  = false
 
@@ -27,7 +27,7 @@ describe "Traject::Indexer#map_record" do
         called = true # by the power of closure!
         accumulator << "Some Title"
       end
-      
+
       output = @indexer.map_record(@record)
 
       assert called
@@ -46,8 +46,8 @@ describe "Traject::Indexer#map_record" do
         accumulator << "Some Title"
       end
 
-      @indexer.to_field("title", logic) 
-      
+      @indexer.to_field("title", logic)
+
       output = @indexer.map_record(@record)
 
       assert called
@@ -103,10 +103,11 @@ describe "Traject::Indexer#map_record" do
         called = true
 
         assert_kind_of Traject::Indexer::Context, context
-        
+
         assert_kind_of Hash, context.clipboard
         assert_kind_of Hash, context.output_hash
 
+        assert_same @record, record
         assert_same record, context.source_record
         assert_same @indexer.settings, context.settings
       end
@@ -116,5 +117,74 @@ describe "Traject::Indexer#map_record" do
       assert called
     end
   end
-  
+
+  describe "#each_record" do
+    it "is called with one-arg record" do
+      called = false
+      @indexer.each_record do |record|
+        called = true
+        assert_kind_of MARC::Record, record
+      end
+      @indexer.map_record(@record)
+
+      assert called, "each_record was called"
+    end
+    it "is called with two-arg record and context" do
+      called = false
+      @indexer.each_record do |record, context|
+        called = true
+        assert_kind_of MARC::Record, record
+        assert_kind_of Traject::Indexer::Context, context
+      end
+      @indexer.map_record(@record)
+
+      assert called, "each_record was called"
+    end
+    it "accepts lambda AND block" do
+      lambda_arg = lambda do |record, context|
+        context.output_hash["field"] ||= []
+        context.output_hash["field"] << "first"
+      end
+
+      @indexer.each_record(lambda_arg) do |record, context|
+        context.output_hash["field"] ||= []
+        context.output_hash["field"] << "second"
+      end
+
+      output = @indexer.map_record(@record)
+
+      assert_equal %w{first second}, output["field"]
+    end
+    it "is called in order with #to_field" do
+      @indexer.to_field("foo") {|record, accumulator| accumulator << "first"}
+      @indexer.each_record {|record, context| context.output_hash["foo"] << "second" }
+      @indexer.to_field("foo") {|record, accumulator| accumulator << "third"}
+
+      output = @indexer.map_record(@record)
+
+      assert_equal %w{first second third}, output["foo"]
+    end
+  end
+
+  describe "map_to_context!" do
+    before do
+      @context = Traject::Indexer::Context.new(:source_record => @record, :settings => @indexer.settings, :position => 10 )
+    end
+    it "passes context to indexing routines"  do
+      called = false
+      @indexer.to_field("title") do |record, accumulator, context|
+        called = true
+        assert_kind_of Traject::Indexer::Context, context
+        assert_same @context, context
+      end
+
+      context = @indexer.map_to_context!(@context)
+
+      assert_same @context, context
+
+      assert called, "Called mapping routine"
+    end
+
+  end
+
 end

data/test/indexer/read_write_test.rb

@@ -4,7 +4,8 @@ require 'test_helper'
 # in an array, just added to settings for easy access
 memory_writer_class = Class.new do
     def initialize(settings)
-      @settings = settings
+      # store them in a class variable so we can test em later
+      @@last_writer_settings = @settings = settings
       @settings["memory_writer.added"] = []
     end
 
@@ -17,31 +18,63 @@ memory_writer_class = Class.new do
     end
   end
 
-describe "Traject::Indexer#process" do 
+describe "Traject::Indexer#process" do
   before do
-    @indexer = Traject::Indexer.new
+    # no threading for these tests
+    @indexer = Traject::Indexer.new("processing_thread_pool" => nil)
     @indexer.writer_class = memory_writer_class
     @file = File.open(support_file_path "test_data.utf8.mrc")
   end
 
   it "works" do
+    # oops, this times_called counter isn't thread-safe under multi-threading 
+    # is why this fails sometimes.
+    # fixed to be single-threaded for these tests. 
+    times_called = 0
     @indexer.to_field("title") do |record, accumulator, context|
+      times_called += 1
       accumulator << "ADDED TITLE"
       assert_equal "title", context.field_name
+
+      assert_equal times_called, context.position
     end
 
-    @indexer.process( @file )
+    return_value = @indexer.process( @file )
+
+    assert return_value, "Returns `true` on success"
 
-    assert @indexer.settings["memory_writer.added"]
-    assert_equal 30, @indexer.settings["memory_writer.added"].length
-    assert_kind_of Hash, @indexer.settings["memory_writer.added"].first
-    assert_equal ["ADDED TITLE"], @indexer.settings["memory_writer.added"].first["title"]
+    # Grab the settings out of a class variable where we left em,
+    # as a convenient place to store outcomes so we can test em.
+    writer_settings = memory_writer_class.class_variable_get("@@last_writer_settings")
 
-    assert @indexer.settings["memory_writer.closed"]
+    assert writer_settings["memory_writer.added"]
+    assert_equal 30, writer_settings["memory_writer.added"].length
+    assert_kind_of Traject::Indexer::Context, writer_settings["memory_writer.added"].first
+    assert_equal ["ADDED TITLE"], writer_settings["memory_writer.added"].first.output_hash["title"]
 
+    # logger provided in settings
+    assert writer_settings["logger"]
+
+    assert writer_settings["memory_writer.closed"]
   end
 
-  
+  it "returns false if skipped records" do
+    @indexer = Traject::Indexer.new(
+      "solrj_writer.server_class_name" => "MockSolrServer",
+      "solr.url" => "http://example.org",
+      "writer_class_name" => "Traject::SolrJWriter"
+    )
+    @file = File.open(support_file_path "manufacturing_consent.marc")    
+
+
+    @indexer.to_field("id") do |record, accumulator|
+      # intentionally make error
+      accumulator.concat ["one_id", "two_id"]
+    end
+    return_value = @indexer.process(@file)
+
+    assert ! return_value, "returns false on skipped record errors"
+  end
 
 
 end

data/test/indexer/settings_test.rb

@@ -1,13 +1,31 @@
 require 'test_helper'
 
-describe "Traject::Indexer#settings" do 
+describe "Traject::Indexer#settings" do
   before do
     @indexer = Traject::Indexer.new
   end
 
-  it "starts out default hash" do
+  it "starts out a Hash, that can fill in it's defaults" do
     assert_kind_of Hash, @indexer.settings
-    assert_equal Traject::Indexer.default_settings, @indexer.settings
+
+    Traject::Indexer::Settings.defaults.each_pair do |key, value|
+      assert_equal value, @indexer.settings[key]
+    end
+  end
+
+  it "can fill_in_defaults!" do
+    @indexer.settings.fill_in_defaults!
+
+    assert_equal Traject::Indexer::Settings.defaults, @indexer.settings
+  end
+
+  it "doesn't overwrite with fill_in_defaults!" do
+    key = Traject::Indexer::Settings.defaults.keys.first
+    @indexer.settings[ key  ] = "MINE KEEP IT"
+
+    @indexer.settings.fill_in_defaults!
+
+    assert_equal "MINE KEEP IT", @indexer.settings[key]
   end
 
   it "can take argument to set" do
@@ -42,7 +60,7 @@ describe "Traject::Indexer#settings" do
     @indexer.settings("four" => "fourth")
 
     {"one" => "original", "two" => "second", "three" => "third", "four" => "fourth"}.each_pair do |key, value|
-      assert_equal value, @indexer.settings[key] 
+      assert_equal value, @indexer.settings[key]
     end
   end
 
@@ -60,6 +78,44 @@ describe "Traject::Indexer#settings" do
     assert_equal "foo 4", @indexer.settings["foo"]
   end
 
+  it "implements #provide as cautious setter" do
+    @indexer.settings[:a] = "original"
+
+    @indexer.settings do
+      provide :a, "new"
+      provide :b, "new"
+    end
+
+    assert_equal "original", @indexer.settings[:a]
+    assert_equal "new", @indexer.settings[:b]
+  end
+
+  it "has reverse_merge" do
+    settings = Traject::Indexer::Settings.new("a" => "original", "b" => "original")
+
+    new_settings = settings.reverse_merge(:a => "new",  :c => "new")
+
+    assert_kind_of Traject::Indexer::Settings, new_settings
+
+    assert_equal "original", new_settings["a"]
+    assert_equal "original", new_settings["b"]
+    assert_equal "new", new_settings["c"]
+  end
+
+  it "has reverse_merge!" do
+    settings = Traject::Indexer::Settings.new("a" => "original", "b" => "original")
 
+    settings.reverse_merge!(:a => "new",  :c => "new")
+
+    assert_kind_of Traject::Indexer::Settings, settings
+
+    assert_equal "original", settings["a"]
+    assert_equal "original", settings["b"]
+    assert_equal "new", settings["c"]
+  end
+
+  describe "defaults" do
+
+  end
 
 end

data/test/indexer/to_field_test.rb

@@ -0,0 +1,39 @@
+require 'test_helper'
+
+describe "Traject::Indexer.to_field" do
+  before do 
+    @indexer = Traject::Indexer.new
+  end
+  describe "checks it's arguments" do
+    it "rejects nil first arg" do
+      assert_raises(ArgumentError) { @indexer.to_field(nil) }
+    end
+    it "rejects empty string first arg" do
+      assert_raises(ArgumentError) {@indexer.to_field("")}
+    end
+    it "rejects one-arg lambda" do
+      assert_raises(ArgumentError) do
+        @indexer.to_field("foo") do |one_arg|
+
+        end
+      end
+    end
+    it "rejects four-arg lambda" do
+      assert_raises(ArgumentError) do 
+        @indexer.to_field("foo") do |one_arg, two_arg, three_arg, four_arg|
+        end
+      end
+    end
+    it "accepts two arg lambda" do
+      @indexer.to_field("foo") do |one, two|
+      end
+    end
+    it "accepts three arg lambda" do
+      @indexer.to_field("foo") {|one, two, three| one }
+    end
+    it "accepts variable lambda" do
+      @indexer.to_field("foo") do |*variable|
+      end
+    end
+  end
+end

data/test/marc4j_reader_test.rb

@@ -0,0 +1,75 @@
+# Encoding: UTF-8
+
+require 'test_helper'
+
+require 'traject'
+require 'traject/indexer'
+require 'traject/marc4j_reader'
+
+require 'marc'
+
+describe "Marc4JReader" do
+  it "reads Marc binary" do
+    file = File.new(support_file_path("test_data.utf8.mrc"))
+    settings = Traject::Indexer::Settings.new() # binary type is default
+    reader = Traject::Marc4JReader.new(file, settings)
+
+    array = reader.to_a
+
+    assert_equal 30, array.length
+    first = array.first
+
+    assert_kind_of MARC::Record, first
+    assert first['245']['a'].encoding.name, "UTF-8"
+  end
+
+  it "can skip a bad subfield code" do
+    file = File.new(support_file_path("bad_subfield_code.marc"))
+    settings = Traject::Indexer::Settings.new() # binary type is default
+    reader = Traject::Marc4JReader.new(file, settings)
+
+    array = reader.to_a
+
+    assert_equal 1, array.length
+    assert_kind_of MARC::Record, array.first
+    assert_length 2, array.first['260'].subfields
+  end
+
+  it "reads Marc binary in Marc8 encoding" do
+    file = File.new(support_file_path("one-marc8.mrc"))
+    settings = Traject::Indexer::Settings.new("marc4j_reader.source_encoding" => "MARC8")
+    reader = Traject::Marc4JReader.new(file, settings)
+
+    array = reader.to_a
+
+    assert_length 1, array
+
+
+    assert_kind_of MARC::Record, array.first
+    a245a = array.first['245']['a']
+
+    assert a245a.encoding.name, "UTF-8"
+    assert a245a.valid_encoding?
+    # marc4j converts to denormalized unicode, bah. Although
+    # it's legal, it probably looks weird as a string literal
+    # below, depending on your editor.
+    assert_equal "Por uma outra globalização :", a245a
+  end
+
+
+  it "reads XML" do
+    file = File.new(support_file_path "test_data.utf8.marc.xml")
+    settings = Traject::Indexer::Settings.new("marc_source.type" => "xml")
+    reader = Traject::Marc4JReader.new(file, settings)
+
+    array = reader.to_a
+
+    assert_equal 30, array.length
+
+    first = array.first
+
+    assert_kind_of MARC::Record, first
+    assert first['245']['a'].encoding.name, "UTF-8"
+    assert_equal "Fikr-i Ayāz /", first['245']['a']
+  end
+end

data/test/marc_extractor_test.rb

@@ -3,6 +3,7 @@
 require 'test_helper'
 require 'traject/marc_extractor'
 
+require 'marc'
 
 describe "Traject::MarcExtractor" do
   describe "#parse_marc_spec" do
@@ -161,6 +162,67 @@ describe "Traject::MarcExtractor" do
       assert values.first.include?("Manufacturing consent"), "Extracted value includes title"
     end
 
+    it "returns empty array if no matching tags" do
+      values = Traject::MarcExtractor.extract_by_spec(@record, "999abc")
+      assert_equal [], values
+
+      values = Traject::MarcExtractor.extract_by_spec(@record, "999")
+      assert_equal [], values
+    end
+
+    it "returns empty array if matching tag but no subfield" do 
+      values = Traject::MarcExtractor.extract_by_spec(@record, "245xyz")
+      assert_equal [], values
+    end
+
+  end
+
+  describe "with bad data" do
+    it "can ignore an 880 with no $6" do
+      @record = MARC::Reader.new(support_file_path  "880_with_no_6.utf8.marc").to_a.first
+      values = Traject::MarcExtractor.extract_by_spec(@record, "001")
+      assert_equal ["3468569"], values
+    end
+  end
+
+  describe "#each_matching_line" do
+    before do
+      @record = MARC::Reader.new(support_file_path  "manufacturing_consent.marc").to_a.first
+      @extractor = Traject::MarcExtractor.new(@record, "245abc")
+    end
+    it "yields two args" do
+      called = false
+      @extractor.each_matching_line do |field, spec|
+        called = true
+        assert_kind_of MARC::DataField, field
+        assert_kind_of Hash, spec
+      end
+      assert called, "calls block"
+    end
+    it "yields three args" do
+      called = false
+      @extractor.each_matching_line do |field, spec, extractor|
+        called = true
+        assert_kind_of MARC::DataField, field
+        assert_kind_of Hash, spec
+        assert_kind_of Traject::MarcExtractor, extractor
+        assert_same @extractor, extractor
+      end
+      assert called, "calls block"
+    end
+  end
+
+  describe "#collect_matching_lines" do
+    before do
+      @record = MARC::Reader.new(support_file_path  "manufacturing_consent.marc").to_a.first
+      @extractor = Traject::MarcExtractor.new(@record, "245abc")
+    end
+    it "collects with custom block" do
+      results = @extractor.collect_matching_lines do |field, spec, extractor|
+        extractor.collect_subfields(field, spec)
+      end
+      assert_equal ["Manufacturing consent : the political economy of the mass media / Edward S. Herman and Noam Chomsky ; with a new introduction by the authors."], results
+    end
   end
 
 

data/test/marc_format_classifier_test.rb

@@ -0,0 +1,91 @@
+require 'test_helper'
+
+require 'traject/macros/marc_format_classifier'
+
+MarcFormatClassifier = Traject::Macros::MarcFormatClassifier
+
+def classifier_for(filename)
+  record = MARC::Reader.new(support_file_path  filename).to_a.first
+  return MarcFormatClassifier.new( record  )
+end
+
+describe "MarcFormatClassifier" do
+
+  describe "genre" do
+    # We don't have the patience to test every case, just a sampling
+    it "says book" do
+      assert_equal ["Book"], classifier_for("manufacturing_consent.marc").genre
+    end
+    it "says Book for a weird one" do
+      assert_equal ["Book"], classifier_for("microform_online_conference.marc").genre
+    end
+    it "says Musical Recording" do
+      assert_equal ["Musical Recording"], classifier_for("musical_cage.marc").genre
+    end
+    it "says Journal" do
+      assert_equal ["Journal/Newspaper"], classifier_for("the_business_ren.marc").genre
+    end
+  end
+
+
+  describe "print?" do
+    it "says print when it is" do
+      assert classifier_for("manufacturing_consent.marc").print?
+    end
+    it "does not say print for online only" do
+      assert ! classifier_for("online_only.marc").print?
+    end
+  end
+
+  describe "online?" do
+    it "says online when it is" do
+      assert classifier_for("online_only.marc").online?
+      assert classifier_for("microform_online_conference.marc").online?
+      assert classifier_for("manuscript_online_thesis.marc").online?
+    end
+    it "does not say online for a print only" do
+      assert ! classifier_for("manufacturing_consent.marc").online?
+    end
+  end
+
+  describe "microform?" do
+    it "says microform when it is" do
+      assert classifier_for("microform_online_conference.marc").microform?
+    end
+    it "does not say microform when it ain't" do
+       assert ! classifier_for("manufacturing_consent.marc").microform?
+       assert ! classifier_for("online_only.marc").microform?
+    end
+  end
+
+  describe "conference?" do
+    it "says conference when it is" do
+      assert classifier_for("microform_online_conference.marc").proceeding?      
+    end
+    it "does not say conference when it ain't" do
+      assert ! classifier_for("manufacturing_consent.marc").proceeding?
+      assert ! classifier_for("online_only.marc").proceeding?
+    end
+  end
+
+  describe "thesis?" do
+    it "says thesis when it is" do
+      assert classifier_for("manuscript_online_thesis.marc").thesis?
+    end
+    it "does not say thesis when it ain't" do
+      assert ! classifier_for("manufacturing_consent.marc").thesis?
+      assert ! classifier_for("online_only.marc").thesis?
+    end
+  end
+
+  describe "manuscript_archive?" do
+    it "says manuscript when it is" do
+      assert classifier_for("manuscript_online_thesis.marc").manuscript_archive?
+    end
+    it "does not say manuscript when it ain't" do
+      assert ! classifier_for("manufacturing_consent.marc").manuscript_archive?
+      assert ! classifier_for("online_only.marc").manuscript_archive?
+    end
+  end 
+
+end

data/test/marc_reader_test.rb

@@ -1,5 +1,8 @@
+# Encoding: UTF-8
+
 require 'test_helper'
 require 'traject/marc_reader'
+require 'marc'
 
 describe "Traject::MarcReader" do
 
@@ -22,8 +25,17 @@ describe "Traject::MarcReader" do
     array = reader.to_a
 
     assert_equal 30, array.length
+
+    first = array.first
+
+    assert_kind_of MARC::Record, first
+
+    assert first['245']['a'].encoding.name, "UTF-8"
+    assert_equal "Fikr-i Ayāz /", first['245']['a']
   end
 
+  
+
 
 
 end