traject 2.3.4 → 3.0.0.alpha.1

data/test/indexer/read_write_test.rb

@@ -25,7 +25,7 @@ memory_writer_class = Class.new do
 describe "Traject::Indexer#process" do
   before do
     # no threading for these tests
-    @indexer = Traject::Indexer.new("processing_thread_pool" => nil)
+    @indexer = Traject::Indexer::MarcIndexer.new("processing_thread_pool" => nil)
     @indexer.writer_class = memory_writer_class
     @file = File.open(support_file_path "test_data.utf8.mrc")
   end
@@ -68,7 +68,7 @@ describe "Traject::Indexer#process" do
 
   require 'traject/null_writer'
   it "calls after_processing after processing" do
-    @indexer = Traject::Indexer.new(
+    @indexer = Traject::Indexer::MarcIndexer.new(
       "writer_class_name" => "Traject::NullWriter"
     )
     @file = File.open(support_file_path "test_data.utf8.mrc")
@@ -87,6 +87,37 @@ describe "Traject::Indexer#process" do
     assert_equal [:one, :two], called, "Both after_processing hooks called, in order"
   end
 
+  it "calls after_processing from #run_after_processing_steps" do
+    @indexer = Traject::Indexer.new(
+      "writer_class_name" => "Traject::NullWriter"
+    )
+    @file = File.open(support_file_path "test_data.utf8.mrc")
+
+    called = []
+
+    @indexer.after_processing do
+      called << :one
+    end
+    @indexer.after_processing do
+      called << :two
+    end
+
+    @indexer.run_after_processing_steps
+    assert_equal [:one, :two], called, "Both after_processing hooks called, in order"
+  end
+
+  it "can't be run twice" do
+    @file = File.open(support_file_path "test_data.utf8.mrc")
+    @indexer = Traject::Indexer::MarcIndexer.new(
+      "writer_class_name" => "Traject::NullWriter"
+    )
+    @indexer.process(@file)
+
+    assert_raises Traject::Indexer::CompletedStateError do
+      @indexer.process(@file)
+    end
+  end
+
   describe "demo_config.rb" do
     before do
       @indexer = Traject::Indexer.new(
@@ -102,4 +133,23 @@ describe "Traject::Indexer#process" do
     end
   end
 
+  describe "multi stream" do
+    before do
+      @file2 = File.open(support_file_path "george_eliot.marc")
+      @file1 = File.open(support_file_path "musical_cage.marc")
+      @indexer = Traject::Indexer::MarcIndexer.new do
+        self.writer_class = memory_writer_class
+        to_field "title", extract_marc("245")
+      end
+    end
+
+    it "parses and loads" do
+      @indexer.process([@file1, @file2])
+      # kinda ridic, yeah.
+      output_hashes = memory_writer_class.class_variable_get("@@last_writer_settings")["memory_writer.added"].collect(&:output_hash)
+
+      assert_length 2, output_hashes
+      assert output_hashes.all? { |hash| hash["title"].length > 0 }
+    end
+  end
 end

data/test/indexer/settings_test.rb

@@ -5,10 +5,10 @@ describe "Traject::Indexer#settings" do
     @indexer = Traject::Indexer.new
   end
 
-  it "starts out a Hash, that can fill in it's defaults" do
+  it "starts out a Hash, that uses it's defaults" do
     assert_kind_of Hash, @indexer.settings
 
-    Traject::Indexer::Settings.defaults.each_pair do |key, value|
+    Traject::Indexer.default_settings.each_pair do |key, value|
       assert_equal value, @indexer.settings[key]
     end
   end
@@ -16,13 +16,15 @@ describe "Traject::Indexer#settings" do
   it "can fill_in_defaults!" do
     @indexer.settings.fill_in_defaults!
 
-    assert_equal Traject::Indexer::Settings.defaults, @indexer.settings
+    assert_equal Traject::Indexer.default_settings, @indexer.settings
   end
 
   it "doesn't overwrite with fill_in_defaults!" do
-    key = Traject::Indexer::Settings.defaults.keys.first
+    key = Traject::Indexer.default_settings.keys.first
     @indexer.settings[ key  ] = "MINE KEEP IT"
 
+    assert_equal "MINE KEEP IT", @indexer.settings[key]
+
     @indexer.settings.fill_in_defaults!
 
     assert_equal "MINE KEEP IT", @indexer.settings[key]
@@ -36,7 +38,7 @@ describe "Traject::Indexer#settings" do
   end
 
   it "has settings DSL to set" do
-    @indexer.instance_eval do
+    @indexer.configure do
       settings do
         store "foo", "foo"
       end
@@ -124,28 +126,36 @@ describe "Traject::Indexer#settings" do
       assert_equal( {"a" => "a", "password" => "[hidden]", "some_password" => "[hidden]", "some.password" => "[hidden]"}, parsed)
     end
   end
-  
-  describe "JRuby / MRI" do
-    before do
-      @indexer = Traject::Indexer.new
-    end
-    
-    it "has the right indexer name" do
-      if defined? JRUBY_VERSION
-        assert_equal "Traject::Marc4JReader", @indexer.settings['reader_class_name']
-      else
-        assert_equal "Traject::MarcReader", @indexer.settings['reader_class_name']
+
+  describe "order of precedence" do
+    it "args beat 'provides'" do
+      # args come from command-line in typical use
+
+      @indexer = Traject::Indexer.new(sample: "from args")
+      @indexer.settings do
+        provide :sample, "from config"
       end
+      @indexer.settings.fill_in_defaults!
+
+      assert_equal "from args", @indexer.settings["sample"]
     end
-    
-    # This next one has the added effect of making sure the correct class
-    # has actually been loaded -- otherwise the constant wouldn't be available
-    it "has the correct default indexer class based on platform" do
-      if defined? JRUBY_VERSION
-        assert_equal Traject::Marc4JReader, @indexer.reader_class
-      else
-        assert_equal Traject::MarcReader, @indexer.reader_class
+
+    it "args beat defaults" do
+      key = Traject::Indexer.default_settings.keys.first
+      @indexer = Traject::Indexer.new(key.to_sym => "from args")
+      @indexer.settings.fill_in_defaults!
+
+      assert_equal "from args", @indexer.settings[key]
+    end
+
+    it "provide beats defaults" do
+      key = Traject::Indexer.default_settings.keys.first
+      @indexer.settings do
+        provide key, "from config"
       end
+      @indexer.settings.fill_in_defaults!
+
+      assert_equal "from config", @indexer.settings[key]
     end
   end
 

data/test/indexer/to_field_test.rb

@@ -69,9 +69,34 @@ describe "Traject::Indexer.to_field" do
     assert_equal ['hello'], output['foo']
   end
 
+  describe "supports multiple procs" do
+    it "with no block" do
+      @indexer.to_field "foo",
+        lambda {|record, acc| acc << "one"},
+        lambda {|record, acc| acc << "two"},
+        lambda {|record, acc| acc << "three"}
 
-end
-
+      output = @indexer.map_record('never looked at')
+      assert_equal ['one', 'two', 'three'], output['foo']
+    end
 
+    it "with a block too" do
+      @indexer.to_field "foo",
+        lambda {|record, acc| acc << "one"},
+        lambda {|record, acc| acc << "two"} do |record, acc|
+          acc << "three"
+      end
 
+      output = @indexer.map_record('never looked at')
+      assert_equal ['one', 'two', 'three'], output['foo']
+    end
+  end
 
+  describe "with an array argument" do
+    it "indexes to multiple fields" do
+      @indexer.to_field ["field1", "field2", "field3"], lambda {|rec, acc| acc << "value" }
+      output = @indexer.map_record('never looked at')
+      assert_equal({ "field1" => ["value"], "field2" => ["value"], "field3" => ["value"] }, output)
+    end
+  end
+end

data/test/marc_extractor_test.rb

@@ -28,7 +28,7 @@ describe "Traject::MarcExtractor" do
 
       assert_kind_of Array, spec.subfields
     end
-    
+
     it "parses specset from an array" do
       parsed  = Traject::MarcExtractor::SpecSet.new(%w[245abcde 810 700|*4|bcd])
       assert_equal parsed.tags, %w[245 810 700]
@@ -60,17 +60,17 @@ describe "Traject::MarcExtractor" do
       assert_equal "4", spec700.indicator2
       assert_equal %w{b c d}, spec700.subfields
     end
-    
+
     it "parses from an array" do
       parsed  = Traject::MarcExtractor::Spec.hash_from_string(%w[245abcde 810 700|*4|bcd])
-      spec245 = parsed['245'].first
-      spec810 = parsed['810'].first
-      spec700 = parsed['700'].first
+      _spec245 = parsed['245'].first
+      _spec810 = parsed['810'].first
+      _spec700 = parsed['700'].first
 
       assert_length 3, parsed
     end
-      
-      
+
+
 
     it "parses fixed field byte offsets" do
       parsed = Traject::MarcExtractor::Spec.hash_from_string("005[5]:008[7-10]")

data/test/marc_reader_test.rb

@@ -50,13 +50,13 @@ describe "Traject::MarcReader" do
       a245a = array.first['245']['a']
 
       assert a245a.encoding.name, "UTF-8"
-      assert a245a.valid_encoding?    
+      assert a245a.valid_encoding?
       assert_equal "Por uma outra globalização :", a245a
     end
 
     it "replaces unicode character reference in Marc8 transcode" do
       file = File.new(support_file_path("escaped_character_reference.marc8.marc"))
-      
+
       settings = Traject::Indexer::Settings.new("marc_source.encoding" => "MARC-8") # binary type is default
       record = Traject::MarcReader.new(file, settings).to_a.first
 
@@ -67,7 +67,7 @@ describe "Traject::MarcReader" do
       file = File.new(support_file_path "one-marc8.mrc")
       settings = Traject::Indexer::Settings.new("marc_source.encoding" => "ADFADFADF")
       assert_raises(ArgumentError) do
-        record = Traject::MarcReader.new(file, settings).to_a.first
+        _record = Traject::MarcReader.new(file, settings).to_a.first
       end
     end
 
@@ -78,7 +78,7 @@ describe "Traject::MarcReader" do
       reader = Traject::MarcReader.new(file, settings)
 
       record = reader.to_a.first
-      
+
       value = record['300']['a']
 
       assert_equal value.encoding.name, "UTF-8"

data/test/nokogiri_reader_test.rb

@@ -0,0 +1,158 @@
+require 'test_helper'
+require 'traject/nokogiri_reader'
+
+describe "Traject::NokogiriReader" do
+  describe "with namespaces" do
+    before do
+      @namespaces = { "oai" => "http://www.openarchives.org/OAI/2.0/" }
+      @xml_sample_path = support_file_path("sample-oai-pmh.xml")
+    end
+
+    describe "invalid settings" do
+      it "default_namespaces not a hash raises" do
+        error = assert_raises(ArgumentError) {
+          @reader = Traject::NokogiriReader.new(File.open(@xml_sample_path), {
+            "nokogiri.namespaces" => "i am not a hash",
+          })
+        }
+        assert(error.message =~ /nokogiri.namespaces must be a hash/)
+      end
+
+      it "each_record_xpath with unregistered prefix raises" do
+        error = assert_raises(ArgumentError) {
+          @reader = Traject::NokogiriReader.new(File.open(@xml_sample_path), {
+            "nokogiri.namespaces" => @namespaces,
+            "nokogiri.each_record_xpath" => "//foo:bar"
+          })
+        }
+        assert(error.message =~ %r{Can't find namespace prefix 'foo' in '//foo:bar'})
+      end
+    end
+
+    describe "fixed path" do
+      before do
+        @each_record_xpath = "/oai:OAI-PMH/oai:ListRecords/oai:record"
+      end
+
+      it "reads" do
+        shared_tests
+      end
+    end
+
+    describe "floating path" do
+      before do
+        @each_record_xpath = "//oai:record"
+      end
+
+      it "reads" do
+        shared_tests
+      end
+    end
+
+
+    describe "extra_xpath_hooks" do
+      it "catches oai-pmh resumption token" do
+        @reader = Traject::NokogiriReader.new(File.open(@xml_sample_path), {
+          "nokogiri.namespaces" => @namespaces,
+          "nokogiri.each_record_xpath" => "//oai:record",
+          "nokogiri_reader.extra_xpath_hooks" => {
+            "//oai:resumptionToken" => lambda do |node, clipboard|
+              clipboard[:resumptionToken] = node.text
+            end
+          }
+        })
+        _records = @reader.to_a
+        assert_equal "oai_dc.f(2018-05-03T18:09:08Z).u(2018-06-15T19:25:21Z).t(6387):100", @reader.clipboard[:resumptionToken]
+      end
+    end
+
+    describe "outer namespaces" do
+      it "are preserved" do
+        @reader = Traject::NokogiriReader.new(File.open(support_file_path("namespace-test.xml")), {
+          "nokogiri.namespaces" => { mytop: "http://example.org/top" },
+          "nokogiri.each_record_xpath" => "//mytop:record"
+        })
+        yielded_records = []
+        @reader.each { |record|
+          yielded_records << record
+        }
+
+        assert yielded_records.length > 0
+
+        expected_namespaces = {"xmlns"=>"http://example.org/top", "xmlns:a"=>"http://example.org/a", "xmlns:b"=>"http://example.org/b"}
+        yielded_records.each do |rec|
+          assert_equal expected_namespaces, rec.namespaces
+        end
+      end
+    end
+  end
+
+  describe "without namespaces" do
+    before do
+      @namespaces = {}
+      @xml_sample_path = support_file_path("sample-oai-no-namespace.xml")
+    end
+
+    describe "fixed path" do
+      before do
+        @each_record_xpath = "/OAI-PMH/ListRecords/record"
+      end
+
+      it "reads" do
+        shared_tests
+      end
+    end
+
+    describe "floating path" do
+      before do
+        @each_record_xpath = "//record"
+      end
+
+      it "reads" do
+        shared_tests
+      end
+    end
+  end
+
+
+  def shared_tests
+    @reader = Traject::NokogiriReader.new(File.open(@xml_sample_path), {
+      "nokogiri.namespaces" => @namespaces,
+      "nokogiri.each_record_xpath" => @each_record_xpath
+    })
+
+    yielded_records = []
+    @reader.each { |record|
+      yielded_records << record
+    }
+
+
+    manually_extracted = Nokogiri::XML.parse(File.open(@xml_sample_path)).xpath(@each_record_xpath, @namespaces)
+    manually_extracted.collect do |node|
+      # nokogiri makes it so hard to reliably get an Element to serialize to XML with all
+      # it's inherited namespace declerations. :(  We're only doing this for testing purposes
+      # anyway.  This may not handle everything, but handles what we need in the test right now
+      if node.namespace
+        node["xmlns"] = node.namespace.href
+      end
+    end
+
+    assert_length manually_extracted.size, yielded_records
+    assert yielded_records.all? {|r| r.kind_of? Nokogiri::XML::Document }
+    assert_equal manually_extracted.collect(&:to_xml), yielded_records.collect(&:root).collect(&:to_xml)
+  end
+
+  describe "without each_record_xpath" do
+    before do
+      @xml_sample_path = support_file_path("namespace-test.xml")
+    end
+    it "yields whole file as one record" do
+      @reader = Traject::NokogiriReader.new(File.open(@xml_sample_path), {})
+
+      yielded_records = @reader.to_a
+
+      assert_length 1, yielded_records
+      assert_equal Nokogiri::XML.parse(File.open(@xml_sample_path)).to_xml, yielded_records.first.to_xml
+    end
+  end
+end

data/test/oai_pmh_nokogiri_reader_test.rb

@@ -0,0 +1,23 @@
+require 'test_helper'
+require 'traject/oai_pmh_nokogiri_reader'
+
+describe "Traject::OaiPmhNokogiriReader" do
+
+  it "smoke test" do
+    @reader = Traject::OaiPmhNokogiriReader.new(nil,
+      "oai_pmh.start_url" => "http://example.com/oai?verb=ListRecords&metadataPrefix=oai_dc"
+    )
+
+    fetched = @reader.to_a
+
+    assert_length 2, fetched
+  end
+
+  before do
+    stub_request(:get, "http://example.com/oai?metadataPrefix=oai_dc&verb=ListRecords").
+      to_return(status: 200, body: File.read(support_file_path("oai-pmh-one-record-first.xml")))
+
+    stub_request(:get, "http://example.com/oai?resumptionToken=dummy_resumption&verb=ListRecords").
+      to_return(status: 200, body: File.read(support_file_path("oai-pmh-one-record-2.xml")))
+  end
+end