traject 3.1.0 → 3.6.0

data/test/command_line_test.rb

@@ -0,0 +1,52 @@
+# we mostly unit test with a Traject::Indexer itself and lower-level, but
+# we need at least some basic top-level integration actually command line tests,
+# this is a start, we can add more.
+#
+# Should we be testing Traject::CommandLine as an object instead of/in addition to
+# actually testing shell-out to command line call? Maybe.
+
+require 'test_helper'
+
+describe "Shell out to command line" do
+  # just encapsuluate using the minitest capture helper, but also
+  # getting and returning exit code
+  #
+  #     out, err, result = execute_with_args("-c configuration")
+  def execute_with_args(args)
+    out, err = capture_subprocess_io do
+      system("./bin/traject #{args}")
+    end
+
+    return out, err, $?
+  end
+
+  it "can display version" do
+    out, err, result = execute_with_args("-v")
+
+    assert result.success?, "Expected subprocess exit code to be success.\nSTDERR:\n#{err}\n\nSTDOUT:\n#{out}"
+    assert_equal err, "traject version #{Traject::VERSION}\n"
+  end
+
+  it "can display help text" do
+    out, err, result = execute_with_args("-h")
+
+    assert result.success?, "Expected subprocess exit code to be success.\nSTDERR:\n#{err}\n\nSTDOUT:\n#{out}"
+    assert err.start_with?("traject [options] -c configuration.rb [-c config2.rb] file.mrc")
+  end
+
+  it "handles bad argument" do
+    out, err, result = execute_with_args("--no-such-arg")
+    refute result.success?
+
+    assert err.start_with?("Error: unknown option `--no-such-arg'\nExiting...\n")
+  end
+
+  it "does basic dry run" do
+    out, err, result = execute_with_args("--debug-mode -s one=two -s three=four -c test/test_support/demo_config.rb test/test_support/emptyish_record.marc")
+
+    assert result.success?, "Expected subprocess exit code to be success.\nSTDERR:\n#{err}\n\nSTDOUT:\n#{out}"
+    assert_includes err, "executing with: `--debug-mode -s one=two -s three=four"
+    assert_match /bib_1000165 +author_sort +Collection la/, out
+  end
+end
+

data/test/debug_writer_test.rb

@@ -73,6 +73,19 @@ describe 'Simple output' do
 
   end
 
+  it "deals ok with nil values" do
+    record_with_nil_value = {"id"=>["2710183"], "title"=>["Manufacturing consent : the political economy of the mass media /"], "xyz"=>nil}
+    @writer.put Traject::Indexer::Context.new(:output_hash => record_with_nil_value)
+    expected = [
+      "#{@id} id #{@id}",
+      "#{@id} title #{@title}",
+      "#{@id} xyz",
+      "\n"
+    ]
+    assert_equal expected.join("\n").gsub(/\s/, ''), @io.string.gsub(/\s/, '')
+    @writer.close
+
+  end
 end
 
 

data/test/delimited_writer_test.rb

@@ -24,40 +24,40 @@ describe "Delimited/CSV Writers" do
 
     it "creates a dw with defaults" do
       dw = Traject::DelimitedWriter.new(@settings)
-      dw.delimiter.must_equal "\t"
-      dw.internal_delimiter.must_equal '|'
-      dw.edelim.must_equal ' '
-      dw.eidelim.must_equal '\\|'
+      assert_equal dw.delimiter, "\t"
+      assert_equal dw.internal_delimiter, '|'
+      assert_equal dw.edelim, ' '
+      assert_equal dw.eidelim, '\\|'
     end
 
     it "respects different delimiter" do
       @settings['delimited_writer.delimiter'] = '^'
       dw                                      = Traject::DelimitedWriter.new(@settings)
-      dw.delimiter.must_equal '^'
-      dw.edelim.must_equal '\\^'
-      dw.internal_delimiter.must_equal '|'
+      assert_equal dw.delimiter, '^'
+      assert_equal dw.edelim, '\\^'
+      assert_equal dw.internal_delimiter, '|'
     end
 
     it "outputs a header if asked to" do
       Traject::DelimitedWriter.new(@settings)
-      @out.string.chomp.must_equal %w[four one two].join("\t")
+      assert_equal @out.string.chomp, %w[four one two].join("\t")
     end
 
     it "doesn't output a header if asked not to" do
       @settings['delimited_writer.header'] = 'false'
       Traject::DelimitedWriter.new(@settings)
-      @out.string.must_be_empty
+      assert_empty @out.string
     end
 
     it "deals with multiple values" do
       dw = Traject::DelimitedWriter.new(@settings)
       dw.put @context
-      @out.string.split("\n").last.must_equal ['four', 'one', 'two1|two2'].join(dw.delimiter)
+      assert_equal @out.string.split("\n").last, ['four', 'one', 'two1|two2'].join(dw.delimiter)
     end
 
     it "bails if delimited_writer.fields isn't set" do
       @settings.delete 'delimited_writer.fields'
-      proc { Traject::DelimitedWriter.new(@settings) }.must_raise(ArgumentError)
+      assert_raises(ArgumentError)  { Traject::DelimitedWriter.new(@settings) }
     end
 
   end
@@ -65,18 +65,18 @@ describe "Delimited/CSV Writers" do
   describe "Traject::CSVWriter" do
     it "unsets the delimiter" do
       cw = Traject::CSVWriter.new(@settings)
-      cw.delimiter.must_be_nil
+      assert_nil cw.delimiter
     end
 
     it "writes the header" do
       Traject::CSVWriter.new(@settings)
-      @out.string.chomp.must_equal 'four,one,two'
+      assert_equal @out.string.chomp, 'four,one,two'
     end
 
     it "uses the internal delimiter" do
       cw = Traject::CSVWriter.new(@settings)
       cw.put @context
-      @out.string.split("\n").last.must_equal ['four', 'one', 'two1|two2'].join(',')
+      assert_equal @out.string.split("\n").last, ['four', 'one', 'two1|two2'].join(',')
     end
 
     it "produces complex output" do
@@ -97,8 +97,6 @@ describe "Delimited/CSV Writers" do
       traject_csvwriter_output = @out.string.split("\n").last.chomp
 
       assert_equal(csv_output, traject_csvwriter_output)
-
     end
-
   end
 end

data/test/indexer/class_level_configuration_test.rb

@@ -25,6 +25,7 @@ describe "Class-level configuration of Indexer sub-class" do
     end
   end
 
+
   before do
     @indexer = TestIndexerSubclass.new
   end
@@ -47,6 +48,28 @@ describe "Class-level configuration of Indexer sub-class" do
     assert_equal ['from-instance-config'], result["instance_field"]
   end
 
+  describe "multiple class-level configure" do
+    class MultipleConfigureIndexer < Traject::Indexer
+      configure do
+        to_field "field", literal("value")
+      end
+      configure do
+        to_field "field", literal("value from second configure")
+        to_field "second_call", literal("value from second configure")
+      end
+    end
+
+    before do
+      @indexer = MultipleConfigureIndexer.new
+    end
+
+    it "lets you call class-level configure multiple times and aggregates" do
+      result = @indexer.map_record(Object.new)
+      assert_equal ['value', 'value from second configure'], result['field']
+      assert_equal ['value from second configure'], result['second_call']
+    end
+  end
+
   describe "with multi-level subclass" do
     class TestIndexerSubclassSubclass < TestIndexerSubclass
       configure do

data/test/indexer/macros/macros_marc21_semantics_test.rb

@@ -197,6 +197,10 @@ describe "Traject::Macros::Marc21Semantics" do
       # we take the first date. And need to deal with the u.
       assert_equal 1845, Marc21Semantics.publication_date(@record)
     end
+    it "resorts to 264c" do
+      @record = MARC::Reader.new(support_file_path  "date_resort_to_264.marc").to_a.first
+      assert_equal 2015, Marc21Semantics.publication_date(@record)
+    end
     it "resorts to 260c" do
       @record = MARC::Reader.new(support_file_path  "date_resort_to_260.marc").to_a.first
       assert_equal 1980, Marc21Semantics.publication_date(@record)

data/test/indexer/nokogiri_indexer_test.rb

@@ -109,6 +109,41 @@ describe "Traject::NokogiriIndexer" do
         result["name"].name == "name"
       })
     end
+  end
 
+  describe "xpath to attribute" do
+    let(:indexer) do
+      namespaces = @namespaces
+      Traject::Indexer::NokogiriIndexer.new("nokogiri.namespaces" => namespaces,
+                                            "nokogiri.each_record_xpath" => "//oai:record") do
+        to_field "status", extract_xpath("//oai:record/oai:header/@status")
+      end
+    end
+
+    let(:records) { Traject::NokogiriReader.new(StringIO.new(
+                                                      <<-XML
+      <?xml version="1.0" encoding="UTF-8"?>
+      <OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
+        <responseDate>2020-03-03T04:16:09Z</responseDate>
+        <request verb="ListRecords" metadataPrefix="marc21" set="blacklight" from="2020-03-02T20:47:11Z">https://na02.alma.exlibrisgroup.com/view/oai/01TULI_INST/request</request>
+        <ListRecords>
+          <record>
+            <header status="deleted">
+              <identifier>oai:alma.01TULI_INST:991025803889703811</identifier>
+              <datestamp>2020-03-03T03:54:35Z</datestamp>
+              <setSpec>blacklight</setSpec>
+              <setSpec>rapid_print_journals</setSpec>
+              <setSpec>blacklight_qa</setSpec>
+            </header>
+          </record>
+        </ListRecords>
+      </OAI-PMH>
+          XML
+    ), []).to_a }
+
+    it "extracts the correct attribute" do
+      statuses = indexer.map_record(records.first)["status"]
+      assert_equal ["deleted"], statuses
+    end
   end
 end

data/test/indexer/read_write_test.rb

@@ -7,7 +7,8 @@ memory_writer_class = Class.new do
       # store them in a class variable so we can test em later
       # Supress the warning message
       original_verbose, $VERBOSE = $VERBOSE, nil
-      @@last_writer_settings = @settings = settings
+      @settings = settings
+      self.class.store_last_writer_settings(@settings)
       # Activate warning messages again.
       $VERBOSE = original_verbose
       @settings["memory_writer.added"] = []
@@ -20,6 +21,16 @@ memory_writer_class = Class.new do
     def close
       @settings["memory_writer.closed"] = true
     end
+
+    private
+
+    def self.store_last_writer_settings(settings)
+      @last_writer_settings = settings
+    end
+
+    def self.last_writer_settings
+      @last_writer_settings
+    end
   end
 
 describe "Traject::Indexer#process" do
@@ -53,7 +64,7 @@ describe "Traject::Indexer#process" do
 
     # Grab the settings out of a class variable where we left em,
     # as a convenient place to store outcomes so we can test em.
-    writer_settings = memory_writer_class.class_variable_get("@@last_writer_settings")
+    writer_settings = memory_writer_class.last_writer_settings
 
     assert writer_settings["memory_writer.added"]
     assert_equal 30, writer_settings["memory_writer.added"].length
@@ -146,7 +157,7 @@ describe "Traject::Indexer#process" do
     it "parses and loads" do
       @indexer.process([@file1, @file2])
       # kinda ridic, yeah.
-      output_hashes = memory_writer_class.class_variable_get("@@last_writer_settings")["memory_writer.added"].collect(&:output_hash)
+      output_hashes = memory_writer_class.last_writer_settings["memory_writer.added"].collect(&:output_hash)
 
       assert_length 2, output_hashes
       assert output_hashes.all? { |hash| hash["title"].length > 0 }

data/test/nokogiri_reader_test.rb

@@ -134,6 +134,16 @@ describe "Traject::NokogiriReader" do
     end
   end
 
+  describe "strict_mode" do
+    it "raises on non-well-formed" do
+      # invalid because two sibling root nodes, XML requiers one root node
+      reader = Traject::NokogiriReader.new(StringIO.new("<doc></doc><doc></doc>"), {"nokogiri.strict_mode" => "true" })
+      assert_raises(Nokogiri::XML::SyntaxError) {
+        reader.each { |r| }
+      }
+    end
+  end
+
 
   def shared_tests
     @reader = Traject::NokogiriReader.new(File.open(@xml_sample_path), {

data/test/solr_json_writer_test.rb

@@ -170,6 +170,62 @@ describe "Traject::SolrJsonWriter" do
     assert_length 1, @fake_http_client.post_args, "Has flushed to solr"
   end
 
+  it "defaults to not setting basic authentication" do
+    settings = { "solr.url" => "http://example.com/solr/foo" }
+    writer = Traject::SolrJsonWriter.new(settings)
+    auth = writer.instance_variable_get("@http_client")
+      .www_auth.basic_auth.instance_variable_get("@auth")
+    assert(auth.empty?)
+  end
+
+  describe "HTTP basic auth" do
+
+    it "supports basic authentication settings" do
+      settings = {
+        "solr.url" => "http://example.com/solr/foo",
+        "solr_writer.basic_auth_user" => "foo",
+        "solr_writer.basic_auth_password" => "bar",
+      }
+
+      # testing with some internal implementation of HTTPClient sorry
+
+      writer = Traject::SolrJsonWriter.new(settings)
+
+      auth = writer.instance_variable_get("@http_client")
+        .www_auth.basic_auth.instance_variable_get("@auth")
+      assert(!auth.empty?)
+      assert_equal(auth.values.first, Base64.encode64("foo:bar").chomp)
+    end
+
+    it "supports basic auth from solr.url" do
+      settings = {
+        "solr.url" => "http://foo:bar@example.com/solr/foo",
+      }
+
+      # testing with some internal implementation of HTTPClient sorry
+
+      writer = Traject::SolrJsonWriter.new(settings)
+      auth = writer.instance_variable_get("@http_client")
+        .www_auth.basic_auth.instance_variable_get("@auth")
+      assert(!auth.empty?)
+      assert_equal(auth.values.first, Base64.encode64("foo:bar").chomp)
+    end
+
+    it "does not log basic auth from solr.url" do
+      string_io = StringIO.new
+      settings = {
+        "solr.url" => "http://secret_username:secret_password@example.com/solr/foo",
+        "logger"   => Logger.new(string_io)
+      }
+
+
+      writer = Traject::SolrJsonWriter.new(settings)
+
+      refute_includes string_io.string, "secret_username:secret_password"
+      assert_includes string_io.string, "(with HTTP basic auth)"
+    end
+  end
+
   describe "commit" do
     it "commits on close when set" do
       @writer = create_writer("solr.url" => "http://example.com", "solr_writer.commit_on_close" => "true")
@@ -365,4 +421,13 @@ describe "Traject::SolrJsonWriter" do
       end
     end
   end
+
+  describe "#delete_all!" do
+    it "deletes all" do
+      @writer.delete_all!
+      post_args = @fake_http_client.post_args.first
+      assert_equal "http://example.com/solr/update/json", post_args[0]
+      assert_equal JSON.generate({"delete" => { "query" => "*:*"}}), post_args[1]
+    end
+  end
 end

data/test/test_support/date_resort_to_264.marc

@@ -0,0 +1 @@
+01180aam a2200337 a 4500001001000000008004100010015001900051016001800070020002500088020002200113040005900135043001200194050002400206082001400230100003500244245006400279260003800343264003800381264001100419300003600430336002100466336002800487337002500515338002300540504006700563651004000630651005000670651004300720651005600763035002300819a11417842130723t20uu20uuenkb     b    001 0 eng d  aGBB3854302bnb7 a0164999372Uk  a9781849043427 (pbk.)  a1849043426 (pbk.)  aUKMGBcUKMGBdOCLCOdYDXCPdOCLCOdZWZdOCLCOdCaONFJC  aa-ii--- 4aDS485.K25bS64 201504a954.62231 aSnedden, Christopher,eauthor.10aUnderstanding Kashmir and Kashmiris /cChristopher Snedden. 1aLondon :bHurst & Company,c2014. 1aLondon :bHurst & Company,c2015. 4c©2015  axix, 372 pages :bmaps ;c22 cm  atext2rdacontent  astill image2rdacontent  aunmediated2rdamedia  avolume2rdacarrier  aIncludes bibliographical references (pages 331-355) and index. 0aJammu and Kashmir (India)xHistory. 0aJammu and Kashmir (India)xForeign relations. 0aJammu and Kashmir (India)vBoundaries. 0aJammu and Kashmir (India)xPolitics and government.  a(OCoLC-M)858826393

data/traject.gemspec

@@ -24,12 +24,12 @@ Gem::Specification.new do |spec|
   spec.add_dependency "concurrent-ruby", ">= 0.8.0"
   spec.add_dependency "marc", "~> 1.0"
 
-  spec.add_dependency "hashie", "~> 3.1" # used for Indexer#settings
-  spec.add_dependency "slop", ">= 3.4.5", "< 4.0" # command line parsing
+  spec.add_dependency "hashie", ">= 3.1", "< 5" # used for Indexer#settings
+  spec.add_dependency "slop", "~> 4.0" # command line parsing
   spec.add_dependency "yell" # logging
   spec.add_dependency "dot-properties", ">= 0.1.1" # reading java style .properties
   spec.add_dependency "httpclient", "~> 2.5"
-  spec.add_dependency "http", "~> 3.0" # used in oai_pmh_reader, may use more extensively in future instead of httpclient
+  spec.add_dependency "http", ">= 3.0", "< 6" # used in oai_pmh_reader, may use more extensively in future instead of httpclient
   spec.add_dependency 'marc-fastxmlwriter', '~>1.0' # fast marc->xml
   spec.add_dependency "nokogiri", "~> 1.9" # NokogiriIndexer
 

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: traject
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.6.0
 platform: ruby
 authors:
 - Jonathan Rochkind
 - Bill Dueber
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-04-22 00:00:00.000000000 Z
+date: 2021-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -43,34 +43,34 @@ dependencies:
   name: hashie
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '3.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '3.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5'
 - !ruby/object:Gem::Dependency
   name: slop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.4.5
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.4.5
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
 - !ruby/object:Gem::Dependency
@@ -119,16 +119,22 @@ dependencies:
   name: http
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: marc-fastxmlwriter
   requirement: !ruby/object:Gem::Requirement
@@ -219,7 +225,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.4'
-description: 
+description:
 email:
 - none@nowhere.org
 executables:
@@ -234,8 +240,8 @@ extra_rdoc_files:
 - doc/settings.md
 - doc/xml.md
 files:
+- ".github/workflows/ruby.yml"
 - ".gitignore"
-- ".travis.yml"
 - ".yardopts"
 - CHANGES.md
 - Gemfile
@@ -251,6 +257,7 @@ files:
 - doc/programmatic_use.md
 - doc/settings.md
 - doc/xml.md
+- examples/marc/tiny.xml
 - lib/tasks/load_maps.rake
 - lib/traject.rb
 - lib/traject/array_writer.rb
@@ -295,6 +302,7 @@ files:
 - lib/translation_maps/marc_geographic.yaml
 - lib/translation_maps/marc_instruments.yaml
 - lib/translation_maps/marc_languages.yaml
+- test/command_line_test.rb
 - test/debug_writer_test.rb
 - test/delimited_writer_test.rb
 - test/experimental_nokogiri_streaming_reader_test.rb
@@ -330,6 +338,7 @@ files:
 - test/test_support/bad_subfield_code.marc
 - test/test_support/bad_utf_byte.utf8.marc
 - test/test_support/date_resort_to_260.marc
+- test/test_support/date_resort_to_264.marc
 - test/test_support/date_type_r_missing_date2.marc
 - test/test_support/date_with_u.marc
 - test/test_support/demo_config.rb
@@ -377,7 +386,7 @@ homepage: http://github.com/traject/traject
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -392,13 +401,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.0.3
+signing_key:
 specification_version: 4
 summary: An easy to use, high-performance, flexible and extensible metadata transformation
   system, focused on library-archives-museums input, and indexing to Solr as output.
 test_files:
+- test/command_line_test.rb
 - test/debug_writer_test.rb
 - test/delimited_writer_test.rb
 - test/experimental_nokogiri_streaming_reader_test.rb
@@ -434,6 +443,7 @@ test_files:
 - test/test_support/bad_subfield_code.marc
 - test/test_support/bad_utf_byte.utf8.marc
 - test/test_support/date_resort_to_260.marc
+- test/test_support/date_resort_to_264.marc
 - test/test_support/date_type_r_missing_date2.marc
 - test/test_support/date_with_u.marc
 - test/test_support/demo_config.rb